From 723717acfb0895397791f235980fffb805c085b1 Mon Sep 17 00:00:00 2001 From: Christophe Duong Date: Tue, 9 Nov 2021 19:41:18 +0100 Subject: [PATCH 1/6] Change OAuth API --- airbyte-api/src/main/openapi/config.yaml | 31 +++++++++++-- .../airbyte/server/apis/ConfigurationApi.java | 7 +-- .../airbyte/server/handlers/OAuthHandler.java | 17 ++++--- .../api/generated-api-html/index.html | 46 +++++++++++++++++-- 4 files changed, 83 insertions(+), 18 deletions(-) diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index ef75e61ba9521..5947a69470f5e 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -1311,7 +1311,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/CompleteOauthResponse" + $ref: "#/components/schemas/CompleteSourceOauthResponse" "404": $ref: "#/components/responses/NotFoundResponse" "422": @@ -1357,7 +1357,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/CompleteOauthResponse" + $ref: "#/components/schemas/CompleteDestinationOauthResponse" "404": $ref: "#/components/responses/NotFoundResponse" "422": @@ -3227,6 +3227,8 @@ components: items: $ref: "#/components/schemas/DbMigrationRead" # OAuth + AuthConfiguration: + description: The values required to configure OAuth flows. The schema for this must match the authSpecification schema. SourceOauthConsentRequest: type: object required: @@ -3241,6 +3243,8 @@ components: redirectUrl: description: The url to redirect to after getting the user consent type: string + authConfiguration: + $ref: "#/components/schemas/AuthConfiguration" DestinationOauthConsentRequest: type: object required: @@ -3255,6 +3259,8 @@ components: redirectUrl: description: The url to redirect to after getting the user consent type: string + authConfiguration: + $ref: "#/components/schemas/AuthConfiguration" OAuthConsentRead: type: object required: @@ -3279,6 +3285,8 @@ components: description: The query parameters present in the redirect URL after a user granted consent e.g auth code type: object additionalProperties: true # Oauth parameters like code, state, etc.. will be different per API so we don't specify them in advance + authConfiguration: + $ref: "#/components/schemas/AuthConfiguration" CompleteDestinationOAuthRequest: type: object required: @@ -3296,9 +3304,24 @@ components: description: The query parameters present in the redirect URL after a user granted consent e.g auth code type: object additionalProperties: true # Oauth parameters like code, state, etc.. will be different per API so we don't specify them in advance - CompleteOauthResponse: + authConfiguration: + $ref: "#/components/schemas/AuthConfiguration" + CompleteSourceOauthResponse: type: object - additionalProperties: true # Oauth parameters like refresh/access token etc.. will be different per API so we don't specify them in advance + properties: + oauthOutput: + type: object + additionalProperties: true # Oauth parameters like refresh/access token etc.. will be different per API so we don't specify them in advance + authConfiguration: + $ref: "#/components/schemas/AuthConfiguration" + CompleteDestinationOauthResponse: + type: object + properties: + oauthOutput: + type: object + additionalProperties: true # Oauth parameters like refresh/access token etc.. will be different per API so we don't specify them in advance + authConfiguration: + $ref: "#/components/schemas/AuthConfiguration" SetInstancewideSourceOauthParamsRequestBody: type: object required: diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 2820c89b77040..7f5309314bf91 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -8,7 +8,9 @@ import io.airbyte.api.model.CheckConnectionRead; import io.airbyte.api.model.CheckOperationRead; import io.airbyte.api.model.CompleteDestinationOAuthRequest; +import io.airbyte.api.model.CompleteDestinationOauthResponse; import io.airbyte.api.model.CompleteSourceOauthRequest; +import io.airbyte.api.model.CompleteSourceOauthResponse; import io.airbyte.api.model.ConnectionCreate; import io.airbyte.api.model.ConnectionIdRequestBody; import io.airbyte.api.model.ConnectionRead; @@ -122,7 +124,6 @@ import java.io.IOException; import java.net.http.HttpClient; import java.nio.file.Path; -import java.util.Map; @javax.ws.rs.Path("/v1") public class ConfigurationApi implements io.airbyte.api.V1Api { @@ -300,7 +301,7 @@ public OAuthConsentRead getSourceOAuthConsent(final SourceOauthConsentRequest so } @Override - public Map completeSourceOAuth(final CompleteSourceOauthRequest completeSourceOauthRequest) { + public CompleteSourceOauthResponse completeSourceOAuth(final CompleteSourceOauthRequest completeSourceOauthRequest) { return execute(() -> oAuthHandler.completeSourceOAuth(completeSourceOauthRequest)); } @@ -310,7 +311,7 @@ public OAuthConsentRead getDestinationOAuthConsent(final DestinationOauthConsent } @Override - public Map completeDestinationOAuth(final CompleteDestinationOAuthRequest requestBody) { + public CompleteDestinationOauthResponse completeDestinationOAuth(final CompleteDestinationOAuthRequest requestBody) { return execute(() -> oAuthHandler.completeDestinationOAuth(requestBody)); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java index 49143c76ca2f4..4fcc01b62221f 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java @@ -7,7 +7,9 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.analytics.TrackingClient; import io.airbyte.api.model.CompleteDestinationOAuthRequest; +import io.airbyte.api.model.CompleteDestinationOauthResponse; import io.airbyte.api.model.CompleteSourceOauthRequest; +import io.airbyte.api.model.CompleteSourceOauthResponse; import io.airbyte.api.model.DestinationOauthConsentRequest; import io.airbyte.api.model.OAuthConsentRead; import io.airbyte.api.model.SetInstancewideDestinationOauthParamsRequestBody; @@ -26,7 +28,6 @@ import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.net.http.HttpClient; -import java.util.Map; import java.util.UUID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,17 +84,18 @@ public OAuthConsentRead getDestinationOAuthConsent(final DestinationOauthConsent return result; } - public Map completeSourceOAuth(final CompleteSourceOauthRequest oauthSourceRequestBody) + public CompleteSourceOauthResponse completeSourceOAuth(final CompleteSourceOauthRequest oauthSourceRequestBody) throws JsonValidationException, ConfigNotFoundException, IOException { final StandardSourceDefinition sourceDefinition = configRepository.getStandardSourceDefinition(oauthSourceRequestBody.getSourceDefinitionId()); final OAuthFlowImplementation oAuthFlowImplementation = oAuthImplementationFactory.create(sourceDefinition, oauthSourceRequestBody.getWorkspaceId()); final ImmutableMap metadata = generateSourceMetadata(oauthSourceRequestBody.getSourceDefinitionId()); - final Map result = oAuthFlowImplementation.completeSourceOAuth( + final CompleteSourceOauthResponse result = new CompleteSourceOauthResponse(); + result.oauthOutput(oAuthFlowImplementation.completeSourceOAuth( oauthSourceRequestBody.getWorkspaceId(), oauthSourceRequestBody.getSourceDefinitionId(), oauthSourceRequestBody.getQueryParams(), - oauthSourceRequestBody.getRedirectUrl()); + oauthSourceRequestBody.getRedirectUrl())); try { trackingClient.track(oauthSourceRequestBody.getWorkspaceId(), "Complete OAuth Flow - Backend", metadata); } catch (final Exception e) { @@ -102,18 +104,19 @@ public Map completeSourceOAuth(final CompleteSourceOauthRequest return result; } - public Map completeDestinationOAuth(final CompleteDestinationOAuthRequest oauthDestinationRequestBody) + public CompleteDestinationOauthResponse completeDestinationOAuth(final CompleteDestinationOAuthRequest oauthDestinationRequestBody) throws JsonValidationException, ConfigNotFoundException, IOException { final StandardDestinationDefinition destinationDefinition = configRepository.getStandardDestinationDefinition(oauthDestinationRequestBody.getDestinationDefinitionId()); final OAuthFlowImplementation oAuthFlowImplementation = oAuthImplementationFactory.create(destinationDefinition, oauthDestinationRequestBody.getWorkspaceId()); final ImmutableMap metadata = generateDestinationMetadata(oauthDestinationRequestBody.getDestinationDefinitionId()); - final Map result = oAuthFlowImplementation.completeDestinationOAuth( + final CompleteDestinationOauthResponse result = new CompleteDestinationOauthResponse(); + result.oauthOutput(oAuthFlowImplementation.completeDestinationOAuth( oauthDestinationRequestBody.getWorkspaceId(), oauthDestinationRequestBody.getDestinationDefinitionId(), oauthDestinationRequestBody.getQueryParams(), - oauthDestinationRequestBody.getRedirectUrl()); + oauthDestinationRequestBody.getRedirectUrl())); try { trackingClient.track(oauthDestinationRequestBody.getWorkspaceId(), "Complete OAuth Flow - Backend", metadata); } catch (final Exception e) { diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index a43ac05add712..031c80f688965 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -3032,12 +3032,20 @@

Request body

Return type

+ CompleteDestinationOauthResponse - map[String, Object]
+

Example data

+
Content-Type: application/json
+ 
{
+  "oauthOutput" : {
+    "key" : "{}"
+  },
+  "authConfiguration" : ""
+}

Produces

This API call produces the following media types according to the Accept request header; @@ -3049,7 +3057,7 @@

Produces

Responses

200

Successful operation - map[String, Object] + CompleteDestinationOauthResponse

404

Object with given id was not found. NotFoundKnownExceptionInfo @@ -3085,12 +3093,20 @@

Request body

Return type

+ CompleteSourceOauthResponse - map[String, Object]
+

Example data

+
Content-Type: application/json
+ 
{
+  "oauthOutput" : {
+    "key" : "{}"
+  },
+  "authConfiguration" : ""
+}

Produces

This API call produces the following media types according to the Accept request header; @@ -3102,7 +3118,7 @@

Produces

Responses

200

Successful operation - map[String, Object] + CompleteSourceOauthResponse

404

Object with given id was not found. NotFoundKnownExceptionInfo @@ -6594,7 +6610,9 @@

Table of Contents

  • CheckConnectionRead -
  • CheckOperationRead -
  • CompleteDestinationOAuthRequest -
    • +
  • CompleteDestinationOauthResponse -
  • CompleteSourceOauthRequest -
    • +
  • CompleteSourceOauthResponse -
  • ConnectionCreate -
  • ConnectionIdRequestBody -
  • ConnectionRead -
    • @@ -6802,6 +6820,15 @@

    CompleteDestinationOAuthRequ
    workspaceId
    UUID format: uuid
    redirectUrl (optional)
    String When completing OAuth flow to gain an access token, some API sometimes requires to verify that the app re-send the redirectUrl that was used when consent was given.
    queryParams (optional)
    map[String, Object] The query parameters present in the redirect URL after a user granted consent e.g auth code
    +
    authConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.
    + + +
    +

    CompleteDestinationOauthResponse - Up

    +
    +
    +
    oauthOutput (optional)
    map[String, Object]
    +
    authConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.
    @@ -6812,6 +6839,15 @@

    CompleteSourceOauthRequest
    workspaceId
    UUID format: uuid
    redirectUrl (optional)
    String When completing OAuth flow to gain an access token, some API sometimes requires to verify that the app re-send the redirectUrl that was used when consent was given.
    queryParams (optional)
    map[String, Object] The query parameters present in the redirect URL after a user granted consent e.g auth code
    +
    authConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.
    +

    + +
    +

    CompleteSourceOauthResponse - Up

    +
    +
    +
    oauthOutput (optional)
    map[String, Object]
    +
    authConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.
    @@ -7058,6 +7094,7 @@

    DestinationOauthConsentReques
    destinationDefinitionId
    UUID format: uuid
    workspaceId
    UUID format: uuid
    redirectUrl
    String The url to redirect to after getting the user consent
    +
    authConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.

    @@ -7526,6 +7563,7 @@

    SourceOauthConsentRequest -
    sourceDefinitionId
    UUID format: uuid
    workspaceId
    UUID format: uuid
    redirectUrl
    String The url to redirect to after getting the user consent
    +
    authConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.

    From 83b49a3800900092798d1a8fdae24e16b8ef2006 Mon Sep 17 00:00:00 2001 From: Christophe Duong Date: Tue, 9 Nov 2021 20:46:29 +0100 Subject: [PATCH 2/6] Changing type from Object to JsonNode --- airbyte-api/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-api/build.gradle b/airbyte-api/build.gradle index 7be1fd48720bd..a19368e314f3e 100644 --- a/airbyte-api/build.gradle +++ b/airbyte-api/build.gradle @@ -22,6 +22,7 @@ task generateApiServer(type: GenerateTask) { modelPackage = "io.airbyte.api.model" importMappings = [ + 'AuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', From 6b620352ecfb21be6ec67716991fed072b3bf983 Mon Sep 17 00:00:00 2001 From: Christophe Duong Date: Wed, 10 Nov 2021 13:35:03 +0100 Subject: [PATCH 3/6] Change api --- airbyte-api/build.gradle | 2 +- airbyte-api/src/main/openapi/config.yaml | 39 +++++---------- .../airbyte/server/apis/ConfigurationApi.java | 9 ++-- .../airbyte/server/handlers/OAuthHandler.java | 17 +++---- .../api/generated-api-html/index.html | 50 +++---------------- 5 files changed, 33 insertions(+), 84 deletions(-) diff --git a/airbyte-api/build.gradle b/airbyte-api/build.gradle index a19368e314f3e..c78a2c35f5260 100644 --- a/airbyte-api/build.gradle +++ b/airbyte-api/build.gradle @@ -22,7 +22,7 @@ task generateApiServer(type: GenerateTask) { modelPackage = "io.airbyte.api.model" importMappings = [ - 'AuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'AuthInputConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 5947a69470f5e..bf97652120e24 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -1311,7 +1311,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/CompleteSourceOauthResponse" + $ref: "#/components/schemas/CompleteOAuthResponse" "404": $ref: "#/components/responses/NotFoundResponse" "422": @@ -1357,7 +1357,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/CompleteDestinationOauthResponse" + $ref: "#/components/schemas/CompleteOAuthResponse" "404": $ref: "#/components/responses/NotFoundResponse" "422": @@ -3227,7 +3227,7 @@ components: items: $ref: "#/components/schemas/DbMigrationRead" # OAuth - AuthConfiguration: + AuthInputConfiguration: description: The values required to configure OAuth flows. The schema for this must match the authSpecification schema. SourceOauthConsentRequest: type: object @@ -3243,8 +3243,8 @@ components: redirectUrl: description: The url to redirect to after getting the user consent type: string - authConfiguration: - $ref: "#/components/schemas/AuthConfiguration" + authInputConfiguration: + $ref: "#/components/schemas/AuthInputConfiguration" DestinationOauthConsentRequest: type: object required: @@ -3259,8 +3259,8 @@ components: redirectUrl: description: The url to redirect to after getting the user consent type: string - authConfiguration: - $ref: "#/components/schemas/AuthConfiguration" + authInputConfiguration: + $ref: "#/components/schemas/AuthInputConfiguration" OAuthConsentRead: type: object required: @@ -3285,8 +3285,8 @@ components: description: The query parameters present in the redirect URL after a user granted consent e.g auth code type: object additionalProperties: true # Oauth parameters like code, state, etc.. will be different per API so we don't specify them in advance - authConfiguration: - $ref: "#/components/schemas/AuthConfiguration" + authInputConfiguration: + $ref: "#/components/schemas/AuthInputConfiguration" CompleteDestinationOAuthRequest: type: object required: @@ -3304,24 +3304,11 @@ components: description: The query parameters present in the redirect URL after a user granted consent e.g auth code type: object additionalProperties: true # Oauth parameters like code, state, etc.. will be different per API so we don't specify them in advance - authConfiguration: - $ref: "#/components/schemas/AuthConfiguration" - CompleteSourceOauthResponse: + authInputConfiguration: + $ref: "#/components/schemas/AuthInputConfiguration" + CompleteOAuthResponse: type: object - properties: - oauthOutput: - type: object - additionalProperties: true # Oauth parameters like refresh/access token etc.. will be different per API so we don't specify them in advance - authConfiguration: - $ref: "#/components/schemas/AuthConfiguration" - CompleteDestinationOauthResponse: - type: object - properties: - oauthOutput: - type: object - additionalProperties: true # Oauth parameters like refresh/access token etc.. will be different per API so we don't specify them in advance - authConfiguration: - $ref: "#/components/schemas/AuthConfiguration" + additionalProperties: true # Oauth parameters like refresh/access token etc.. will be different per API so we don't specify them in advance SetInstancewideSourceOauthParamsRequestBody: type: object required: diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 7f5309314bf91..f49ebfea13870 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -8,9 +8,7 @@ import io.airbyte.api.model.CheckConnectionRead; import io.airbyte.api.model.CheckOperationRead; import io.airbyte.api.model.CompleteDestinationOAuthRequest; -import io.airbyte.api.model.CompleteDestinationOauthResponse; import io.airbyte.api.model.CompleteSourceOauthRequest; -import io.airbyte.api.model.CompleteSourceOauthResponse; import io.airbyte.api.model.ConnectionCreate; import io.airbyte.api.model.ConnectionIdRequestBody; import io.airbyte.api.model.ConnectionRead; @@ -124,6 +122,7 @@ import java.io.IOException; import java.net.http.HttpClient; import java.nio.file.Path; +import java.util.Map; @javax.ws.rs.Path("/v1") public class ConfigurationApi implements io.airbyte.api.V1Api { @@ -165,7 +164,7 @@ public ConfigurationApi(final ConfigRepository configRepository, final String webappUrl, final AirbyteVersion airbyteVersion, final Path workspaceRoot, - HttpClient httpClient) { + final HttpClient httpClient) { this.workerEnvironment = workerEnvironment; this.logConfigs = logConfigs; this.workspaceRoot = workspaceRoot; @@ -301,7 +300,7 @@ public OAuthConsentRead getSourceOAuthConsent(final SourceOauthConsentRequest so } @Override - public CompleteSourceOauthResponse completeSourceOAuth(final CompleteSourceOauthRequest completeSourceOauthRequest) { + public Map completeSourceOAuth(final CompleteSourceOauthRequest completeSourceOauthRequest) { return execute(() -> oAuthHandler.completeSourceOAuth(completeSourceOauthRequest)); } @@ -311,7 +310,7 @@ public OAuthConsentRead getDestinationOAuthConsent(final DestinationOauthConsent } @Override - public CompleteDestinationOauthResponse completeDestinationOAuth(final CompleteDestinationOAuthRequest requestBody) { + public Map completeDestinationOAuth(final CompleteDestinationOAuthRequest requestBody) { return execute(() -> oAuthHandler.completeDestinationOAuth(requestBody)); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java index 4fcc01b62221f..49143c76ca2f4 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java @@ -7,9 +7,7 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.analytics.TrackingClient; import io.airbyte.api.model.CompleteDestinationOAuthRequest; -import io.airbyte.api.model.CompleteDestinationOauthResponse; import io.airbyte.api.model.CompleteSourceOauthRequest; -import io.airbyte.api.model.CompleteSourceOauthResponse; import io.airbyte.api.model.DestinationOauthConsentRequest; import io.airbyte.api.model.OAuthConsentRead; import io.airbyte.api.model.SetInstancewideDestinationOauthParamsRequestBody; @@ -28,6 +26,7 @@ import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.net.http.HttpClient; +import java.util.Map; import java.util.UUID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -84,18 +83,17 @@ public OAuthConsentRead getDestinationOAuthConsent(final DestinationOauthConsent return result; } - public CompleteSourceOauthResponse completeSourceOAuth(final CompleteSourceOauthRequest oauthSourceRequestBody) + public Map completeSourceOAuth(final CompleteSourceOauthRequest oauthSourceRequestBody) throws JsonValidationException, ConfigNotFoundException, IOException { final StandardSourceDefinition sourceDefinition = configRepository.getStandardSourceDefinition(oauthSourceRequestBody.getSourceDefinitionId()); final OAuthFlowImplementation oAuthFlowImplementation = oAuthImplementationFactory.create(sourceDefinition, oauthSourceRequestBody.getWorkspaceId()); final ImmutableMap metadata = generateSourceMetadata(oauthSourceRequestBody.getSourceDefinitionId()); - final CompleteSourceOauthResponse result = new CompleteSourceOauthResponse(); - result.oauthOutput(oAuthFlowImplementation.completeSourceOAuth( + final Map result = oAuthFlowImplementation.completeSourceOAuth( oauthSourceRequestBody.getWorkspaceId(), oauthSourceRequestBody.getSourceDefinitionId(), oauthSourceRequestBody.getQueryParams(), - oauthSourceRequestBody.getRedirectUrl())); + oauthSourceRequestBody.getRedirectUrl()); try { trackingClient.track(oauthSourceRequestBody.getWorkspaceId(), "Complete OAuth Flow - Backend", metadata); } catch (final Exception e) { @@ -104,19 +102,18 @@ public CompleteSourceOauthResponse completeSourceOAuth(final CompleteSourceOauth return result; } - public CompleteDestinationOauthResponse completeDestinationOAuth(final CompleteDestinationOAuthRequest oauthDestinationRequestBody) + public Map completeDestinationOAuth(final CompleteDestinationOAuthRequest oauthDestinationRequestBody) throws JsonValidationException, ConfigNotFoundException, IOException { final StandardDestinationDefinition destinationDefinition = configRepository.getStandardDestinationDefinition(oauthDestinationRequestBody.getDestinationDefinitionId()); final OAuthFlowImplementation oAuthFlowImplementation = oAuthImplementationFactory.create(destinationDefinition, oauthDestinationRequestBody.getWorkspaceId()); final ImmutableMap metadata = generateDestinationMetadata(oauthDestinationRequestBody.getDestinationDefinitionId()); - final CompleteDestinationOauthResponse result = new CompleteDestinationOauthResponse(); - result.oauthOutput(oAuthFlowImplementation.completeDestinationOAuth( + final Map result = oAuthFlowImplementation.completeDestinationOAuth( oauthDestinationRequestBody.getWorkspaceId(), oauthDestinationRequestBody.getDestinationDefinitionId(), oauthDestinationRequestBody.getQueryParams(), - oauthDestinationRequestBody.getRedirectUrl())); + oauthDestinationRequestBody.getRedirectUrl()); try { trackingClient.track(oauthDestinationRequestBody.getWorkspaceId(), "Complete OAuth Flow - Backend", metadata); } catch (final Exception e) { diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index 031c80f688965..4fa786d921f04 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -3032,20 +3032,12 @@

    Request body

    Return type

    - CompleteDestinationOauthResponse + map[String, Object]
    -

    Example data

    -
    Content-Type: application/json
    - 
    {
-  "oauthOutput" : {
-    "key" : "{}"
-  },
-  "authConfiguration" : ""
-}

    Produces

    This API call produces the following media types according to the Accept request header; @@ -3057,7 +3049,7 @@

    Produces

    Responses

    200

    Successful operation - CompleteDestinationOauthResponse + map[String, Object]

    404

    Object with given id was not found. NotFoundKnownExceptionInfo @@ -3093,20 +3085,12 @@

    Request body

    Return type

    - CompleteSourceOauthResponse + map[String, Object]
    -

    Example data

    -
    Content-Type: application/json
    - 
    {
-  "oauthOutput" : {
-    "key" : "{}"
-  },
-  "authConfiguration" : ""
-}

    Produces

    This API call produces the following media types according to the Accept request header; @@ -3118,7 +3102,7 @@

    Produces

    Responses

    200

    Successful operation - CompleteSourceOauthResponse + map[String, Object]

    404

    Object with given id was not found. NotFoundKnownExceptionInfo @@ -6610,9 +6594,7 @@

    Table of Contents

  • CheckConnectionRead -
  • CheckOperationRead -
  • CompleteDestinationOAuthRequest -
    • -
  • CompleteDestinationOauthResponse -
  • CompleteSourceOauthRequest -
    • -
  • CompleteSourceOauthResponse -
  • ConnectionCreate -
  • ConnectionIdRequestBody -
  • ConnectionRead -
    • @@ -6820,15 +6802,7 @@

    CompleteDestinationOAuthRequ
    workspaceId
    UUID format: uuid
    redirectUrl (optional)
    String When completing OAuth flow to gain an access token, some API sometimes requires to verify that the app re-send the redirectUrl that was used when consent was given.
    queryParams (optional)
    map[String, Object] The query parameters present in the redirect URL after a user granted consent e.g auth code
    -
    authConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.
    -

    - -
    -

    CompleteDestinationOauthResponse - Up

    -
    -
    -
    oauthOutput (optional)
    map[String, Object]
    -
    authConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.
    +
    authInputConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.
    @@ -6839,15 +6813,7 @@

    CompleteSourceOauthRequest
    workspaceId
    UUID format: uuid
    redirectUrl (optional)
    String When completing OAuth flow to gain an access token, some API sometimes requires to verify that the app re-send the redirectUrl that was used when consent was given.
    queryParams (optional)
    map[String, Object] The query parameters present in the redirect URL after a user granted consent e.g auth code
    -
    authConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.
    -

    - -
    -

    CompleteSourceOauthResponse - Up

    -
    -
    -
    oauthOutput (optional)
    map[String, Object]
    -
    authConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.
    +
    authInputConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.
    @@ -7094,7 +7060,7 @@

    DestinationOauthConsentReques
    destinationDefinitionId
    UUID format: uuid
    workspaceId
    UUID format: uuid
    redirectUrl
    String The url to redirect to after getting the user consent
    -
    authConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.
    +
    authInputConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.

    @@ -7563,7 +7529,7 @@

    SourceOauthConsentRequest -
    sourceDefinitionId
    UUID format: uuid
    workspaceId
    UUID format: uuid
    redirectUrl
    String The url to redirect to after getting the user consent
    -
    authConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.
    +
    authInputConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.

    From 722d40bb4a33d3026443de13c854a309202a9d53 Mon Sep 17 00:00:00 2001 From: Christophe Duong Date: Fri, 12 Nov 2021 11:02:53 +0100 Subject: [PATCH 4/6] Rename authInputConfiguration --- airbyte-api/build.gradle | 4 +++- airbyte-api/src/main/openapi/config.yaml | 18 +++++++++--------- .../api/generated-api-html/index.html | 8 ++++---- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/airbyte-api/build.gradle b/airbyte-api/build.gradle index c78a2c35f5260..a009cf500b7ad 100644 --- a/airbyte-api/build.gradle +++ b/airbyte-api/build.gradle @@ -22,7 +22,7 @@ task generateApiServer(type: GenerateTask) { modelPackage = "io.airbyte.api.model" importMappings = [ - 'AuthInputConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'OAuthInputConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', @@ -56,6 +56,7 @@ task generateApiClient(type: GenerateTask) { modelPackage = "io.airbyte.api.client.model" importMappings = [ + 'OAuthInputConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', @@ -88,6 +89,7 @@ task generateApiDocs(type: GenerateTask) { modelPackage = "io.airbyte.api.client.model" importMappings = [ + 'OAuthInputConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index bf97652120e24..bb2a4b07588b3 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -3227,7 +3227,7 @@ components: items: $ref: "#/components/schemas/DbMigrationRead" # OAuth - AuthInputConfiguration: + OAuthInputConfiguration: description: The values required to configure OAuth flows. The schema for this must match the authSpecification schema. SourceOauthConsentRequest: type: object @@ -3243,8 +3243,8 @@ components: redirectUrl: description: The url to redirect to after getting the user consent type: string - authInputConfiguration: - $ref: "#/components/schemas/AuthInputConfiguration" + oAuthInputConfiguration: + $ref: "#/components/schemas/OAuthInputConfiguration" DestinationOauthConsentRequest: type: object required: @@ -3259,8 +3259,8 @@ components: redirectUrl: description: The url to redirect to after getting the user consent type: string - authInputConfiguration: - $ref: "#/components/schemas/AuthInputConfiguration" + oAuthInputConfiguration: + $ref: "#/components/schemas/OAuthInputConfiguration" OAuthConsentRead: type: object required: @@ -3285,8 +3285,8 @@ components: description: The query parameters present in the redirect URL after a user granted consent e.g auth code type: object additionalProperties: true # Oauth parameters like code, state, etc.. will be different per API so we don't specify them in advance - authInputConfiguration: - $ref: "#/components/schemas/AuthInputConfiguration" + oAuthInputConfiguration: + $ref: "#/components/schemas/OAuthInputConfiguration" CompleteDestinationOAuthRequest: type: object required: @@ -3304,8 +3304,8 @@ components: description: The query parameters present in the redirect URL after a user granted consent e.g auth code type: object additionalProperties: true # Oauth parameters like code, state, etc.. will be different per API so we don't specify them in advance - authInputConfiguration: - $ref: "#/components/schemas/AuthInputConfiguration" + oAuthInputConfiguration: + $ref: "#/components/schemas/OAuthInputConfiguration" CompleteOAuthResponse: type: object additionalProperties: true # Oauth parameters like refresh/access token etc.. will be different per API so we don't specify them in advance diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index 4fa786d921f04..5a566a29207d7 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -6802,7 +6802,7 @@

    CompleteDestinationOAuthRequ
    workspaceId
    UUID format: uuid
    redirectUrl (optional)
    String When completing OAuth flow to gain an access token, some API sometimes requires to verify that the app re-send the redirectUrl that was used when consent was given.
    queryParams (optional)
    map[String, Object] The query parameters present in the redirect URL after a user granted consent e.g auth code
    -
    authInputConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.
    +
    oAuthInputConfiguration (optional)
    OAuthInputConfiguration

    @@ -6813,7 +6813,7 @@

    CompleteSourceOauthRequest
    workspaceId
    UUID format: uuid
    redirectUrl (optional)
    String When completing OAuth flow to gain an access token, some API sometimes requires to verify that the app re-send the redirectUrl that was used when consent was given.
    queryParams (optional)
    map[String, Object] The query parameters present in the redirect URL after a user granted consent e.g auth code
    -
    authInputConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.
    +
    oAuthInputConfiguration (optional)
    OAuthInputConfiguration

    @@ -7060,7 +7060,7 @@

    DestinationOauthConsentReques
    destinationDefinitionId
    UUID format: uuid
    workspaceId
    UUID format: uuid
    redirectUrl
    String The url to redirect to after getting the user consent
    -
    authInputConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.
    +
    oAuthInputConfiguration (optional)
    OAuthInputConfiguration

    @@ -7529,7 +7529,7 @@

    SourceOauthConsentRequest -
    sourceDefinitionId
    UUID format: uuid
    workspaceId
    UUID format: uuid
    redirectUrl
    String The url to redirect to after getting the user consent
    -
    authInputConfiguration (optional)
    oas_any_type_not_mapped The values required to configure OAuth flows. The schema for this must match the authSpecification schema.
    +
    oAuthInputConfiguration (optional)
    OAuthInputConfiguration

    From 4ae1a1bd26a29936a01b41995a7c47a4960aa347 Mon Sep 17 00:00:00 2001 From: Christophe Duong Date: Mon, 15 Nov 2021 12:50:03 +0100 Subject: [PATCH 5/6] Change protocol for new OAuth Spec (#7827) * Change protocol for new OAuthConfigSpecification * Add examples * Add protocol object to api too Co-authored-by: Sherif A. Nada --- airbyte-api/build.gradle | 6 +- airbyte-api/src/main/openapi/config.yaml | 113 ++++++++++++++++- .../airbyte_cdk/models/airbyte_protocol.py | 69 ++++++++--- .../models/airbyte_protocol.py | 69 ++++++++--- .../airbyte_protocol/airbyte_protocol.yaml | 116 +++++++++++++++++- .../api/generated-api-html/index.html | 46 ++++++- 6 files changed, 382 insertions(+), 37 deletions(-) diff --git a/airbyte-api/build.gradle b/airbyte-api/build.gradle index a009cf500b7ad..c083974052be5 100644 --- a/airbyte-api/build.gradle +++ b/airbyte-api/build.gradle @@ -22,7 +22,7 @@ task generateApiServer(type: GenerateTask) { modelPackage = "io.airbyte.api.model" importMappings = [ - 'OAuthInputConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', @@ -56,7 +56,7 @@ task generateApiClient(type: GenerateTask) { modelPackage = "io.airbyte.api.client.model" importMappings = [ - 'OAuthInputConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', @@ -89,7 +89,7 @@ task generateApiDocs(type: GenerateTask) { modelPackage = "io.airbyte.api.client.model" importMappings = [ - 'OAuthInputConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index bb2a4b07588b3..0d985e020f994 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -2066,6 +2066,8 @@ components: $ref: "#/components/schemas/SourceDefinitionSpecification" authSpecification: $ref: "#/components/schemas/SourceAuthSpecification" + advancedAuth: + $ref: "#/components/schemas/AdvancedAuth" jobInfo: $ref: "#/components/schemas/SynchronousJobRead" # SOURCE @@ -2268,6 +2270,8 @@ components: $ref: "#/components/schemas/DestinationDefinitionSpecification" authSpecification: $ref: "#/components/schemas/DestinationAuthSpecification" + advancedAuth: + $ref: "#/components/schemas/AdvancedAuth" jobInfo: $ref: "#/components/schemas/SynchronousJobRead" supportedDestinationSyncModes: @@ -3227,8 +3231,115 @@ components: items: $ref: "#/components/schemas/DbMigrationRead" # OAuth + OAuthConfiguration: + description: OAuth specific blob. OAuthInputConfiguration: - description: The values required to configure OAuth flows. The schema for this must match the authSpecification schema. + description: The values required to configure OAuth flows. + The schema for this must match the `OAuthConfigSpecification.oauthUserInputFromConnectorConfigSpecification` schema. + $ref: "#/components/schemas/OAuthConfiguration" + AdvancedAuth: + type: object + properties: + authFlowType: + type: string + enum: ["oauth1.0", "oauth2.0"] + predicateKey: + description: Json Path to a field in the connectorSpecification that should exist for the advanced auth to be applicable. + type: array + items: + type: string + predicateValue: + description: Value of the predicate_key fields for the advanced auth to be applicable. + type: string + oauthConfigSpecification: + "$ref": "#/components/schemas/OAuthConfigSpecification" + OAuthConfigSpecification: + type: object + properties: + oauthUserInputFromConnectorConfigSpecification: + description: |- + OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth. + Must be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification + using special annotation 'path_in_connector_config'. + These are input values the user is entering through the UI to authenticate to the connector, that might also shared + as inputs for syncing data via the connector. + + Examples: + + if no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[] + if connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow, + oauth_user_input_from_connector_config_specification={ + app_id: { + type: string + path_in_connector_config: ['app_id'] + } + } + if connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow, + oauth_user_input_from_connector_config_specification={ + app_id: { + type: string + path_in_connector_config: ['info', 'app_id'] + } + } + $ref: "#/components/schemas/OAuthConfiguration" + completeOAuthOutputSpecification: + description: |- + OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are + returned by the distant OAuth APIs. + Must be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`. + For each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it, + + Examples: + + complete_oauth_output_specification={ + refresh_token: { + type: string, + path_in_connector_config: ['credentials', 'refresh_token'] + } + } + $ref: "#/components/schemas/OAuthConfiguration" + completeOAuthServerInputSpecification: + description: |- + OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations. + Must be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the + server when completing an OAuth flow (typically exchanging an auth code for refresh token). + + Examples: + + complete_oauth_server_input_specification={ + client_id: { + type: string + }, + client_secret: { + type: string + } + } + $ref: "#/components/schemas/OAuthConfiguration" + completeOAuthServerOutputSpecification: + description: |- + OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that + also need to be merged back into the connector configuration at runtime. + This is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that + are necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore + they would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`) + Must be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the + connector when using OAuth flow APIs. + These fields are to be merged back to `ConnectorSpecification.connectionSpecification`. + For each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it, + + Examples: + + complete_oauth_server_output_specification={ + client_id: { + type: string, + path_in_connector_config: ['credentials', 'client_id'] + }, + client_secret: { + type: string, + path_in_connector_config: ['credentials', 'client_secret'] + } + } + $ref: "#/components/schemas/OAuthConfiguration" SourceOauthConsentRequest: type: object required: diff --git a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py index 0e7477a838ff0..0de561d5478db 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py +++ b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py @@ -113,23 +113,28 @@ class AuthSpecification(BaseModel): ) -class ConnectorSpecification(BaseModel): - class Config: - extra = Extra.allow +class AuthFlowType(Enum): + oauth1_0 = "oauth1.0" + oauth2_0 = "oauth2.0" - documentationUrl: Optional[AnyUrl] = None - changelogUrl: Optional[AnyUrl] = None - connectionSpecification: Dict[str, Any] = Field( - ..., - description="ConnectorDefinition specific blob. Must be a valid JSON string.", + +class OAuthConfigSpecification(BaseModel): + oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\n\nExamples:\n\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", ) - supportsIncremental: Optional[bool] = Field(None, description="If the connector supports incremental mode or not.") - supportsNormalization: Optional[bool] = Field(False, description="If the connector supports normalization or not.") - supportsDBT: Optional[bool] = Field(False, description="If the connector supports DBT or not.") - supported_destination_sync_modes: Optional[List[DestinationSyncMode]] = Field( - None, description="List of destination sync modes supported by the connector" + complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\n\nExamples:\n\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", + ) + complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\n\nExamples:\n\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", + ) + complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\n\nExamples:\n\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", ) - authSpecification: Optional[AuthSpecification] = None class AirbyteStream(BaseModel): @@ -174,6 +179,42 @@ class Config: ) +class AdvancedAuth(BaseModel): + auth_flow_type: Optional[AuthFlowType] = None + predicate_key: Optional[List[str]] = Field( + None, + description="Json Path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", + ) + predicate_value: Optional[str] = Field( + None, + description="Value of the predicate_key fields for the advanced auth to be applicable.", + ) + oauth_config_specification: Optional[OAuthConfigSpecification] = None + + +class ConnectorSpecification(BaseModel): + class Config: + extra = Extra.allow + + documentationUrl: Optional[AnyUrl] = None + changelogUrl: Optional[AnyUrl] = None + connectionSpecification: Dict[str, Any] = Field( + ..., + description="ConnectorDefinition specific blob. Must be a valid JSON string.", + ) + supportsIncremental: Optional[bool] = Field(None, description="If the connector supports incremental mode or not.") + supportsNormalization: Optional[bool] = Field(False, description="If the connector supports normalization or not.") + supportsDBT: Optional[bool] = Field(False, description="If the connector supports DBT or not.") + supported_destination_sync_modes: Optional[List[DestinationSyncMode]] = Field( + None, description="List of destination sync modes supported by the connector" + ) + authSpecification: Optional[AuthSpecification] = Field(None, description="deprecated, switching to advanced_auth instead") + advanced_auth: Optional[AdvancedAuth] = Field( + None, + description="Additional and optional specification object to describe what an 'advanced' Auth flow would need to function.\n - A connector should be able to fully function with the configuration as described by the ConnectorSpecification in a 'basic' mode.\n - The 'advanced' mode provides easier UX for the user with UI improvements and automations. However, this requires further setup on the\n server side by instance or workspace admins beforehand. The trade-off is that the user does not have to provide as many technical\n inputs anymore and the auth process is faster and easier to complete.", + ) + + class AirbyteCatalog(BaseModel): class Config: extra = Extra.allow diff --git a/airbyte-integrations/bases/airbyte-protocol/airbyte_protocol/models/airbyte_protocol.py b/airbyte-integrations/bases/airbyte-protocol/airbyte_protocol/models/airbyte_protocol.py index 0e7477a838ff0..0de561d5478db 100644 --- a/airbyte-integrations/bases/airbyte-protocol/airbyte_protocol/models/airbyte_protocol.py +++ b/airbyte-integrations/bases/airbyte-protocol/airbyte_protocol/models/airbyte_protocol.py @@ -113,23 +113,28 @@ class AuthSpecification(BaseModel): ) -class ConnectorSpecification(BaseModel): - class Config: - extra = Extra.allow +class AuthFlowType(Enum): + oauth1_0 = "oauth1.0" + oauth2_0 = "oauth2.0" - documentationUrl: Optional[AnyUrl] = None - changelogUrl: Optional[AnyUrl] = None - connectionSpecification: Dict[str, Any] = Field( - ..., - description="ConnectorDefinition specific blob. Must be a valid JSON string.", + +class OAuthConfigSpecification(BaseModel): + oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\n\nExamples:\n\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", ) - supportsIncremental: Optional[bool] = Field(None, description="If the connector supports incremental mode or not.") - supportsNormalization: Optional[bool] = Field(False, description="If the connector supports normalization or not.") - supportsDBT: Optional[bool] = Field(False, description="If the connector supports DBT or not.") - supported_destination_sync_modes: Optional[List[DestinationSyncMode]] = Field( - None, description="List of destination sync modes supported by the connector" + complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\n\nExamples:\n\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", + ) + complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\n\nExamples:\n\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", + ) + complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\n\nExamples:\n\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", ) - authSpecification: Optional[AuthSpecification] = None class AirbyteStream(BaseModel): @@ -174,6 +179,42 @@ class Config: ) +class AdvancedAuth(BaseModel): + auth_flow_type: Optional[AuthFlowType] = None + predicate_key: Optional[List[str]] = Field( + None, + description="Json Path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", + ) + predicate_value: Optional[str] = Field( + None, + description="Value of the predicate_key fields for the advanced auth to be applicable.", + ) + oauth_config_specification: Optional[OAuthConfigSpecification] = None + + +class ConnectorSpecification(BaseModel): + class Config: + extra = Extra.allow + + documentationUrl: Optional[AnyUrl] = None + changelogUrl: Optional[AnyUrl] = None + connectionSpecification: Dict[str, Any] = Field( + ..., + description="ConnectorDefinition specific blob. Must be a valid JSON string.", + ) + supportsIncremental: Optional[bool] = Field(None, description="If the connector supports incremental mode or not.") + supportsNormalization: Optional[bool] = Field(False, description="If the connector supports normalization or not.") + supportsDBT: Optional[bool] = Field(False, description="If the connector supports DBT or not.") + supported_destination_sync_modes: Optional[List[DestinationSyncMode]] = Field( + None, description="List of destination sync modes supported by the connector" + ) + authSpecification: Optional[AuthSpecification] = Field(None, description="deprecated, switching to advanced_auth instead") + advanced_auth: Optional[AdvancedAuth] = Field( + None, + description="Additional and optional specification object to describe what an 'advanced' Auth flow would need to function.\n - A connector should be able to fully function with the configuration as described by the ConnectorSpecification in a 'basic' mode.\n - The 'advanced' mode provides easier UX for the user with UI improvements and automations. However, this requires further setup on the\n server side by instance or workspace admins beforehand. The trade-off is that the user does not have to provide as many technical\n inputs anymore and the auth process is faster and easier to complete.", + ) + + class AirbyteCatalog(BaseModel): class Config: extra = Extra.allow diff --git a/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index e621be83553cf..ec505bd0b617c 100644 --- a/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -209,7 +209,7 @@ definitions: #- upsert_dedup # TODO chris: SCD Type 1 can be implemented later - append_dedup # SCD Type 1 & 2 OAuth2Specification: - description: An object containing any metadata needed to describe this connector's Oauth flow + description: An object containing any metadata needed to describe this connector's Oauth flow. Deprecated, switching to advanced_auth instead type: object additionalProperties: true properties: @@ -297,6 +297,7 @@ definitions: items: "$ref": "#/definitions/DestinationSyncMode" authSpecification: + description: deprecated, switching to advanced_auth instead type: object properties: auth_type: @@ -305,3 +306,116 @@ definitions: oauth2Specification: description: If the connector supports OAuth, this field should be non-null. "$ref": "#/definitions/OAuth2Specification" + advanced_auth: + description: |- + Additional and optional specification object to describe what an 'advanced' Auth flow would need to function. + - A connector should be able to fully function with the configuration as described by the ConnectorSpecification in a 'basic' mode. + - The 'advanced' mode provides easier UX for the user with UI improvements and automations. However, this requires further setup on the + server side by instance or workspace admins beforehand. The trade-off is that the user does not have to provide as many technical + inputs anymore and the auth process is faster and easier to complete. + type: object + properties: + auth_flow_type: + type: string + enum: ["oauth1.0", "oauth2.0"] # Future auth types should be added here + predicate_key: + description: Json Path to a field in the connectorSpecification that should exist for the advanced auth to be applicable. + type: array + items: + type: string + predicate_value: + description: Value of the predicate_key fields for the advanced auth to be applicable. + type: string + oauth_config_specification: + "$ref": "#/definitions/OAuthConfigSpecification" + OAuthConfigSpecification: + type: object + properties: + oauth_user_input_from_connector_config_specification: + description: |- + OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth. + Must be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification + using special annotation 'path_in_connector_config'. + These are input values the user is entering through the UI to authenticate to the connector, that might also shared + as inputs for syncing data via the connector. + + Examples: + + if no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[] + if connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow, + oauth_user_input_from_connector_config_specification={ + app_id: { + type: string + path_in_connector_config: ['app_id'] + } + } + if connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow, + oauth_user_input_from_connector_config_specification={ + app_id: { + type: string + path_in_connector_config: ['info', 'app_id'] + } + } + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode + complete_oauth_output_specification: + description: |- + OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are + returned by the distant OAuth APIs. + Must be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`. + For each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it, + + Examples: + + complete_oauth_output_specification={ + refresh_token: { + type: string, + path_in_connector_config: ['credentials', 'refresh_token'] + } + } + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode + complete_oauth_server_input_specification: + description: |- + OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations. + Must be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the + server when completing an OAuth flow (typically exchanging an auth code for refresh token). + + Examples: + + complete_oauth_server_input_specification={ + client_id: { + type: string + }, + client_secret: { + type: string + } + } + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode + complete_oauth_server_output_specification: + description: |- + OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that + also need to be merged back into the connector configuration at runtime. + This is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that + are necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore + they would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`) + Must be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the + connector when using OAuth flow APIs. + These fields are to be merged back to `ConnectorSpecification.connectionSpecification`. + For each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it, + + Examples: + + complete_oauth_server_output_specification={ + client_id: { + type: string, + path_in_connector_config: ['credentials', 'client_id'] + }, + client_secret: { + type: string, + path_in_connector_config: ['credentials', 'client_secret'] + } + } + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index 5a566a29207d7..3a93ccfe5b888 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -2530,6 +2530,12 @@

    Example data

    "supportedDestinationSyncModes" : [ null, null ], "supportsDbt" : true, "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", + "advancedAuth" : { + "predicateValue" : "predicateValue", + "oauthConfigSpecification" : { }, + "predicateKey" : [ "predicateKey", "predicateKey" ], + "authFlowType" : "oauth1.0" + }, "authSpecification" : { "auth_type" : "oauth2.0", "oauth2Specification" : { @@ -4956,6 +4962,12 @@

    Example data

    } }, "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", + "advancedAuth" : { + "predicateValue" : "predicateValue", + "oauthConfigSpecification" : { }, + "predicateKey" : [ "predicateKey", "predicateKey" ], + "authFlowType" : "oauth1.0" + }, "authSpecification" : { "auth_type" : "oauth2.0", "oauth2Specification" : { @@ -6583,6 +6595,7 @@

    Models

    Table of Contents

      +
    1. AdvancedAuth -
    2. AirbyteCatalog -
    3. AirbyteStream -
    4. AirbyteStreamAndConfiguration -
      5. @@ -6648,6 +6661,7 @@

      Table of Contents

    5. NotificationRead -
    6. NotificationType -
    7. OAuth2Specification -
      8. +
    8. OAuthConfigSpecification -
    9. OAuthConsentRead -
    10. OperationCreate -
    11. OperationIdRequestBody -
      12. @@ -6696,6 +6710,18 @@

      Table of Contents

    12. WorkspaceUpdate -
    +
    +

    AdvancedAuth - Up

    +
    +
    +
    authFlowType (optional)
    String
    +
    Enum:
    +
    oauth1.0
    oauth2.0
    +
    predicateKey (optional)
    array[String] Json Path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.
    +
    predicateValue (optional)
    String Value of the predicate_key fields for the advanced auth to be applicable.
    +
    oauthConfigSpecification (optional)
    OAuthConfigSpecification
    +
    +

    AirbyteCatalog - Up

    describes the available schema (catalog).
    @@ -6802,7 +6828,7 @@

    CompleteDestinationOAuthRequ
    workspaceId
    UUID format: uuid
    redirectUrl (optional)
    String When completing OAuth flow to gain an access token, some API sometimes requires to verify that the app re-send the redirectUrl that was used when consent was given.
    queryParams (optional)
    map[String, Object] The query parameters present in the redirect URL after a user granted consent e.g auth code
    -
    oAuthInputConfiguration (optional)
    OAuthInputConfiguration
    +
    oAuthInputConfiguration (optional)
    OAuthConfiguration

    @@ -6813,7 +6839,7 @@

    CompleteSourceOauthRequest
    workspaceId
    UUID format: uuid
    redirectUrl (optional)
    String When completing OAuth flow to gain an access token, some API sometimes requires to verify that the app re-send the redirectUrl that was used when consent was given.
    queryParams (optional)
    map[String, Object] The query parameters present in the redirect URL after a user granted consent e.g auth code
    -
    oAuthInputConfiguration (optional)
    OAuthInputConfiguration
    +
    oAuthInputConfiguration (optional)
    OAuthConfiguration

    @@ -7032,6 +7058,7 @@

    DestinationDefinition
    documentationUrl (optional)
    String
    connectionSpecification (optional)
    DestinationDefinitionSpecification
    authSpecification (optional)
    AuthSpecification
    +
    advancedAuth (optional)
    AdvancedAuth
    jobInfo
    SynchronousJobRead
    supportedDestinationSyncModes (optional)
    array[DestinationSyncMode]
    supportsDbt (optional)
    Boolean
    @@ -7060,7 +7087,7 @@

    DestinationOauthConsentReques
    destinationDefinitionId
    UUID format: uuid
    workspaceId
    UUID format: uuid
    redirectUrl
    String The url to redirect to after getting the user consent
    -
    oAuthInputConfiguration (optional)
    OAuthInputConfiguration
    +
    oAuthInputConfiguration (optional)
    OAuthConfiguration

    @@ -7302,6 +7329,16 @@

    OAuth2Specification - oauthFlowOutputParameters

    array[array[String]] Pointers to the fields in the rootObject which can be populated from successfully completing the oauth flow using the init parameters. This is typically a refresh/access token. Each inner array represents the path in the rootObject of the referenced field.
    +
    +

    OAuthConfigSpecification - Up

    +
    +
    +
    oauthUserInputFromConnectorConfigSpecification (optional)
    OAuthConfiguration
    +
    completeOAuthOutputSpecification (optional)
    OAuthConfiguration
    +
    completeOAuthServerInputSpecification (optional)
    OAuthConfiguration
    +
    completeOAuthServerOutputSpecification (optional)
    OAuthConfiguration
    +
    +

    OAuthConsentRead - Up

    @@ -7496,6 +7533,7 @@

    SourceDefinitionSpecificat
    documentationUrl (optional)
    String
    connectionSpecification (optional)
    SourceDefinitionSpecification
    authSpecification (optional)
    AuthSpecification
    +
    advancedAuth (optional)
    AdvancedAuth
    jobInfo
    SynchronousJobRead

    @@ -7529,7 +7567,7 @@

    SourceOauthConsentRequest -
    sourceDefinitionId
    UUID format: uuid
    workspaceId
    UUID format: uuid
    redirectUrl
    String The url to redirect to after getting the user consent
    -
    oAuthInputConfiguration (optional)
    OAuthInputConfiguration
    +
    oAuthInputConfiguration (optional)
    OAuthConfiguration
    From b9236f0b13d259548be28aa2664d686fb6f4c86a Mon Sep 17 00:00:00 2001 From: Christophe Duong Date: Mon, 15 Nov 2021 12:55:05 +0100 Subject: [PATCH 6/6] Apply suggestions from code review --- airbyte-api/src/main/openapi/config.yaml | 2 +- .../src/main/resources/airbyte_protocol/airbyte_protocol.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 0d985e020f994..78ff0c3d7476c 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -3242,7 +3242,7 @@ components: properties: authFlowType: type: string - enum: ["oauth1.0", "oauth2.0"] + enum: ["oauth2.0", "oauth1.0"] predicateKey: description: Json Path to a field in the connectorSpecification that should exist for the advanced auth to be applicable. type: array diff --git a/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index ec505bd0b617c..e3146e26b6e83 100644 --- a/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -317,7 +317,7 @@ definitions: properties: auth_flow_type: type: string - enum: ["oauth1.0", "oauth2.0"] # Future auth types should be added here + enum: ["oauth2.0", "oauth1.0"] # Future auth types should be added here predicate_key: description: Json Path to a field in the connectorSpecification that should exist for the advanced auth to be applicable. type: array