diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index f459b343b270..e6988e245a87 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -1620,6 +1620,52 @@ paths: $ref: "#/components/schemas/ImportRead" "404": $ref: "#/components/responses/NotFoundResponse" + /v1/source_oauths/oauth_params/create: + post: + tags: + - oauth + summary: > + Sets instancewide variables to be used for the oauth flow when creating this source. When set, these variables will be injected + into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with + consistent variables e.g: the company's Google Ads developer_token, client_id, and client_secret without the user having to know + about these variables. + operationId: setInstancewideSourceOauthParams + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/SetInstancewideSourceOauthParamsRequestBody" + required: true + responses: + "200": + description: Successful + "400": + $ref: "#/components/responses/ExceptionResponse" + "404": + $ref: "#/components/responses/NotFoundResponse" + /v1/destination_oauths/oauth_params/create: + post: + tags: + - oauth + summary: > + Sets instancewide variables to be used for the oauth flow when creating this destination. When set, these variables will be injected + into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with + consistent variables e.g: the company's Google Ads developer_token, client_id, and client_secret without the user having to know + about these variables. + operationId: setInstancewideDestinationOauthParams + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/SetInstancewideDestinationOauthParamsRequestBody" + required: true + responses: + "200": + description: Successful + "400": + $ref: "#/components/responses/ExceptionResponse" + "404": + $ref: "#/components/responses/NotFoundResponse" components: securitySchemes: bearerAuth: @@ -2995,6 +3041,28 @@ components: CompleteOauthResponse: type: object additionalProperties: true # Oauth parameters like refresh/access token etc.. will be different per API so we don't specify them in advance + SetInstancewideSourceOauthParamsRequestBody: + type: object + required: + - sourceConnectorDefinitionId + - params + properties: + sourceDefinitionId: + $ref: "#/components/schemas/SourceDefinitionId" + params: + type: object + additionalProperties: true + SetInstancewideDestinationOauthParamsRequestBody: + type: object + required: + - destinationDefinitionId + - params + properties: + destinationDefinitionId: + $ref: "#/components/schemas/DestinationDefinitionId" + params: + type: object + additionalProperties: true # Web Backend WebBackendConnectionRead: type: object diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java b/airbyte-db/lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java index 8acb9c89d4b0..b3fa7aadd0d6 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java @@ -110,8 +110,8 @@ private static void readBson(final Document document, final ObjectNode o, final } /** - * Gets 10.000 documents from collection, gathers all unique fields and its type. In case when one field has different types in 2 and more - * documents, the type is set to String. + * Gets 10.000 documents from collection, gathers all unique fields and its type. In case when one + * field has different types in 2 and more documents, the type is set to String. * * @param collection mongo collection * @return map of unique fields and its type diff --git a/airbyte-integrations/connectors/source-github/fixtures/github.py b/airbyte-integrations/connectors/source-github/fixtures/github.py index 19d8173abe8e..36efb84bcf35 100644 --- a/airbyte-integrations/connectors/source-github/fixtures/github.py +++ b/airbyte-integrations/connectors/source-github/fixtures/github.py @@ -1,6 +1,31 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +# type: ignore import json import logging -from typing import Optional, Iterator +from typing import Iterator, Optional import requests @@ -29,7 +54,7 @@ def __init__(self, token: str, repository: str): self.session = requests.Session() self.session.headers.update(self.get_headers(self.token)) - self.branches = None + self.branches: Optional[list] = None @staticmethod def get_headers(token: str): @@ -78,9 +103,7 @@ def pull_requests(self) -> Iterator: "event": "COMMENT", } review_url = f"{self.BASE_URL}/repos/{self.repository}/pulls/{response.json().get('number')}/reviews" - response = self.session.post( - url=review_url, data=json.dumps(create_review_data) - ) + response = self.session.post(url=review_url, data=json.dumps(create_review_data)) yield response # create comment for commit @@ -88,9 +111,7 @@ def pull_requests(self) -> Iterator: "body": f'comment for {branch.get("commit").get("sha")} branch', } commit_url = f"https://api.github.com/repos/{self.repository}/commits/{branch.get('commit').get('sha')}/comments" - response = self.session.post( - url=commit_url, data=json.dumps(create_comment_data) - ) + response = self.session.post(url=commit_url, data=json.dumps(create_comment_data)) yield response @logger diff --git a/airbyte-integrations/connectors/source-github/fixtures/main.py b/airbyte-integrations/connectors/source-github/fixtures/main.py index 6df4dc668d65..279c9c179cc8 100644 --- a/airbyte-integrations/connectors/source-github/fixtures/main.py +++ b/airbyte-integrations/connectors/source-github/fixtures/main.py @@ -1,4 +1,31 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +# type: ignore +# this is just a script that doesn't need mypy linting import sys + from github import GitHubFiller if __name__ == "__main__": diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAcceptanceTest.java index 84b0a3866d56..c9bd7adee725 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAcceptanceTest.java @@ -115,10 +115,10 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception { .withDestinationSyncMode(DestinationSyncMode.APPEND) .withCursorField(List.of("_id")) .withStream(CatalogHelpers.createAirbyteStream( - "test.acceptance_test", - Field.of("_id", JsonSchemaPrimitive.STRING), - Field.of("id", JsonSchemaPrimitive.STRING), - Field.of("name", JsonSchemaPrimitive.STRING)) + "test.acceptance_test", + Field.of("_id", JsonSchemaPrimitive.STRING), + Field.of("id", JsonSchemaPrimitive.STRING), + Field.of("name", JsonSchemaPrimitive.STRING)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.INCREMENTAL)) .withDefaultCursorField(List.of("_id"))))); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 89e13ac05c13..dfcc65ad1030 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -67,6 +67,8 @@ import io.airbyte.api.model.OperationReadList; import io.airbyte.api.model.OperationUpdate; import io.airbyte.api.model.OperatorConfiguration; +import io.airbyte.api.model.SetInstancewideDestinationOauthParamsRequestBody; +import io.airbyte.api.model.SetInstancewideSourceOauthParamsRequestBody; import io.airbyte.api.model.SlugRequestBody; import io.airbyte.api.model.SourceCoreConfig; import io.airbyte.api.model.SourceCreate; @@ -276,7 +278,7 @@ public SourceDefinitionSpecificationRead getSourceDefinitionSpecification(final return execute(() -> schedulerHandler.getSourceDefinitionSpecification(sourceDefinitionIdRequestBody)); } - // SOURCE OAUTH + // OAUTH @Override public OAuthConsentRead getSourceOAuthConsent(SourceOauthConsentRequest sourceOauthConsentRequest) { @@ -288,6 +290,32 @@ public Map completeSourceOAuth(CompleteSourceOauthRequest comple return execute(() -> oAuthHandler.completeSourceOAuth(completeSourceOauthRequest)); } + @Override + public OAuthConsentRead getDestinationOAuthConsent(DestinationOauthConsentRequest destinationOauthConsentRequest) { + return execute(() -> oAuthHandler.getDestinationOAuthConsent(destinationOauthConsentRequest)); + } + + @Override + public Map completeDestinationOAuth(CompleteDestinationOAuthRequest requestBody) { + return execute(() -> oAuthHandler.completeDestinationOAuth(requestBody)); + } + + @Override + public void setInstancewideDestinationOauthParams(SetInstancewideDestinationOauthParamsRequestBody requestBody) { + execute(() -> { + oAuthHandler.setDestinationInstancewideOauthParams(requestBody); + return null; + }); + } + + @Override + public void setInstancewideSourceOauthParams(SetInstancewideSourceOauthParamsRequestBody requestBody) { + execute(() -> { + oAuthHandler.setSourceInstancewideOauthParams(requestBody); + return null; + }); + } + // SOURCE IMPLEMENTATION @Override @@ -379,17 +407,6 @@ public DestinationDefinitionSpecificationRead getDestinationDefinitionSpecificat return execute(() -> schedulerHandler.getDestinationSpecification(destinationDefinitionIdRequestBody)); } - // DESTINATION OAUTH - @Override - public OAuthConsentRead getDestinationOAuthConsent(DestinationOauthConsentRequest destinationOauthConsentRequest) { - return execute(() -> oAuthHandler.getDestinationOAuthConsent(destinationOauthConsentRequest)); - } - - @Override - public Map completeDestinationOAuth(CompleteDestinationOAuthRequest requestBody) { - return execute(() -> oAuthHandler.completeDestinationOAuth(requestBody)); - } - // DESTINATION IMPLEMENTATION @Override diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java index 1c3e7bfc84ca..597b8d43960d 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java @@ -28,7 +28,12 @@ import io.airbyte.api.model.CompleteSourceOauthRequest; import io.airbyte.api.model.DestinationOauthConsentRequest; import io.airbyte.api.model.OAuthConsentRead; +import io.airbyte.api.model.SetInstancewideDestinationOauthParamsRequestBody; +import io.airbyte.api.model.SetInstancewideSourceOauthParamsRequestBody; import io.airbyte.api.model.SourceOauthConsentRequest; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.DestinationOAuthParameter; +import io.airbyte.config.SourceOAuthParameter; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.persistence.ConfigNotFoundException; @@ -74,6 +79,23 @@ public Map completeDestinationOAuth(CompleteDestinationOAuthRequ return oAuthFlowImplementation.completeOAuth(oauthDestinationRequestBody.getWorkspaceId(), oauthDestinationRequestBody.getQueryParams()); } + public void setDestinationInstancewideOauthParams(SetInstancewideDestinationOauthParamsRequestBody requestBody) + throws JsonValidationException, IOException { + DestinationOAuthParameter param = new DestinationOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withConfiguration(Jsons.jsonNode(requestBody.getParams())) + .withDestinationDefinitionId(requestBody.getDestinationDefinitionId()); + configRepository.writeDestinationOAuthParam(param); + } + + public void setSourceInstancewideOauthParams(SetInstancewideSourceOauthParamsRequestBody requestBody) throws JsonValidationException, IOException { + SourceOAuthParameter param = new SourceOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withConfiguration(Jsons.jsonNode(requestBody.getParams())) + .withSourceDefinitionId(requestBody.getSourceDefinitionId()); + configRepository.writeSourceOAuthParam(param); + } + private OAuthFlowImplementation getSourceOAuthFlowImplementation(UUID sourceDefinitionId) throws JsonValidationException, ConfigNotFoundException, IOException { final StandardSourceDefinition standardSourceDefinition = configRepository diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/OAuthHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/OAuthHandlerTest.java new file mode 100644 index 000000000000..34f883c3ff74 --- /dev/null +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/OAuthHandlerTest.java @@ -0,0 +1,94 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.server.handlers; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import io.airbyte.api.model.SetInstancewideDestinationOauthParamsRequestBody; +import io.airbyte.api.model.SetInstancewideSourceOauthParamsRequestBody; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.DestinationOAuthParameter; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +class OAuthHandlerTest { + + ConfigRepository configRepository; + OAuthHandler handler; + + @BeforeEach + public void init() { + configRepository = Mockito.mock(ConfigRepository.class); + handler = new OAuthHandler(configRepository); + } + + @Test + void setDestinationInstancewideOauthParams() throws JsonValidationException, IOException { + UUID destinationDefId = UUID.randomUUID(); + Map params = new HashMap<>(); + params.put("client_id", "123"); + params.put("client_secret", "hunter2"); + + SetInstancewideDestinationOauthParamsRequestBody actualRequest = new SetInstancewideDestinationOauthParamsRequestBody() + .destinationDefinitionId(destinationDefId) + .params(params); + + handler.setDestinationInstancewideOauthParams(actualRequest); + + ArgumentCaptor argument = ArgumentCaptor.forClass(DestinationOAuthParameter.class); + Mockito.verify(configRepository).writeDestinationOAuthParam(argument.capture()); + assertEquals(Jsons.jsonNode(params), argument.getValue().getConfiguration()); + assertEquals(destinationDefId, argument.getValue().getDestinationDefinitionId()); + } + + @Test + void setSourceInstancewideOauthParams() throws JsonValidationException, IOException { + UUID sourceDefId = UUID.randomUUID(); + Map params = new HashMap<>(); + params.put("client_id", "123"); + params.put("client_secret", "hunter2"); + + SetInstancewideSourceOauthParamsRequestBody actualRequest = new SetInstancewideSourceOauthParamsRequestBody() + .sourceDefinitionId(sourceDefId) + .params(params); + + handler.setSourceInstancewideOauthParams(actualRequest); + + ArgumentCaptor argument = ArgumentCaptor.forClass(SourceOAuthParameter.class); + Mockito.verify(configRepository).writeSourceOAuthParam(argument.capture()); + assertEquals(Jsons.jsonNode(params), argument.getValue().getConfiguration()); + assertEquals(sourceDefId, argument.getValue().getSourceDefinitionId()); + } + +} diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index b49578d3c5e9..60e645113ced 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -288,6 +288,8 @@

Oauth

  • post /v1/source_oauths/complete_oauth
  • post /v1/destination_oauths/get_consent_url
  • post /v1/source_oauths/get_consent_url
    • +
  • post /v1/destination_oauths/oauth_params/create
    • +
  • post /v1/source_oauths/oauth_params/create

    • Openapi

      @@ -2983,6 +2985,102 @@

      422

      InvalidInputExceptionInfo
      +
      +
      + Up + 
      post /v1/destination_oauths/oauth_params/create
      +
      Sets instancewide variables to be used for the oauth flow when creating this destination. When set, these variables will be injected into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with consistent variables e.g: the company's Google Ads developer_token, client_id, and client_secret without the user having to know about these variables. (setInstancewideDestinationOauthParams)
      +
      + + +

      Consumes

      + This API call consumes the following media types via the Content-Type request header: +
        +
      • application/json
        • +
      + +

      Request body

      +
      +
      SetInstancewideDestinationOauthParamsRequestBody SetInstancewideDestinationOauthParamsRequestBody (required)
      + +
      Body Parameter —
      + +
      + + + + + + + + +

      Produces

      + This API call produces the following media types according to the Accept request header; + the media type will be conveyed by the Content-Type response header. +
        +
      • application/json
        • +
      + +

      Responses

      +

      200

      + Successful + +

      400

      + Exception occurred; see message for details. + KnownExceptionInfo +

      404

      + Object with given id was not found. + NotFoundKnownExceptionInfo +
      +
      +
      +
      + Up + 
      post /v1/source_oauths/oauth_params/create
      +
      Sets instancewide variables to be used for the oauth flow when creating this source. When set, these variables will be injected into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with consistent variables e.g: the company's Google Ads developer_token, client_id, and client_secret without the user having to know about these variables. (setInstancewideSourceOauthParams)
      +
      + + +

      Consumes

      + This API call consumes the following media types via the Content-Type request header: +
        +
      • application/json
        • +
      + +

      Request body

      +
      +
      SetInstancewideSourceOauthParamsRequestBody SetInstancewideSourceOauthParamsRequestBody (required)
      + +
      Body Parameter —
      + +
      + + + + + + + + +

      Produces

      + This API call produces the following media types according to the Accept request header; + the media type will be conveyed by the Content-Type response header. +
        +
      • application/json
        • +
      + +

      Responses

      +

      200

      + Successful + +

      400

      + Exception occurred; see message for details. + KnownExceptionInfo +

      404

      + Object with given id was not found. + NotFoundKnownExceptionInfo +
      +

      Openapi

      @@ -5972,6 +6070,8 @@

      Table of Contents

    • OperatorType -
    • Pagination -
    • ResourceRequirements -
      • +
    • SetInstancewideDestinationOauthParamsRequestBody -
      • +
    • SetInstancewideSourceOauthParamsRequestBody -
    • SlackNotificationConfiguration -
    • SlugRequestBody -
    • SourceCoreConfig -
      • @@ -6654,6 +6754,22 @@

      ResourceRequirements - memory_limit (optional)

      String
      +
      +

      SetInstancewideDestinationOauthParamsRequestBody - Up

      +
      +
      +
      destinationDefinitionId
      UUID format: uuid
      +
      params
      map[String, Object]
      +
      +
      +
      +

      SetInstancewideSourceOauthParamsRequestBody - Up

      +
      +
      +
      sourceDefinitionId (optional)
      UUID format: uuid
      +
      params
      map[String, Object]
      +
      +

      SlackNotificationConfiguration - Up

      diff --git a/settings.gradle b/settings.gradle index 95fa49f51db1..8e1893211657 100644 --- a/settings.gradle +++ b/settings.gradle @@ -45,7 +45,7 @@ include ':airbyte-notification' // transitively used by airbyte-workers. include ':airbyte-scheduler:models' // transitively used by airbyte-workers. include ':airbyte-scheduler:persistence' // used by airbyte-workers. -// platform +// platformm if(!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "PLATFORM") { include ':airbyte-cli' include ':airbyte-e2e-testing'