add instancewide variables setter endpoints

airbyte-api/src/main/openapi/config.yaml

@@ -1620,6 +1620,52 @@ paths:
                 $ref: "#/components/schemas/ImportRead"
         "404":
           $ref: "#/components/responses/NotFoundResponse"
+  /v1/source_oauths/oauth_params/create:
+    post:
+      tags:
+        - oauth
+      summary: >
+        Sets instancewide variables to be used for the oauth flow when creating this source. When set, these variables will be injected
+        into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with
+        consistent variables e.g: the company's Google Ads developer_token, client_id, and client_secret without the user having to know
+        about these variables.
+      operationId: setInstancewideSourceOauthParams
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/SetInstancewideSourceOauthParamsRequestBody"
+        required: true
+      responses:
+        "200":
+          description: Successful
+        "400":
+          $ref: "#/components/responses/ExceptionResponse"
+        "404":
+          $ref: "#/components/responses/NotFoundResponse"
+  /v1/destination_oauths/oauth_params/create:
+    post:
+      tags:
+        - oauth
+      summary: >
+        Sets instancewide variables to be used for the oauth flow when creating this destination. When set, these variables will be injected
+        into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with
+        consistent variables e.g: the company's Google Ads developer_token, client_id, and client_secret without the user having to know
+        about these variables.
+      operationId: setInstancewideDestinationOauthParams
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/SetInstancewideDestinationOauthParamsRequestBody"
+        required: true
+      responses:
+        "200":
+          description: Successful
+        "400":
+          $ref: "#/components/responses/ExceptionResponse"
+        "404":
+          $ref: "#/components/responses/NotFoundResponse"
 components:
   securitySchemes:
     bearerAuth:
@@ -2995,6 +3041,28 @@ components:
     CompleteOauthResponse:
       type: object
       additionalProperties: true # Oauth parameters like refresh/access token etc.. will be different per API so we don't specify them in advance
+    SetInstancewideSourceOauthParamsRequestBody:
+      type: object
+      required:
+        - sourceConnectorDefinitionId
+        - params
+      properties:
+        sourceDefinitionId:
+          $ref: "#/components/schemas/SourceDefinitionId"
+        params:
+          type: object
+          additionalProperties: true
+    SetInstancewideDestinationOauthParamsRequestBody:
+      type: object
+      required:
+        - destinationDefinitionId
+        - params
+      properties:
+        destinationDefinitionId:
+          $ref: "#/components/schemas/DestinationDefinitionId"
+        params:
+          type: object
+          additionalProperties: true
     # Web Backend
     WebBackendConnectionRead:
       type: object

airbyte-db/lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java

@@ -110,8 +110,8 @@ private static void readBson(final Document document, final ObjectNode o, final
   }
 
   /**
-   * Gets 10.000 documents from collection, gathers all unique fields and its type. In case when one field has different types in 2 and more
-   * documents, the type is set to String.
+   * Gets 10.000 documents from collection, gathers all unique fields and its type. In case when one
+   * field has different types in 2 and more documents, the type is set to String.
    *
    * @param collection mongo collection
    * @return map of unique fields and its type

airbyte-integrations/connectors/source-github/fixtures/github.py

@@ -1,6 +1,31 @@
+#
+# MIT License
+#
+# Copyright (c) 2020 Airbyte
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+
+# type: ignore
 import json
 import logging
-from typing import Optional, Iterator
+from typing import Iterator, Optional
 
 import requests
 
@@ -29,7 +54,7 @@ def __init__(self, token: str, repository: str):
         self.session = requests.Session()
         self.session.headers.update(self.get_headers(self.token))
 
-        self.branches = None
+        self.branches: Optional[list] = None
 
     @staticmethod
     def get_headers(token: str):
@@ -78,19 +103,15 @@ def pull_requests(self) -> Iterator:
                     "event": "COMMENT",
                 }
                 review_url = f"{self.BASE_URL}/repos/{self.repository}/pulls/{response.json().get('number')}/reviews"
-                response = self.session.post(
-                    url=review_url, data=json.dumps(create_review_data)
-                )
+                response = self.session.post(url=review_url, data=json.dumps(create_review_data))
                 yield response
 
                 # create comment for commit
                 create_comment_data = {
                     "body": f'comment for {branch.get("commit").get("sha")} branch',
                 }
                 commit_url = f"https://api.github.com/repos/{self.repository}/commits/{branch.get('commit').get('sha')}/comments"
-                response = self.session.post(
-                    url=commit_url, data=json.dumps(create_comment_data)
-                )
+                response = self.session.post(url=commit_url, data=json.dumps(create_comment_data))
                 yield response
 
     @logger

airbyte-integrations/connectors/source-github/fixtures/main.py

@@ -1,4 +1,31 @@
+#
+# MIT License
+#
+# Copyright (c) 2020 Airbyte
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+
+# type: ignore
+# this is just a script that doesn't need mypy linting
 import sys
+
 from github import GitHubFiller
 
 if __name__ == "__main__":

airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAcceptanceTest.java

@@ -115,10 +115,10 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception {
             .withDestinationSyncMode(DestinationSyncMode.APPEND)
             .withCursorField(List.of("_id"))
             .withStream(CatalogHelpers.createAirbyteStream(
-                    "test.acceptance_test",
-                    Field.of("_id", JsonSchemaPrimitive.STRING),
-                    Field.of("id", JsonSchemaPrimitive.STRING),
-                    Field.of("name", JsonSchemaPrimitive.STRING))
+                "test.acceptance_test",
+                Field.of("_id", JsonSchemaPrimitive.STRING),
+                Field.of("id", JsonSchemaPrimitive.STRING),
+                Field.of("name", JsonSchemaPrimitive.STRING))
                 .withSupportedSyncModes(Lists.newArrayList(SyncMode.INCREMENTAL))
                 .withDefaultCursorField(List.of("_id")))));
   }

airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java

@@ -67,6 +67,8 @@
 import io.airbyte.api.model.OperationReadList;
 import io.airbyte.api.model.OperationUpdate;
 import io.airbyte.api.model.OperatorConfiguration;
+import io.airbyte.api.model.SetInstancewideDestinationOauthParamsRequestBody;
+import io.airbyte.api.model.SetInstancewideSourceOauthParamsRequestBody;
 import io.airbyte.api.model.SlugRequestBody;
 import io.airbyte.api.model.SourceCoreConfig;
 import io.airbyte.api.model.SourceCreate;
@@ -276,7 +278,7 @@ public SourceDefinitionSpecificationRead getSourceDefinitionSpecification(final
     return execute(() -> schedulerHandler.getSourceDefinitionSpecification(sourceDefinitionIdRequestBody));
   }
 
-  // SOURCE OAUTH
+  // OAUTH
 
   @Override
   public OAuthConsentRead getSourceOAuthConsent(SourceOauthConsentRequest sourceOauthConsentRequest) {
@@ -288,6 +290,32 @@ public Map<String, Object> completeSourceOAuth(CompleteSourceOauthRequest comple
     return execute(() -> oAuthHandler.completeSourceOAuth(completeSourceOauthRequest));
   }
 
+  @Override
+  public OAuthConsentRead getDestinationOAuthConsent(DestinationOauthConsentRequest destinationOauthConsentRequest) {
+    return execute(() -> oAuthHandler.getDestinationOAuthConsent(destinationOauthConsentRequest));
+  }
+
+  @Override
+  public Map<String, Object> completeDestinationOAuth(CompleteDestinationOAuthRequest requestBody) {
+    return execute(() -> oAuthHandler.completeDestinationOAuth(requestBody));
+  }
+
+  @Override
+  public void setInstancewideDestinationOauthParams(SetInstancewideDestinationOauthParamsRequestBody requestBody) {
+    execute(() -> {
+      oAuthHandler.setDestinationInstancewideOauthParams(requestBody);
+      return null;
+    });
+  }
+
+  @Override
+  public void setInstancewideSourceOauthParams(SetInstancewideSourceOauthParamsRequestBody requestBody) {
+    execute(() -> {
+      oAuthHandler.setSourceInstancewideOauthParams(requestBody);
+      return null;
+    });
+  }
+
   // SOURCE IMPLEMENTATION
 
   @Override
@@ -379,17 +407,6 @@ public DestinationDefinitionSpecificationRead getDestinationDefinitionSpecificat
     return execute(() -> schedulerHandler.getDestinationSpecification(destinationDefinitionIdRequestBody));
   }
 
-  // DESTINATION OAUTH
-  @Override
-  public OAuthConsentRead getDestinationOAuthConsent(DestinationOauthConsentRequest destinationOauthConsentRequest) {
-    return execute(() -> oAuthHandler.getDestinationOAuthConsent(destinationOauthConsentRequest));
-  }
-
-  @Override
-  public Map<String, Object> completeDestinationOAuth(CompleteDestinationOAuthRequest requestBody) {
-    return execute(() -> oAuthHandler.completeDestinationOAuth(requestBody));
-  }
-
   // DESTINATION IMPLEMENTATION
 
   @Override

airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java

@@ -28,7 +28,12 @@
 import io.airbyte.api.model.CompleteSourceOauthRequest;
 import io.airbyte.api.model.DestinationOauthConsentRequest;
 import io.airbyte.api.model.OAuthConsentRead;
+import io.airbyte.api.model.SetInstancewideDestinationOauthParamsRequestBody;
+import io.airbyte.api.model.SetInstancewideSourceOauthParamsRequestBody;
 import io.airbyte.api.model.SourceOauthConsentRequest;
+import io.airbyte.commons.json.Jsons;
+import io.airbyte.config.DestinationOAuthParameter;
+import io.airbyte.config.SourceOAuthParameter;
 import io.airbyte.config.StandardDestinationDefinition;
 import io.airbyte.config.StandardSourceDefinition;
 import io.airbyte.config.persistence.ConfigNotFoundException;
@@ -74,6 +79,23 @@ public Map<String, Object> completeDestinationOAuth(CompleteDestinationOAuthRequ
     return oAuthFlowImplementation.completeOAuth(oauthDestinationRequestBody.getWorkspaceId(), oauthDestinationRequestBody.getQueryParams());
   }
 
+  public void setDestinationInstancewideOauthParams(SetInstancewideDestinationOauthParamsRequestBody requestBody)
+      throws JsonValidationException, IOException {
+    DestinationOAuthParameter param = new DestinationOAuthParameter()
+        .withOauthParameterId(UUID.randomUUID())
+        .withConfiguration(Jsons.jsonNode(requestBody.getParams()))
+        .withDestinationDefinitionId(requestBody.getDestinationDefinitionId());
+    configRepository.writeDestinationOAuthParam(param);
+  }
+
+  public void setSourceInstancewideOauthParams(SetInstancewideSourceOauthParamsRequestBody requestBody) throws JsonValidationException, IOException {
+    SourceOAuthParameter param = new SourceOAuthParameter()
+        .withOauthParameterId(UUID.randomUUID())
+        .withConfiguration(Jsons.jsonNode(requestBody.getParams()))
+        .withSourceDefinitionId(requestBody.getSourceDefinitionId());
+    configRepository.writeSourceOAuthParam(param);
+  }
+
   private OAuthFlowImplementation getSourceOAuthFlowImplementation(UUID sourceDefinitionId)
       throws JsonValidationException, ConfigNotFoundException, IOException {
     final StandardSourceDefinition standardSourceDefinition = configRepository