🎉 Source Gitlab: add OAuth2.0 authentication support

.github/workflows/publish-command.yml

@@ -100,6 +100,7 @@ jobs:
           FILE_SECURE_HTTPS_TEST_CREDS: ${{ secrets.FILE_SECURE_HTTPS_TEST_CREDS }}
           FRESHDESK_TEST_CREDS: ${{ secrets.FRESHDESK_TEST_CREDS }}
           GITLAB_INTEGRATION_TEST_CREDS: ${{ secrets.GITLAB_INTEGRATION_TEST_CREDS }}
+          GITLAB_INTEGRATION_TEST_OAUTH_CREDS: ${{ secrets.GITLAB_INTEGRATION_TEST_OAUTH_CREDS }}
           GH_NATIVE_INTEGRATION_TEST_CREDS: ${{ secrets.GH_NATIVE_INTEGRATION_TEST_CREDS }}
           GOOGLE_ADS_TEST_CREDS: ${{ secrets.GOOGLE_ADS_TEST_CREDS }}
           GOOGLE_ANALYTICS_V4_TEST_CREDS: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS }}

.github/workflows/test-command.yml

@@ -95,6 +95,7 @@ jobs:
           FILE_SECURE_HTTPS_TEST_CREDS: ${{ secrets.FILE_SECURE_HTTPS_TEST_CREDS }}
           FRESHDESK_TEST_CREDS: ${{ secrets.FRESHDESK_TEST_CREDS }}
           GITLAB_INTEGRATION_TEST_CREDS: ${{ secrets.GITLAB_INTEGRATION_TEST_CREDS }}
+          GITLAB_INTEGRATION_TEST_OAUTH_CREDS: ${{ secrets.GITLAB_INTEGRATION_TEST_OAUTH_CREDS }}
           GH_NATIVE_INTEGRATION_TEST_CREDS: ${{ secrets.GH_NATIVE_INTEGRATION_TEST_CREDS }}
           GOOGLE_ADS_TEST_CREDS: ${{ secrets.GOOGLE_ADS_TEST_CREDS }}
           GOOGLE_ANALYTICS_V4_TEST_CREDS: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS }}

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/5e6175e5-68e1-4c17-bff9-56103bbb0d80.json

@@ -2,6 +2,6 @@
   "sourceDefinitionId": "5e6175e5-68e1-4c17-bff9-56103bbb0d80",
   "name": "Gitlab",
   "dockerRepository": "airbyte/source-gitlab",
-  "dockerImageTag": "0.1.2",
+  "dockerImageTag": "0.1.3",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/gitlab"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -167,7 +167,7 @@
 - name: Gitlab
   sourceDefinitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80
   dockerRepository: airbyte/source-gitlab
-  dockerImageTag: 0.1.2
+  dockerImageTag: 0.1.3
   documentationUrl: https://docs.airbyte.io/integrations/sources/gitlab
   sourceType: api
 - name: Google Ads

airbyte-integrations/connectors/source-gitlab/Dockerfile

@@ -13,5 +13,5 @@ RUN pip install .
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.2
+LABEL io.airbyte.version=0.1.3
 LABEL io.airbyte.name=airbyte/source-gitlab

airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml

@@ -5,13 +5,20 @@ tests:
   connection:
     - config_path: "secrets/config.json"
       status: "succeed"
+    - config_path: "secrets/config_oauth.json"
+      status: "succeed"
     - config_path: "integration_tests/invalid_config.json"
       status: "failed"
   discovery:
     - config_path: "secrets/config.json"
+    - config_path: "secrets/config_oauth.json"
   basic_read:
+    # test read with old config
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
+    # test read with OAuth2.0
+    - config_path: "secrets/config_oauth.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
 # We cannot use these tests for testing Incremental, since for Gitlab the State is saved for each Project separately,
 # and the Acceptance Tests at this stage do not support this functionality.
 # Therefore, we hardcode the cursor_paths for our config.

airbyte-integrations/connectors/source-gitlab/integration_tests/invalid_config.json

@@ -1,7 +1,11 @@
 {
   "api_url": "gitlab.com",
   "private_token": "private_token-fake",
-  "groups": "new-group",
-  "projects": "new-ci-test",
-  "start_date": "2021-01-01T00:00:00Z"
-}
+  "start_date": "2021-01-01T00:00:00Z",
+  "groups": "new-group-airbute",
+  "projects": "new-group-airbute/new-ci-test-project",
+  "credentials": {
+      "credentials": "access_token",
+      "access_token": "migrated_from_old_config"
+  }
+}

airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py

@@ -3,7 +3,7 @@
 #
 
 
-from typing import Any, List, Mapping, Tuple
+from typing import Any, Dict, List, Mapping, Tuple
 
 from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources import AbstractSource
@@ -37,6 +37,21 @@
 )
 
 
+class GitlabAuthenticator(TokenAuthenticator):
+    def __init__(self, config: Dict):
+        self.config = config
+
+    def get_auth(self) -> TokenAuthenticator:
+        private_token = self.config.get("private_token")
+        oauth_token = self.config.get("credentials")
+        if private_token:
+            # support of old config
+            return TokenAuthenticator(token=private_token)
+        if oauth_token:
+            # support of new config with oauth2.0
+            return TokenAuthenticator(token=oauth_token.get("access_token"))
+
+
 class SourceGitlab(AbstractSource):
     def _generate_main_streams(self, config: Mapping[str, Any]) -> Tuple[GitlabStream, GitlabStream]:
         gids = list(filter(None, config["groups"].split(" ")))
@@ -45,7 +60,7 @@ def _generate_main_streams(self, config: Mapping[str, Any]) -> Tuple[GitlabStrea
         if not pids and not gids:
             raise Exception("Either groups or projects need to be provided for connect to Gitlab API")
 
-        auth = TokenAuthenticator(token=config["private_token"])
+        auth = GitlabAuthenticator(config).get_auth()
         auth_params = dict(authenticator=auth, api_url=config["api_url"])
         groups = Groups(group_ids=gids, **auth_params)
         if gids:
@@ -65,7 +80,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]:
             return False, f"Unable to connect to Gitlab API with the provided credentials - {repr(error)}"
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
-        auth = TokenAuthenticator(token=config["private_token"])
+        auth = GitlabAuthenticator(config).get_auth()
         auth_params = dict(authenticator=auth, api_url=config["api_url"])
 
         groups, projects = self._generate_main_streams(config)

airbyte-integrations/connectors/source-gitlab/source_gitlab/spec.json

@@ -2,37 +2,106 @@
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/gitlab",
   "connectionSpecification": {
     "$schema": "http://json-schema.org/draft-07/schema#",
-    "title": "Source Gitlab Singer Spec",
+    "title": "Source Gitlab Spec",
     "type": "object",
-    "required": ["api_url", "private_token", "start_date"],
-    "additionalProperties": false,
+    "required": ["api_url", "credentials", "start_date"],
+    "additionalProperties": true,
     "properties": {
       "api_url": {
         "type": "string",
+        "title": "API URL",
         "examples": ["gitlab.com"],
+        "default": "gitlab.com",
         "description": "Please enter your basic URL from Gitlab instance"
       },
-      "private_token": {
-        "type": "string",
-        "description": "Log into your Gitlab account and then generate a personal Access Token.",
-        "airbyte_secret": true
-      },
       "groups": {
         "type": "string",
+        "title": "Groups",
         "examples": ["airbyte.io"],
         "description": "Space-delimited list of groups. e.g. airbyte.io"
       },
       "projects": {
         "type": "string",
+        "title": "Projects",
         "examples": ["airbyte.io/documentation"],
         "description": "Space-delimited list of projects. e.g. airbyte.io/documentation meltano/tap-gitlab"
       },
       "start_date": {
         "type": "string",
+        "title": "Start Date",
         "description": "The date from which you'd like to replicate data for Gitlab API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.",
         "examples": ["2021-03-01T00:00:00Z"],
         "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+      },
+      "credentials": {
+        "title": "Authorization Method",
+        "type": "object",
+        "oneOf": [
+          {
+            "type": "object",
+            "title": "OAuth2.0",
+            "required": ["client_id", "client_secret", "access_token"],
+            "properties": {
+              "credentials": {
+                "type": "string",
+                "const": "oauth2.0",
+                "enum": ["oauth2.0"],
+                "default": "oauth2.0",
+                "order": 0
+              },
+              "client_id": {
+                "type": "string",
+                "description": "The API ID of the Gitlab developer application.",
+                "airbyte_secret": true
+              },
+              "client_secret": {
+                "type": "string",
+                "description": "The API Secret the Gitlab developer application.",
+                "airbyte_secret": true
+              },
+              "access_token": {
+                "type": "string",
+                "description": "Access Token for making authenticated requests.",
+                "airbyte_secret": true
+              },
+              "refresh_token": {
+                "type": "string",
+                "description": "The key to refresh the expired access_token.",
+                "airbyte_secret": true
+              }
+            }
+          },
+          {
+            "title": "Private Token",
+            "type": "object",
+            "required": ["access_token"],
+            "properties": {
+              "credentials": {
+                "type": "string",
+                "const": "access_token",
+                "enum": ["access_token"],
+                "default": "access_token",
+                "order": 0
+              },
+              "access_token": {
+                "type": "string",
+                "title": "Private Token",
+                "description": "Log into your Gitlab account and then generate a personal Access Token.",
+                "airbyte_secret": true
+              }
+            }
+          }
+        ]
       }
     }
+  },
+  "authSpecification": {
+    "auth_type": "oauth2.0",
+    "oauth2Specification": {
+      "rootObject": ["credentials", "0"],
+      "oauthFlowInitParameters": [["client_id"], ["client_secret"]],
+      "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]]
+    }
   }
 }
+

airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java

@@ -8,6 +8,7 @@
 import io.airbyte.config.persistence.ConfigRepository;
 import io.airbyte.oauth.flows.AsanaOAuthFlow;
 import io.airbyte.oauth.flows.GithubOAuthFlow;
+import io.airbyte.oauth.flows.GitlabOAuthFlow;
 import io.airbyte.oauth.flows.SalesforceOAuthFlow;
 import io.airbyte.oauth.flows.TrelloOAuthFlow;
 import io.airbyte.oauth.flows.facebook.FacebookMarketingOAuthFlow;
@@ -37,6 +38,7 @@ public OAuthImplementationFactory(final ConfigRepository configRepository) {
         .put("airbyte/source-instagram", new InstagramOAuthFlow(configRepository))
         .put("airbyte/source-salesforce", new SalesforceOAuthFlow(configRepository))
         .put("airbyte/source-trello", new TrelloOAuthFlow(configRepository))
+        .put("airbyte/source-gitlab", new GitlabOAuthFlow(configRepository))
         .build();
   }
 

airbyte-oauth/src/main/java/io/airbyte/oauth/flows/GitlabOAuthFlow.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.oauth.flows;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import io.airbyte.config.persistence.ConfigRepository;
+import io.airbyte.oauth.BaseOAuthFlow;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.http.HttpClient;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Supplier;
+import org.apache.http.client.utils.URIBuilder;
+
+/**
+ * Following docs from https://docs.gitlab.com/ee/api/oauth2.html#authorization-code-flow
+ */
+public class GitlabOAuthFlow extends BaseOAuthFlow {
+
+  private static final String ACCESS_TOKEN_URL = "https://gitlab.com/oauth/token";
+
+  public GitlabOAuthFlow(ConfigRepository configRepository) {
+    super(configRepository);
+  }
+
+  @VisibleForTesting
+  GitlabOAuthFlow(ConfigRepository configRepository, HttpClient httpClient, Supplier<String> stateSupplier) {
+    super(configRepository, httpClient, stateSupplier);
+  }
+
+  @Override
+  protected String formatConsentUrl(UUID definitionId, String clientId, String redirectUrl) throws IOException {
+    final URIBuilder builder = new URIBuilder()
+        .setScheme("https")
+        .setHost("gitlab.com")
+        .setPath("oauth/authorize")
+        .addParameter("client_id", clientId)
+        .addParameter("redirect_uri", redirectUrl)
+        .addParameter("state", getState())
+        .addParameter("response_type", "code")
+        .addParameter("scope", "read_api");
+    try {
+      return builder.build().toString();
+    } catch (URISyntaxException e) {
+      throw new IOException("Failed to format Consent URL for OAuth flow", e);
+    }
+  }
+
+  @Override
+  protected String extractCodeParameter(Map<String, Object> queryParams) throws IOException {
+    if (queryParams.containsKey("code")) {
+      return (String) queryParams.get("code");
+    } else {
+      throw new IOException("Undefined 'code' from consent redirected url.");
+    }
+  }
+
+  @Override
+  protected String getClientIdUnsafe(JsonNode config) {
+    // the config object containing client ID is nested inside the "credentials" object
+    Preconditions.checkArgument(config.hasNonNull("credentials"));
+    return super.getClientIdUnsafe(config.get("credentials"));
+  }
+
+  @Override
+  protected String getClientSecretUnsafe(JsonNode config) {
+    // the config object containing client SECRET is nested inside the "credentials" object
+    Preconditions.checkArgument(config.hasNonNull("credentials"));
+    return super.getClientSecretUnsafe(config.get("credentials"));
+  }
+
+  @Override
+  protected String getAccessTokenUrl() {
+    return ACCESS_TOKEN_URL;
+  }
+
+  @Override
+  protected Map<String, String> getAccessTokenQueryParameters(String clientId, String clientSecret, String authCode, String redirectUrl) {
+    return ImmutableMap.<String, String>builder()
+        .put("client_id", clientId)
+        .put("client_secret", clientSecret)
+        .put("code", authCode)
+        .put("grant_type", "authorization_code")
+        .put("redirect_uri", redirectUrl)
+        .build();
+  }
+
+  @Override
+  protected Map<String, Object> extractRefreshToken(final JsonNode data, String accessTokenUrl) throws IOException {
+    final Map<String, Object> result = new HashMap<>();
+    // check for refresh_token after successful authentication
+    if (data.has("refresh_token")) {
+      result.put("refresh_token", data.get("refresh_token").asText());
+    } else {
+      throw new IOException(String.format("Missing 'refresh_token' in query params from %s", accessTokenUrl));
+    }
+    // check for access_token after successful authentication
+    if (data.has("access_token")) {
+      result.put("access_token", data.get("access_token").asText());
+    } else {
+      throw new IOException(String.format("Missing 'access_token' in query params from %s", accessTokenUrl));
+    }
+    // return result as mapping
+    return Map.of("credentials", result);
+  }
+
+}