🎉 Source Gitlab: add OAuth2.0 authentication support

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -566,7 +566,7 @@
   documentationUrl: https://docs.airbyte.com/integrations/sources/gitlab
   icon: gitlab.svg
   sourceType: api
-  releaseStage: alpha
+  releaseStage: beta
 - name: Glassfrog
   sourceDefinitionId: cf8ff320-6272-4faa-89e6-4402dc17e5d5
   dockerRepository: airbyte/source-glassfrog

airbyte-integrations/connectors/source-gitlab/Dockerfile

@@ -13,5 +13,5 @@ COPY main.py ./
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.12
+LABEL io.airbyte.version=1.0.0
 LABEL io.airbyte.name=airbyte/source-gitlab

airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml

@@ -4,15 +4,24 @@ acceptance_tests:
   spec:
     tests:
       - spec_path: "source_gitlab/spec.json"
+        backward_compatibility_tests_config:
+          disable_for_version: "0.1.12"
   connection:
     tests:
       - config_path: "secrets/config.json"
         status: "succeed"
+      - config_path: "secrets/config_oauth.json"
+        status: "succeed"
       - config_path: "integration_tests/invalid_config.json"
         status: "failed"
   discovery:
     tests:
       - config_path: "secrets/config.json"
+        backward_compatibility_tests_config:
+          disable_for_version: "0.1.12"
+      - config_path: "secrets/config_oauth.json"
+        backward_compatibility_tests_config:
+          disable_for_version: "0.1.12"
   basic_read:
     tests:
       - config_path: "secrets/config.json"
@@ -26,6 +35,9 @@ acceptance_tests:
             bypass_reason: "Group in this config does not have epics issues. This stream is tested in the above TC."
         expect_records:
           path: "integration_tests/expected_records_with_ids.txt"
+      - config_path: "secrets/config_oauth.json"
+        expect_records:
+          path: "integration_tests/expected_records.txt"
   incremental:
     tests:
       - config_path: "secrets/config_with_ids.json"

airbyte-integrations/connectors/source-gitlab/integration_tests/invalid_config.json

@@ -1,7 +1,11 @@
 {
   "api_url": "gitlab.com",
   "private_token": "private_token-fake",
-  "groups": "new-group",
-  "projects": "new-ci-test",
-  "start_date": "2021-01-01T00:00:00Z"
-}
+  "start_date": "2021-01-01T00:00:00Z",
+  "groups": "new-group-airbute",
+  "projects": "new-group-airbute/new-ci-test-project",
+  "credentials": {
+      "auth_type": "access_token",
+      "access_token": "migrated_from_old_config"
+  }
+}

airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py

@@ -5,10 +5,14 @@
 
 from typing import Any, List, Mapping, MutableMapping, Optional, Tuple, Union
 
+import pendulum
+import requests
 from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
-from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator
+from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import SingleUseRefreshTokenOauth2Authenticator
+from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator
+from requests.auth import AuthBase
 
 from .streams import (
     Branches,
@@ -40,6 +44,34 @@
 )
 
 
+class GitlabOauth2Authenticator(SingleUseRefreshTokenOauth2Authenticator):
+    def __init__(self, connector_config, *args, **kwargs):
+        super().__init__(connector_config, *args, **kwargs)
+        access_token = connector_config["credentials"].get("access_token")
+        if not access_token:
+            return
+        api_url = self._connector_config["api_url"]
+        token_valid, access_token_info = self.get_access_token_info(api_url, access_token)
+        if not token_valid:
+            return
+        now = pendulum.now()
+        self.access_token = access_token
+        self.set_token_expiry_date(now.add(seconds=access_token_info["expires_in"]))
+
+    @staticmethod
+    def get_access_token_info(api_url: str, access_token: str) -> Tuple[bool, MutableMapping]:
+        response = requests.get(f"https://{api_url}/oauth/token/info?access_token={access_token}")
+        if response.status_code == 200:
+            return True, response.json()
+        return False, {}
+
+
+def get_authenticator(config: MutableMapping) -> AuthBase:
+    if config["credentials"]["auth_type"] == "access_token":
+        return TokenAuthenticator(token=config["credentials"]["access_token"])
+    return GitlabOauth2Authenticator(config, token_refresh_endpoint=f"https://{config['api_url']}/oauth/token")
+
+
 class SourceGitlab(AbstractSource):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -67,7 +99,7 @@ def _projects_stream(self, config: MutableMapping[str, Any]) -> Union[Projects,
 
     def _auth_params(self, config: MutableMapping[str, Any]) -> Mapping[str, Any]:
         if not self.__auth_params:
-            auth = TokenAuthenticator(token=config["private_token"])
+            auth = get_authenticator(config)
             self.__auth_params = dict(authenticator=auth, api_url=config["api_url"])
         return self.__auth_params
 

airbyte-integrations/connectors/source-gitlab/source_gitlab/spec.json

@@ -2,22 +2,71 @@
   "documentationUrl": "https://docs.airbyte.com/integrations/sources/gitlab",
   "connectionSpecification": {
     "$schema": "http://json-schema.org/draft-07/schema#",
-    "title": "Source GitLab Singer Spec",
+    "title": "Source Gitlab Spec",
     "type": "object",
-    "required": ["api_url", "private_token", "start_date"],
+    "required": ["api_url", "start_date", "credentials"],
     "additionalProperties": true,
     "properties": {
-      "private_token": {
-        "type": "string",
-        "title": "Private Token",
-        "description": "Log into your GitLab account and then generate a personal [Access Token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html).",
-        "airbyte_secret": true,
-        "order": 0
+      "credentials": {
+        "title": "Authorization Method",
+        "type": "object",
+        "order": 0,
+        "oneOf": [
+          {
+            "type": "object",
+            "title": "OAuth2.0",
+            "required": ["client_id", "client_secret", "refresh_token"],
+            "properties": {
+              "auth_type": {
+                "type": "string",
+                "const": "oauth2.0"
+              },
+              "client_id": {
+                "type": "string",
+                "description": "The API ID of the Gitlab developer application.",
+                "airbyte_secret": true
+              },
+              "client_secret": {
+                "type": "string",
+                "description": "The API Secret the Gitlab developer application.",
+                "airbyte_secret": true
+              },
+              "access_token": {
+                "type": "string",
+                "description": "Access Token for making authenticated requests.",
+                "airbyte_secret": true
+              },
+              "refresh_token": {
+                "type": "string",
+                "description": "The key to refresh the expired access_token.",
+                "airbyte_secret": true
+              }
+            }
+          },
+          {
+            "title": "Private Token",
+            "type": "object",
+            "required": ["access_token"],
+            "properties": {
+              "auth_type": {
+                "type": "string",
+                "const": "access_token"
+              },
+              "access_token": {
+                "type": "string",
+                "title": "Private Token",
+                "description": "Log into your Gitlab account and then generate a personal Access Token.",
+                "airbyte_secret": true
+              }
+            }
+          }
+        ]
       },
       "api_url": {
         "type": "string",
         "examples": ["gitlab.com"],
         "title": "API URL",
+        "default": "gitlab.com",
         "description": "Please enter your basic URL from GitLab instance.",
         "order": 1
       },
@@ -44,5 +93,58 @@
         "order": 4
       }
     }
+  },
+  "advanced_auth": {
+    "auth_flow_type": "oauth2.0",
+    "predicate_key": ["credentials", "auth_type"],
+    "predicate_value": "oauth2.0",
+    "oauth_config_specification": {
+      "oauth_user_input_from_connector_config_specification": {
+        "type": "object",
+        "properties": {
+          "domain": {
+            "type": "string",
+            "path_in_connector_config": ["api_url"]
+          }
+        }
+      },
+      "complete_oauth_output_specification": {
+        "type": "object",
+        "properties": {
+          "access_token": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "access_token"]
+          },
+          "refresh_token": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "refresh_token"]
+          }
+        }
+      },
+      "complete_oauth_server_input_specification": {
+        "type": "object",
+        "properties": {
+          "client_id": {
+            "type": "string"
+          },
+          "client_secret": {
+            "type": "string"
+          }
+        }
+      },
+      "complete_oauth_server_output_specification": {
+        "type": "object",
+        "properties": {
+          "client_id": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "client_id"]
+          },
+          "client_secret": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "client_secret"]
+          }
+        }
+      }
+    }
   }
 }

airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py

@@ -10,5 +10,8 @@ def config(mocker):
     return {
         "start_date": "2021-01-01T00:00:00Z",
         "api_url": "gitlab.com",
-        "private_token": "secret_token"
+        "credentials": {
+            "auth_type": "access_token",
+            "access_token": "token"
+        }
     }

airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java

@@ -52,6 +52,7 @@ public OAuthImplementationFactory(final ConfigRepository configRepository, final
         .put("airbyte/source-strava", new StravaOAuthFlow(configRepository, httpClient))
         .put("airbyte/source-surveymonkey", new SurveymonkeyOAuthFlow(configRepository, httpClient))
         .put("airbyte/source-trello", new TrelloOAuthFlow(configRepository))
+        .put("airbyte/source-gitlab", new GitlabOAuthFlow(configRepository, httpClient))
         .put("airbyte/source-youtube-analytics", new YouTubeAnalyticsOAuthFlow(configRepository, httpClient))
         // revert me
         .put("airbyte/source-youtube-analytics-business", new YouTubeAnalyticsBusinessOAuthFlow(configRepository, httpClient))

airbyte-oauth/src/main/java/io/airbyte/oauth/flows/GitlabOAuthFlow.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.oauth.flows;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.collect.ImmutableMap;
+import io.airbyte.config.persistence.ConfigNotFoundException;
+import io.airbyte.config.persistence.ConfigRepository;
+import io.airbyte.oauth.BaseOAuth2Flow;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.http.HttpClient;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Supplier;
+import org.apache.http.client.utils.URIBuilder;
+
+/**
+ * Following docs from https://docs.gitlab.com/ee/api/oauth2.html#authorization-code-flow
+ */
+public class GitlabOAuthFlow extends BaseOAuth2Flow {
+
+  private static final String ACCESS_TOKEN_URL = "https://%s/oauth/token";
+
+  public GitlabOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) {
+    super(configRepository, httpClient);
+  }
+
+  public GitlabOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier<String> stateSupplier) {
+    super(configRepository, httpClient, stateSupplier);
+  }
+
+  protected static String getDomain(JsonNode inputOAuthConfiguration) throws IOException {
+    final var domain = inputOAuthConfiguration.get("domain");
+    if (domain == null) {
+      throw new IOException("Domain field is empty.");
+    }
+    return domain.asText();
+  }
+
+  @Override
+  protected String formatConsentUrl(final UUID definitionId, final String clientId, final String redirectUrl, final JsonNode inputOAuthConfiguration)
+      throws IOException {
+    final URIBuilder builder = new URIBuilder()
+        .setScheme("https")
+        .setHost(getDomain(inputOAuthConfiguration))
+        .setPath("oauth/authorize")
+        .addParameter("client_id", clientId)
+        .addParameter("redirect_uri", redirectUrl)
+        .addParameter("state", getState())
+        .addParameter("response_type", "code")
+        .addParameter("scope", "read_api");
+    try {
+      return builder.build().toString();
+    } catch (URISyntaxException e) {
+      throw new IOException("Failed to format Consent URL for OAuth flow", e);
+    }
+  }
+
+  @Override
+  protected String getAccessTokenUrl(final JsonNode inputOAuthConfiguration) {
+    final var domain = inputOAuthConfiguration.get("domain");
+    return String.format(ACCESS_TOKEN_URL, domain == null ? "gitlab.com" : domain.asText());
+  }
+
+  @Override
+  protected Map<String, String> getAccessTokenQueryParameters(final String clientId,
+                                                              final String clientSecret,
+                                                              final String authCode,
+                                                              final String redirectUrl) {
+    return ImmutableMap.<String, String>builder()
+        .put("client_id", clientId)
+        .put("client_secret", clientSecret)
+        .put("code", authCode)
+        .put("grant_type", "authorization_code")
+        .put("redirect_uri", redirectUrl)
+        .build();
+  }
+
+  @Override
+  protected Map<String, Object> extractOAuthOutput(final JsonNode data, final String accessTokenUrl) throws IOException {
+    final Map<String, Object> result = new HashMap<>();
+    // check for refresh_token after successful authentication
+    if (data.has("refresh_token")) {
+      result.put("refresh_token", data.get("refresh_token").asText());
+    } else {
+      throw new IOException(String.format("Missing 'refresh_token' in query params from %s", accessTokenUrl));
+    }
+    // check for access_token after successful authentication
+    if (data.has("access_token")) {
+      result.put("access_token", data.get("access_token").asText());
+    } else {
+      throw new IOException(String.format("Missing 'access_token' in query params from %s", accessTokenUrl));
+    }
+    return result;
+  }
+
+  @Override
+  @Deprecated
+  public Map<String, Object> completeSourceOAuth(final UUID workspaceId,
+                                                 final UUID sourceDefinitionId,
+                                                 final Map<String, Object> queryParams,
+                                                 final String redirectUrl)
+      throws IOException, ConfigNotFoundException {
+    throw new IOException("Deprecated API not supported by this connector");
+  }
+
+}