Skip to content

Commit

Permalink
Implement Google Analytics & Google Ads OAuth Flow (#5911)
Browse files Browse the repository at this point in the history
Co-authored-by: Sherif Nada <snadalive@gmail.com>
  • Loading branch information
ChristopheDuong and sherifnada authored Sep 13, 2021
1 parent 51e2718 commit da34bef
Show file tree
Hide file tree
Showing 16 changed files with 1,032 additions and 91 deletions.
110 changes: 64 additions & 46 deletions airbyte-api/src/main/openapi/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1184,6 +1184,29 @@ paths:
$ref: "#/components/responses/NotFoundResponse"
"422":
$ref: "#/components/responses/InvalidInputResponse"
/v1/source_oauths/oauth_params/create:
post:
tags:
- oauth
summary: >
Sets instancewide variables to be used for the oauth flow when creating this source. When set, these variables will be injected
into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with
consistent variables e.g: the company's Google Ads developer_token, client_id, and client_secret without the user having to know
about these variables.
operationId: setInstancewideSourceOauthParams
requestBody:
content:
application/json:
schema:
$ref: "#/components/schemas/SetInstancewideSourceOauthParamsRequestBody"
required: true
responses:
"200":
description: Successful
"400":
$ref: "#/components/responses/ExceptionResponse"
"404":
$ref: "#/components/responses/NotFoundResponse"
/v1/source_oauths/get_consent_url:
post:
tags:
Expand Down Expand Up @@ -1276,6 +1299,29 @@ paths:
$ref: "#/components/responses/NotFoundResponse"
"422":
$ref: "#/components/responses/InvalidInputResponse"
/v1/destination_oauths/oauth_params/create:
post:
tags:
- oauth
summary: >
Sets instancewide variables to be used for the oauth flow when creating this destination. When set, these variables will be injected
into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with
consistent variables e.g: the company's Google Ads developer_token, client_id, and client_secret without the user having to know
about these variables.
operationId: setInstancewideDestinationOauthParams
requestBody:
content:
application/json:
schema:
$ref: "#/components/schemas/SetInstancewideDestinationOauthParamsRequestBody"
required: true
responses:
"200":
description: Successful
"400":
$ref: "#/components/responses/ExceptionResponse"
"404":
$ref: "#/components/responses/NotFoundResponse"
/v1/web_backend/connections/list:
post:
tags:
Expand Down Expand Up @@ -1620,52 +1666,6 @@ paths:
$ref: "#/components/schemas/ImportRead"
"404":
$ref: "#/components/responses/NotFoundResponse"
/v1/source_oauths/oauth_params/create:
post:
tags:
- oauth
summary: >
Sets instancewide variables to be used for the oauth flow when creating this source. When set, these variables will be injected
into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with
consistent variables e.g: the company's Google Ads developer_token, client_id, and client_secret without the user having to know
about these variables.
operationId: setInstancewideSourceOauthParams
requestBody:
content:
application/json:
schema:
$ref: "#/components/schemas/SetInstancewideSourceOauthParamsRequestBody"
required: true
responses:
"200":
description: Successful
"400":
$ref: "#/components/responses/ExceptionResponse"
"404":
$ref: "#/components/responses/NotFoundResponse"
/v1/destination_oauths/oauth_params/create:
post:
tags:
- oauth
summary: >
Sets instancewide variables to be used for the oauth flow when creating this destination. When set, these variables will be injected
into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with
consistent variables e.g: the company's Google Ads developer_token, client_id, and client_secret without the user having to know
about these variables.
operationId: setInstancewideDestinationOauthParams
requestBody:
content:
application/json:
schema:
$ref: "#/components/schemas/SetInstancewideDestinationOauthParamsRequestBody"
required: true
responses:
"200":
description: Successful
"400":
$ref: "#/components/responses/ExceptionResponse"
"404":
$ref: "#/components/responses/NotFoundResponse"
components:
securitySchemes:
bearerAuth:
Expand Down Expand Up @@ -2991,20 +2991,30 @@ components:
type: object
required:
- sourceDefinitionId
- workspaceId
- redirectUrl
properties:
sourceDefinitionId:
$ref: "#/components/schemas/SourceDefinitionId"
workspaceId:
$ref: "#/components/schemas/WorkspaceId"
redirectUrl:
description: The url to redirect to after getting the user consent
type: string
DestinationOauthConsentRequest:
type: object
required:
- destinationDefinitionId
- workspaceId
- redirectUrl
properties:
destinationDefinitionId:
$ref: "#/components/schemas/DestinationDefinitionId"
workspaceId:
$ref: "#/components/schemas/WorkspaceId"
redirectUrl:
description: The url to redirect to after getting the user consent
type: string
OAuthConsentRead:
type: object
required:
Expand All @@ -3016,11 +3026,15 @@ components:
type: object
required:
- sourceDefinitionId
- workspaceId
properties:
sourceDefinitionId:
$ref: "#/components/schemas/SourceDefinitionId"
workspaceId:
$ref: "#/components/schemas/WorkspaceId"
redirectUrl:
description: When completing OAuth flow to gain an access token, some API sometimes requires to verify that the app re-send the redirectUrl that was used when consent was given.
type: string
queryParams:
description: The query parameters present in the redirect URL after a user granted consent e.g auth code
type: object
Expand All @@ -3029,11 +3043,15 @@ components:
type: object
required:
- destinationDefinitionId
- workspaceId
properties:
destinationDefinitionId:
$ref: "#/components/schemas/DestinationDefinitionId"
workspaceId:
$ref: "#/components/schemas/WorkspaceId"
redirectUrl:
description: When completing OAuth flow to gain an access token, some API sometimes requires to verify that the app re-send the redirectUrl that was used when consent was given.
type: string
queryParams:
description: The query parameters present in the redirect URL after a user granted consent e.g auth code
type: object
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,12 +140,6 @@ protected List<TableInfo<CommonField<StandardSQLTypeName>>> discoverInternal(Big
return result;
}

@Override
protected List<TableInfo<CommonField<StandardSQLTypeName>>> discoverInternal(BigQueryDatabase database, String schema) throws Exception {
// todo to be added
return discoverInternal(database);
}

@Override
protected Map<String, List<String>> discoverPrimaryKeys(BigQueryDatabase database, List<TableInfo<CommonField<StandardSQLTypeName>>> tableInfos) {
return Collections.emptyMap();
Expand Down
6 changes: 5 additions & 1 deletion airbyte-oauth/build.gradle
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
plugins {
id "java-library"
id 'airbyte-integration-test-java'
}

dependencies {

implementation project(':airbyte-config:models')
implementation project(':airbyte-config:persistence')
implementation project(':airbyte-json-validation')
testImplementation project(':airbyte-oauth')
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/*
* MIT License
*
* Copyright (c) 2020 Airbyte
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/

package io.airbyte.oauth;

import io.airbyte.config.DestinationOAuthParameter;
import io.airbyte.config.SourceOAuthParameter;
import java.util.Comparator;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Stream;

public class MoreOAuthParameters {

public static Optional<SourceOAuthParameter> getSourceOAuthParameter(
Stream<SourceOAuthParameter> stream,
UUID workspaceId,
UUID sourceDefinitionId) {
return stream
.filter(p -> sourceDefinitionId.equals(p.getSourceDefinitionId()))
.filter(p -> p.getWorkspaceId() == null || workspaceId.equals(p.getWorkspaceId()))
// we prefer params specific to a workspace before global ones (ie workspace is null)
.min(Comparator.comparing(SourceOAuthParameter::getWorkspaceId, Comparator.nullsLast(Comparator.naturalOrder()))
.thenComparing(SourceOAuthParameter::getOauthParameterId));
}

public static Optional<DestinationOAuthParameter> getDestinationOAuthParameter(
Stream<DestinationOAuthParameter> stream,
UUID workspaceId,
UUID destinationDefinitionId) {
return stream
.filter(p -> destinationDefinitionId.equals(p.getDestinationDefinitionId()))
.filter(p -> p.getWorkspaceId() == null || workspaceId.equals(p.getWorkspaceId()))
// we prefer params specific to a workspace before global ones (ie workspace is null)
.min(Comparator.comparing(DestinationOAuthParameter::getWorkspaceId, Comparator.nullsLast(Comparator.naturalOrder()))
.thenComparing(DestinationOAuthParameter::getOauthParameterId));
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,21 @@

package io.airbyte.oauth;

import io.airbyte.config.persistence.ConfigNotFoundException;
import java.io.IOException;
import java.util.Map;
import java.util.UUID;

public interface OAuthFlowImplementation {

String getConsentUrl();
String getSourceConsentUrl(UUID workspaceId, UUID sourceDefinitionId, String redirectUrl) throws IOException, ConfigNotFoundException;

Map<String, Object> completeOAuth(UUID workspaceId, Map<String, Object> queryParams);
String getDestinationConsentUrl(UUID workspaceId, UUID destinationDefinitionId, String redirectUrl) throws IOException, ConfigNotFoundException;

Map<String, Object> completeSourceOAuth(UUID workspaceId, UUID sourceDefinitionId, Map<String, Object> queryParams, String redirectUrl)
throws IOException, ConfigNotFoundException;

Map<String, Object> completeDestinationOAuth(UUID workspaceId, UUID destinationDefinitionId, Map<String, Object> queryParams, String redirectUrl)
throws IOException, ConfigNotFoundException;

}
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,23 @@
package io.airbyte.oauth;

import com.google.common.collect.ImmutableMap;
import io.airbyte.config.persistence.ConfigRepository;
import io.airbyte.oauth.google.GoogleAdsOauthFlow;
import io.airbyte.oauth.google.GoogleAnalyticsOauthFlow;
import java.util.Map;

public class OAuthImplementationFactory {

static final Map<String, OAuthFlowImplementation> OAUTH_FLOW_MAPPING =
ImmutableMap.<String, OAuthFlowImplementation>builder()
.build();
private final Map<String, OAuthFlowImplementation> OAUTH_FLOW_MAPPING;

public static OAuthFlowImplementation create(String imageName) {
public OAuthImplementationFactory(ConfigRepository configRepository) {
OAUTH_FLOW_MAPPING = ImmutableMap.<String, OAuthFlowImplementation>builder()
.put("airbyte/source-google-analytics-v4", new GoogleAnalyticsOauthFlow(configRepository))
.put("airbyte/source-google-ads", new GoogleAdsOauthFlow(configRepository))
.build();
}

public OAuthFlowImplementation create(String imageName) {
if (OAUTH_FLOW_MAPPING.containsKey(imageName)) {
return OAUTH_FLOW_MAPPING.get(imageName);
} else {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* MIT License
*
* Copyright (c) 2020 Airbyte
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/

package io.airbyte.oauth.google;

import com.fasterxml.jackson.databind.JsonNode;
import io.airbyte.config.persistence.ConfigRepository;

public class GoogleAdsOauthFlow extends GoogleOAuthFlow {

public GoogleAdsOauthFlow(ConfigRepository configRepository) {
super(configRepository, "https://www.googleapis.com/auth/adwords");
}

@Override
protected String getClientIdUnsafe(JsonNode config) {
// the config object containing client ID and secret is nested inside the "credentials" object
return super.getClientIdUnsafe(config.get("credentials"));
}

@Override
protected String getClientSecretUnsafe(JsonNode config) {
// the config object containing client ID and secret is nested inside the "credentials" object
return super.getClientSecretUnsafe(config.get("credentials"));
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/*
* MIT License
*
* Copyright (c) 2020 Airbyte
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/

package io.airbyte.oauth.google;

import io.airbyte.config.persistence.ConfigRepository;

public class GoogleAnalyticsOauthFlow extends GoogleOAuthFlow {

public GoogleAnalyticsOauthFlow(ConfigRepository configRepository) {
super(configRepository, "https://www.googleapis.com/auth/analytics.readonly");
}

}
Loading

0 comments on commit da34bef

Please sign in to comment.