diff --git a/.github/workflows/deploy-docs-site.yml b/.github/workflows/deploy-docs-site.yml index 497804a288ff..2313bfdb5e08 100644 --- a/.github/workflows/deploy-docs-site.yml +++ b/.github/workflows/deploy-docs-site.yml @@ -1,22 +1,51 @@ -name: Deploy docs.airbyte.com [DUMMY workflow] +name: Deploy docs.airbyte.com on: - ## XXX uncomment the following when this code gets in good shape - #push: - # branches: - # - master - # paths: - # - 'docs/**' + push: + branches: + - master + paths: + - 'docs/**' # Allows you to run this workflow manually from the Actions tab workflow_dispatch: jobs: dummy-job: - name: A placeholder job + name: Deploy Docs Assets + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - - name: A placeholder step - shell: bash - run: |- - echo "Hello from 'Deploy Docs' workflow!" + - name: Check out the repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + # Node.js is needed for Yarn + - name: Setup Yarn + uses: actions/setup-node@v2 + with: + node-version: '16.14.0' + cache: 'yarn' + cache-dependency-path: docusaurus + + - name: Run Docusaurus + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: ./tools/bin/deploy_docusaurus + + - name: Notify Slack channel on failure + uses: abinoda/slack-action@master + if: failure() + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + with: + # 'C03BEADRPNY' channel => '#oss-master-build-failure' + args: >- + {\"channel\":\"C03BEADRPNY\", \"blocks\":[ + {\"type\":\"divider\"}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"OSS Docs build fails on the latest master :bangbang: \n\n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: :octavia-shocked: \n\"}}, + {\"type\":\"divider\"}]} + diff --git a/.github/workflows/publish-connector-command.yml b/.github/workflows/publish-connector-command.yml index 4aac116c8dbf..9744873bcf38 100644 --- a/.github/workflows/publish-connector-command.yml +++ b/.github/workflows/publish-connector-command.yml @@ -67,7 +67,7 @@ jobs: # aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} # github-token: ${{ needs.find_valid_pat.outputs.pat }} # # 80 gb disk -# ec2-image-id: ami-0d648081937c75a73 +# ec2-image-id: ami-06cf12549e3d9c522 # bump-build-test-connector: # needs: start-bump-build-test-connector-runner # runs-on: ${{ needs.start-bump-build-test-connector-runner.outputs.label }} diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 02f386778070..c424fc828485 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -62,7 +62,7 @@ jobs: aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} # 80 gb disk - ec2-image-id: ami-0d648081937c75a73 + ec2-image-id: ami-06cf12549e3d9c522 integration-test: timeout-minutes: 240 needs: start-test-runner diff --git a/.github/workflows/test-performance-command.yml b/.github/workflows/test-performance-command.yml index 3378921df2bd..0e24f4444b2e 100644 --- a/.github/workflows/test-performance-command.yml +++ b/.github/workflows/test-performance-command.yml @@ -65,7 +65,7 @@ jobs: aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} # 80 gb disk - ec2-image-id: ami-0d648081937c75a73 + ec2-image-id: ami-06cf12549e3d9c522 performance-test: timeout-minutes: 240 needs: start-test-runner diff --git a/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java b/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java index bffc7c276ddb..89c08ff3a45c 100644 --- a/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java +++ b/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java @@ -16,6 +16,7 @@ import io.airbyte.api.client.generated.SourceApi; import io.airbyte.api.client.generated.SourceDefinitionApi; import io.airbyte.api.client.generated.SourceDefinitionSpecificationApi; +import io.airbyte.api.client.generated.StateApi; import io.airbyte.api.client.generated.WorkspaceApi; import io.airbyte.api.client.invoker.generated.ApiClient; @@ -46,8 +47,8 @@ public class AirbyteApiClient { private final WorkspaceApi workspaceApi; private final HealthApi healthApi; private final DbMigrationApi dbMigrationApi; - private final AttemptApi attemptApi; + private final StateApi stateApi; public AirbyteApiClient(final ApiClient apiClient) { connectionApi = new ConnectionApi(apiClient); @@ -64,6 +65,7 @@ public AirbyteApiClient(final ApiClient apiClient) { healthApi = new HealthApi(apiClient); dbMigrationApi = new DbMigrationApi(apiClient); attemptApi = new AttemptApi(apiClient); + stateApi = new StateApi(apiClient); } public ConnectionApi getConnectionApi() { @@ -122,4 +124,8 @@ public AttemptApi getAttemptApi() { return attemptApi; } + public StateApi getStateApi() { + return stateApi; + } + } diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index baa488de77df..5681bbdbcc70 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -63,6 +63,8 @@ tags: description: Export/Import Airbyte Configuration and Database resources. - name: attempt description: Interactions with attempt related resources. + - name: state + description: Interactions with state related resources. paths: /v1/workspaces/create: @@ -1389,7 +1391,7 @@ paths: /v1/state/get: post: tags: - - connection + - state summary: Fetch the current state for a connection. operationId: getState requestBody: @@ -1412,7 +1414,7 @@ paths: /v1/state/create_or_update: post: tags: - - connection + - state - internal summary: Create or update the state for a connection. operationId: createOrUpdateState @@ -1994,7 +1996,7 @@ paths: /v1/web_backend/state/get_type: post: tags: - - connection + - web_backend summary: Fetch the current state type for a connection. operationId: getStateType requestBody: @@ -2169,6 +2171,26 @@ paths: $ref: "#/components/responses/NotFoundResponse" "422": $ref: "#/components/responses/InvalidInputResponse" + /v1/jobs/get_normalization_status: + post: + tags: + - jobs + - internal + summary: Get normalization status to determine if we can bypass normalization phase + operationId: getAttemptNormalizationStatusesForJob + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/JobIdRequestBody" + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/AttemptNormalizationStatusReadList" + /v1/health: get: tags: @@ -2242,6 +2264,7 @@ paths: application/json: schema: $ref: "#/components/schemas/InternalOperationResult" + components: securitySchemes: bearerAuth: @@ -4838,6 +4861,26 @@ components: properties: succeeded: type: boolean + AttemptNormalizationStatusReadList: + type: object + properties: + attemptNormalizationStatuses: + type: array + items: + $ref: "#/components/schemas/AttemptNormalizationStatusRead" + AttemptNormalizationStatusRead: + type: object + properties: + attemptNumber: + $ref: "#/components/schemas/AttemptNumber" + hasRecordsCommitted: + type: boolean + recordsCommitted: + type: integer + format: int64 + hasNormalizationFailed: + type: boolean + InvalidInputProperty: type: object required: diff --git a/airbyte-bootloader/build.gradle b/airbyte-bootloader/build.gradle index 8d82cee2df36..132c7972d95b 100644 --- a/airbyte-bootloader/build.gradle +++ b/airbyte-bootloader/build.gradle @@ -11,7 +11,7 @@ dependencies { implementation project(':airbyte-protocol:protocol-models') implementation project(':airbyte-persistence:job-persistence') - implementation 'io.temporal:temporal-sdk:1.8.1' + implementation libs.temporal.sdk implementation libs.flyway.core testImplementation libs.platform.testcontainers.postgresql diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index edd080f4400a..75eeccc8cdad 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 0.4.2 +Low-code: Fix off by one error with the stream slicers + +## 0.4.1 +Low-code: Fix a few bugs with the stream slicers + +## 0.4.0 +Low-code: Add support for custom error messages on error response filters + +## 0.3.0 +Publish python typehints via `py.typed` file. + ## 0.2.3 - Propagate options to InterpolatedRequestInputProvider diff --git a/airbyte-cdk/python/airbyte_cdk/models/__init__.py b/airbyte-cdk/python/airbyte_cdk/models/__init__.py index f21750106913..bb6ce3e4a805 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/models/__init__.py @@ -1,2 +1,3 @@ # generated by generate-protocol-files from .airbyte_protocol import * +from .well_known_types import * diff --git a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py index 44c9c552c4a1..a4b654310d00 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py +++ b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py @@ -21,6 +21,7 @@ class Type(Enum): CONNECTION_STATUS = "CONNECTION_STATUS" CATALOG = "CATALOG" TRACE = "TRACE" + CONTROL = "CONTROL" class AirbyteRecordMessage(BaseModel): @@ -97,6 +98,17 @@ class Config: failure_type: Optional[FailureType] = Field(None, description="The type of error") +class OrchestratorType(Enum): + CONNECTOR_CONFIG = "CONNECTOR_CONFIG" + + +class AirbyteControlConnectorConfigMessage(BaseModel): + class Config: + extra = Extra.allow + + config: Dict[str, Any] = Field(..., description="the config items from this connector's spec to update") + + class Status(Enum): SUCCEEDED = "SUCCEEDED" FAILED = "FAILED" @@ -203,6 +215,18 @@ class Config: error: Optional[AirbyteErrorTraceMessage] = Field(None, description="error trace message: the error object") +class AirbyteControlMessage(BaseModel): + class Config: + extra = Extra.allow + + type: OrchestratorType = Field(..., description="the type of orchestrator message", title="orchestrator type") + emitted_at: float = Field(..., description="the time in ms that the message was emitted") + connectorConfig: Optional[AirbyteControlConnectorConfigMessage] = Field( + None, + description="connector config orchestrator message: the updated config for the platform to store for this connector", + ) + + class AirbyteStream(BaseModel): class Config: extra = Extra.allow @@ -333,6 +357,10 @@ class Config: None, description="trace message: a message to communicate information about the status and performance of a connector", ) + control: Optional[AirbyteControlMessage] = Field( + None, + description="connector config message: a message to communicate an updated configuration from a connector that should be persisted", + ) class AirbyteProtocol(BaseModel): diff --git a/airbyte-cdk/python/airbyte_cdk/models/well_known_types.py b/airbyte-cdk/python/airbyte_cdk/models/well_known_types.py new file mode 100644 index 000000000000..4c88ee200c8b --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/models/well_known_types.py @@ -0,0 +1,86 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +# generated by datamodel-codegen: +# filename: well_known_types.yaml + +from __future__ import annotations + +from enum import Enum +from typing import Any, Union + +from pydantic import BaseModel, Field, constr + + +class Model(BaseModel): + __root__: Any + + +class String(BaseModel): + __root__: str = Field(..., description="Arbitrary text") + + +class BinaryData(BaseModel): + __root__: constr(regex=r"^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$") = Field( + ..., + description="Arbitrary binary data. Represented as base64-encoded strings in the JSON transport. In the future, if we support other transports, may be encoded differently.\n", + ) + + +class Date(BaseModel): + __root__: constr(regex=r"^\d{4}-\d{2}-\d{2}( BC)?$") = Field( + ..., description="RFC 3339§5.6's full-date format, extended with BC era support" + ) + + +class TimestampWithTimezone(BaseModel): + __root__: constr(regex=r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+\-]\d{1,2}:\d{2})( BC)?$") = Field( + ..., + description='An instant in time. Frequently simply referred to as just a timestamp, or timestamptz. Uses RFC 3339§5.6\'s date-time format, requiring a "T" separator, and extended with BC era support. Note that we do _not_ accept Unix epochs here.\n', + ) + + +class TimestampWithoutTimezone(BaseModel): + __root__: constr(regex=r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?( BC)?$") = Field( + ..., + description='Also known as a localdatetime, or just datetime. Under RFC 3339§5.6, this would be represented as `full-date "T" partial-time`, extended with BC era support.\n', + ) + + +class TimeWithTimezone(BaseModel): + __root__: constr(regex=r"^\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+\-]\d{1,2}:\d{2})$") = Field(..., description="An RFC 3339§5.6 full-time") + + +class TimeWithoutTimezone(BaseModel): + __root__: constr(regex=r"^\d{2}:\d{2}:\d{2}(\.\d+)?$") = Field(..., description="An RFC 3339§5.6 partial-time") + + +class NumberEnum(Enum): + Infinity = "Infinity" + _Infinity = "-Infinity" + NaN = "NaN" + + +class Number(BaseModel): + __root__: Union[Any, NumberEnum] = Field( + ..., + description="Note the mix of regex validation for normal numbers, and enum validation for special values.", + ) + + +class IntegerEnum(Enum): + Infinity = "Infinity" + _Infinity = "-Infinity" + NaN = "NaN" + + +class Integer(BaseModel): + __root__: Union[Any, IntegerEnum] + + +class Boolean(BaseModel): + __root__: bool = Field( + ..., + description="Note the direct usage of a primitive boolean rather than string. Unlike Numbers and Integers, we don't expect unusual values here.", + ) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py index ffaed5bcb606..8b02ffceb310 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py @@ -49,10 +49,10 @@ def __post_init__(self, options: Mapping[str, Any]): def max_retries(self) -> Union[int, None]: return self.error_handlers[0].max_retries - def should_retry(self, response: requests.Response) -> ResponseStatus: + def interpret_response(self, response: requests.Response) -> ResponseStatus: should_retry = None for retrier in self.error_handlers: - should_retry = retrier.should_retry(response) + should_retry = retrier.interpret_response(response) if should_retry.action == ResponseAction.SUCCESS: return response_status.SUCCESS if should_retry == response_status.IGNORE or should_retry.action == ResponseAction.RETRY: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py index 70168560cac2..9757ec2da21a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py @@ -15,6 +15,7 @@ from airbyte_cdk.sources.declarative.requesters.error_handlers.http_response_filter import HttpResponseFilter from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus +from airbyte_cdk.sources.declarative.types import Config from dataclasses_jsonschema import JsonSchemaMixin @@ -91,6 +92,7 @@ class DefaultErrorHandler(ErrorHandler, JsonSchemaMixin): DEFAULT_BACKOFF_STRATEGY = ExponentialBackoffStrategy + config: Config options: InitVar[Mapping[str, Any]] response_filters: Optional[List[HttpResponseFilter]] = None max_retries: Optional[int] = 5 @@ -102,9 +104,11 @@ def __post_init__(self, options: Mapping[str, Any]): if not self.response_filters: self.response_filters.append( - HttpResponseFilter(ResponseAction.RETRY, http_codes=HttpResponseFilter.DEFAULT_RETRIABLE_ERRORS, options={}) + HttpResponseFilter( + ResponseAction.RETRY, http_codes=HttpResponseFilter.DEFAULT_RETRIABLE_ERRORS, config=self.config, options={} + ) ) - self.response_filters.append(HttpResponseFilter(ResponseAction.IGNORE, options={})) + self.response_filters.append(HttpResponseFilter(ResponseAction.IGNORE, config={}, options={})) if not self.backoff_strategies: self.backoff_strategies = [DefaultErrorHandler.DEFAULT_BACKOFF_STRATEGY()] @@ -122,7 +126,7 @@ def max_retries(self, value: Union[int, None]): if not isinstance(value, property): self._max_retries = value - def should_retry(self, response: requests.Response) -> ResponseStatus: + def interpret_response(self, response: requests.Response) -> ResponseStatus: request = response.request if request not in self._last_request_to_attempt_count: @@ -130,12 +134,12 @@ def should_retry(self, response: requests.Response) -> ResponseStatus: else: self._last_request_to_attempt_count[request] += 1 for response_filter in self.response_filters: - filter_action = response_filter.matches(response) - if filter_action is not None: - if filter_action == ResponseAction.RETRY: - return ResponseStatus(ResponseAction.RETRY, self._backoff_time(response, self._last_request_to_attempt_count[request])) - else: - return ResponseStatus(filter_action) + matched_status = response_filter.matches( + response=response, backoff_time=self._backoff_time(response, self._last_request_to_attempt_count[request]) + ) + if matched_status is not None: + return matched_status + if response.ok: return response_status.SUCCESS # Fail if the response matches no filters diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py index ef72fe9145ac..896fe1124eba 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py @@ -26,7 +26,7 @@ def max_retries(self) -> Union[int, None]: pass @abstractmethod - def should_retry(self, response: requests.Response) -> ResponseStatus: + def interpret_response(self, response: requests.Response) -> ResponseStatus: """ Evaluate response status describing whether a failing request should be retried or ignored. diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py index b37652383b58..e95c5770e210 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py @@ -6,8 +6,11 @@ from typing import Any, Mapping, Optional, Set, Union import requests +from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction +from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus +from airbyte_cdk.sources.declarative.types import Config from airbyte_cdk.sources.streams.http.http import HttpStream from dataclasses_jsonschema import JsonSchemaMixin @@ -22,16 +25,19 @@ class HttpResponseFilter(JsonSchemaMixin): http_codes (Set[int]): http code of matching requests error_message_contains (str): error substring of matching requests predicate (str): predicate to apply to determine if a request is matching + error_message (Union[InterpolatedString, str): error message to display if the response matches the filter """ TOO_MANY_REQUESTS_ERRORS = {429} DEFAULT_RETRIABLE_ERRORS = set([x for x in range(500, 600)]).union(TOO_MANY_REQUESTS_ERRORS) action: Union[ResponseAction, str] + config: Config options: InitVar[Mapping[str, Any]] http_codes: Set[int] = None error_message_contains: str = None predicate: Union[InterpolatedBoolean, str] = "" + error_message: Union[InterpolatedString, str] = "" def __post_init__(self, options: Mapping[str, Any]): if isinstance(self.action, str): @@ -39,8 +45,23 @@ def __post_init__(self, options: Mapping[str, Any]): self.http_codes = self.http_codes or set() if isinstance(self.predicate, str): self.predicate = InterpolatedBoolean(condition=self.predicate, options=options) + self.error_message = InterpolatedString.create(string_or_interpolated=self.error_message, options=options) - def matches(self, response: requests.Response) -> Optional[ResponseAction]: + def matches(self, response: requests.Response, backoff_time: Optional[float] = None) -> Optional[ResponseStatus]: + filter_action = self._matches_filter(response) + if filter_action is not None: + error_message = self._create_error_message(response) + if filter_action == ResponseAction.RETRY: + return ResponseStatus( + response_action=ResponseAction.RETRY, + retry_in=backoff_time, + error_message=error_message, + ) + else: + return ResponseStatus(filter_action, error_message=error_message) + return None + + def _matches_filter(self, response: requests.Response) -> Optional[ResponseAction]: """ Apply the filter on the response and return the action to execute if it matches :param response: The HTTP response to evaluate @@ -55,6 +76,14 @@ def matches(self, response: requests.Response) -> Optional[ResponseAction]: else: return None + def _create_error_message(self, response: requests.Response) -> str: + """ + Construct an error message based on the specified message template of the filter. + :param response: The HTTP response which can be used during interpolation + :return: The evaluated error message string to be emitted + """ + return self.error_message.eval(self.config, response=response.json(), headers=response.headers) + def _response_matches_predicate(self, response: requests.Response) -> bool: return self.predicate and self.predicate.eval(None, response=response.json(), headers=response.headers) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py index d089cef88f6b..2bb59c5cb7a2 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py @@ -9,13 +9,14 @@ class ResponseStatus: """ - ResponseAction amended with backoff time if a action is RETRY + ResponseAction amended with backoff time if an action is RETRY """ - def __init__(self, response_action: Union[ResponseAction, str], retry_in: Optional[float] = None): + def __init__(self, response_action: Union[ResponseAction, str], retry_in: Optional[float] = None, error_message: str = ""): """ :param response_action: response action to execute :param retry_in: backoff time (if action is RETRY) + :param error_message: the error to be displayed back to the customer """ if isinstance(response_action, str): response_action = ResponseAction[response_action] @@ -23,6 +24,7 @@ def __init__(self, response_action: Union[ResponseAction, str], retry_in: Option raise ValueError(f"Unexpected backoff time ({retry_in} for non-retryable response action {response_action}") self._retry_in = retry_in self._action = response_action + self._error_message = error_message @property def action(self): @@ -34,6 +36,11 @@ def retry_in(self) -> Optional[float]: """How long to backoff before retrying a response. None if no wait required.""" return self._retry_in + @property + def error_message(self) -> str: + """The message to be displayed when an error response is received""" + return self._error_message + @classmethod def retry(cls, retry_in: Optional[float]) -> "ResponseStatus": """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py index c1d1212d0448..5704e97c7a6c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py @@ -60,7 +60,7 @@ def __post_init__(self, options: Mapping[str, Any]): if type(self.http_method) == str: self.http_method = HttpMethod[self.http_method] self._method = self.http_method - self.error_handler = self.error_handler or DefaultErrorHandler(options=options) + self.error_handler = self.error_handler or DefaultErrorHandler(options=options, config=self.config) self._options = options # We are using an LRU cache in should_retry() method which requires all incoming arguments (including self) to be hashable. @@ -88,9 +88,9 @@ def get_method(self): # use a tiny cache to limit the memory footprint. It doesn't have to be large because we mostly # only care about the status of the last response received @lru_cache(maxsize=10) - def should_retry(self, response: requests.Response) -> ResponseStatus: + def interpret_response_status(self, response: requests.Response) -> ResponseStatus: # Cache the result because the HttpStream first checks if we should retry before looking at the backoff time - return self.error_handler.should_retry(response) + return self.error_handler.interpret_response(response) def get_request_params( self, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/requester.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/requester.py index de56a6aef8f2..084b28be95de 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/requester.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/requester.py @@ -70,9 +70,9 @@ def get_request_params( """ @abstractmethod - def should_retry(self, response: requests.Response) -> ResponseStatus: + def interpret_response_status(self, response: requests.Response) -> ResponseStatus: """ - Specifies conditions for backoff based on the response from the server. + Specifies conditions for backoff, error handling and reporting based on the response from the server. By default, back off on the following HTTP response statuses: - 429 (Too Many Requests) indicating rate limiting diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index 2eea3237daf2..27fcd98655ee 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -95,7 +95,7 @@ def should_retry(self, response: requests.Response) -> bool: Unexpected but transient exceptions (connection timeout, DNS resolution failed, etc..) are retried by default. """ - return self.requester.should_retry(response).action == ResponseAction.RETRY + return self.requester.interpret_response_status(response).action == ResponseAction.RETRY def backoff_time(self, response: requests.Response) -> Optional[float]: """ @@ -107,12 +107,21 @@ def backoff_time(self, response: requests.Response) -> Optional[float]: :return how long to backoff in seconds. The return value may be a floating point number for subsecond precision. Returning None defers backoff to the default backoff behavior (e.g using an exponential algorithm). """ - should_retry = self.requester.should_retry(response) + should_retry = self.requester.interpret_response_status(response) if should_retry.action != ResponseAction.RETRY: raise ValueError(f"backoff_time can only be applied on retriable response action. Got {should_retry.action}") assert should_retry.action == ResponseAction.RETRY return should_retry.retry_in + def error_message(self, response: requests.Response) -> str: + """ + Constructs an error message which can incorporate the HTTP response received from the partner API. + + :param response: The incoming HTTP response from the partner API + :return The error message string to be emitted + """ + return self.requester.interpret_response_status(response).error_message + def _get_request_options( self, stream_slice: Optional[StreamSlice], @@ -299,9 +308,10 @@ def parse_response( # if fail -> raise exception # if ignore -> ignore response and return no records # else -> delegate to record selector - response_status = self.requester.should_retry(response) + response_status = self.requester.interpret_response_status(response) if response_status.action == ResponseAction.FAIL: - raise ReadException(f"Request {response.request} failed with response {response}") + error_message = response_status.error_message or f"Request {response.request} failed with response {response}" + raise ReadException(error_message) elif response_status.action == ResponseAction.IGNORE: self.logger.info(f"Ignoring response for failed request with error message {HttpStream.parse_response_error_message(response)}") return [] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py index de6c745ef352..d85948ce34bf 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py @@ -110,4 +110,4 @@ def get_stream_state(self) -> Mapping[str, Any]: def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> Iterable[Mapping[str, Any]]: sub_slices = (s.stream_slices(sync_mode, stream_state) for s in self.stream_slicers) - return (ChainMap(*a) for a in itertools.product(*sub_slices)) + return (dict(ChainMap(*a)) for a in itertools.product(*sub_slices)) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/list_stream_slicer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/list_stream_slicer.py index ac83d1a967cf..8379bd7fbc03 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/list_stream_slicer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/list_stream_slicer.py @@ -43,9 +43,12 @@ def __post_init__(self, options: Mapping[str, Any]): self._cursor = None def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): + # This method is called after the records are processed. slice_value = stream_slice.get(self.cursor_field.eval(self.config)) if slice_value and slice_value in self.slice_values: self._cursor = slice_value + else: + raise ValueError(f"Unexpected stream slice: {slice_value}") def get_stream_state(self) -> StreamState: return {self.cursor_field.eval(self.config): self._cursor} if self._cursor else {} @@ -56,7 +59,8 @@ def get_request_params( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: - return self._get_request_option(RequestOptionType.request_parameter) + # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response + return self._get_request_option(RequestOptionType.request_parameter, stream_slice) def get_request_headers( self, @@ -64,7 +68,8 @@ def get_request_headers( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: - return self._get_request_option(RequestOptionType.header) + # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response + return self._get_request_option(RequestOptionType.header, stream_slice) def get_request_body_data( self, @@ -72,7 +77,8 @@ def get_request_body_data( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: - return self._get_request_option(RequestOptionType.body_data) + # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response + return self._get_request_option(RequestOptionType.body_data, stream_slice) def get_request_body_json( self, @@ -80,13 +86,18 @@ def get_request_body_json( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: - return self._get_request_option(RequestOptionType.body_json) + # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response + return self._get_request_option(RequestOptionType.body_json, stream_slice) def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> Iterable[Mapping[str, Any]]: return [{self.cursor_field.eval(self.config): slice_value} for slice_value in self.slice_values] - def _get_request_option(self, request_option_type: RequestOptionType): - if self.request_option and self.request_option.inject_into == request_option_type: - return {self.request_option.field_name: self._cursor} + def _get_request_option(self, request_option_type: RequestOptionType, stream_slice: StreamSlice): + if self.request_option and self.request_option.inject_into == request_option_type and stream_slice: + slice_value = stream_slice.get(self.cursor_field.eval(self.config)) + if slice_value: + return {self.request_option.field_name: slice_value} + else: + return {} else: return {} diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/substream_slicer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/substream_slicer.py index 476f524952fd..dbe46d924269 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/substream_slicer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/substream_slicer.py @@ -51,6 +51,7 @@ def __post_init__(self, options: Mapping[str, Any]): self._options = options def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): + # This method is called after the records are processed. cursor = {} for parent_stream_config in self.parent_stream_configs: slice_value = stream_slice.get(parent_stream_config.stream_slice_field) @@ -64,7 +65,8 @@ def get_request_params( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: - return self._get_request_option(RequestOptionType.request_parameter) + # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response + return self._get_request_option(RequestOptionType.request_parameter, stream_slice) def get_request_headers( self, @@ -72,7 +74,8 @@ def get_request_headers( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: - return self._get_request_option(RequestOptionType.header) + # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response + return self._get_request_option(RequestOptionType.header, stream_slice) def get_request_body_data( self, @@ -80,7 +83,8 @@ def get_request_body_data( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: - return self._get_request_option(RequestOptionType.body_data) + # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response + return self._get_request_option(RequestOptionType.body_data, stream_slice) def get_request_body_json( self, @@ -88,16 +92,18 @@ def get_request_body_json( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Optional[Mapping]: - return self._get_request_option(RequestOptionType.body_json) + # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response + return self._get_request_option(RequestOptionType.body_json, stream_slice) - def _get_request_option(self, option_type: RequestOptionType): + def _get_request_option(self, option_type: RequestOptionType, stream_slice: StreamSlice): params = {} - for parent_config in self.parent_stream_configs: - if parent_config.request_option and parent_config.request_option.inject_into == option_type: - key = parent_config.stream_slice_field - value = self._cursor.get(key) - if value: - params.update({key: value}) + if stream_slice: + for parent_config in self.parent_stream_configs: + if parent_config.request_option and parent_config.request_option.inject_into == option_type: + key = parent_config.stream_slice_field + value = stream_slice.get(key) + if value: + params.update({key: value}) return params def get_stream_state(self) -> StreamState: @@ -137,5 +143,4 @@ def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState) -> Itera yield {stream_state_field: stream_state_value, "parent_slice": parent_slice} # If the parent slice contains no records, if empty_parent_slice: - stream_state_value = parent_stream_slice.get(parent_field) - yield {stream_state_field: stream_state_value, "parent_slice": parent_slice} + yield from [] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/exceptions.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/exceptions.py index a2a91da61493..e15ff9da351a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/exceptions.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/exceptions.py @@ -9,8 +9,10 @@ class BaseBackoffException(requests.exceptions.HTTPError): - def __init__(self, request: requests.PreparedRequest, response: requests.Response): - error_message = f"Request URL: {request.url}, Response Code: {response.status_code}, Response Text: {response.text}" + def __init__(self, request: requests.PreparedRequest, response: requests.Response, error_message: str = ""): + error_message = ( + error_message or f"Request URL: {request.url}, Response Code: {response.status_code}, Response Text: {response.text}" + ) super().__init__(error_message, request=request, response=response) @@ -25,14 +27,14 @@ class UserDefinedBackoffException(BaseBackoffException): An exception that exposes how long it attempted to backoff """ - def __init__(self, backoff: Union[int, float], request: requests.PreparedRequest, response: requests.Response): + def __init__(self, backoff: Union[int, float], request: requests.PreparedRequest, response: requests.Response, error_message: str = ""): """ :param backoff: how long to backoff in seconds :param request: the request that triggered this backoff exception :param response: the response that triggered the backoff exception """ self.backoff = backoff - super().__init__(request=request, response=response) + super().__init__(request=request, response=response, error_message=error_message) class DefaultBackoffException(BaseBackoffException): diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py index 4a1189a1f503..a0faa4610899 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py @@ -243,6 +243,15 @@ def backoff_time(self, response: requests.Response) -> Optional[float]: """ return None + def error_message(self, response: requests.Response) -> str: + """ + Override this method to specify a custom error message which can incorporate the HTTP response received + + :param response: The incoming HTTP response from the partner API + :return: + """ + return "" + def _create_prepared_request( self, path: str, @@ -296,10 +305,13 @@ def _send(self, request: requests.PreparedRequest, request_kwargs: Mapping[str, ) if self.should_retry(response): custom_backoff_time = self.backoff_time(response) + error_message = self.error_message(response) if custom_backoff_time: - raise UserDefinedBackoffException(backoff=custom_backoff_time, request=request, response=response) + raise UserDefinedBackoffException( + backoff=custom_backoff_time, request=request, response=response, error_message=error_message + ) else: - raise DefaultBackoffException(request=request, response=response) + raise DefaultBackoffException(request=request, response=response, error_message=error_message) elif self.raise_on_http_errors: # Raise any HTTP exceptions that happened in case there were unexpected ones try: diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 5bc6577ecf4a..adfc9b1c6d43 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.2.3", + version="0.4.2", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py index 27b47f97368c..6b401ddd5bfc 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py @@ -91,15 +91,15 @@ ) def test_composite_error_handler(test_name, first_handler_behavior, second_handler_behavior, expected_behavior): first_error_handler = MagicMock() - first_error_handler.should_retry.return_value = first_handler_behavior + first_error_handler.interpret_response.return_value = first_handler_behavior second_error_handler = MagicMock() - second_error_handler.should_retry.return_value = second_handler_behavior - second_error_handler.should_retry.return_value = second_handler_behavior + second_error_handler.interpret_response.return_value = second_handler_behavior + second_error_handler.interpret_response.return_value = second_handler_behavior retriers = [first_error_handler, second_error_handler] retrier = CompositeErrorHandler(error_handlers=retriers, options={}) response_mock = MagicMock() response_mock.ok = first_handler_behavior == response_status.SUCCESS or second_handler_behavior == response_status.SUCCESS - assert retrier.should_retry(response_mock) == expected_behavior + assert retrier.interpret_response(response_mock) == expected_behavior def test_composite_error_handler_no_handlers(): diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py index eca5e4a71bd8..6bc94aa6be75 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py @@ -59,7 +59,7 @@ "test_403_ignore_error_message", HTTPStatus.FORBIDDEN, None, - HttpResponseFilter(action=ResponseAction.IGNORE, error_message_contains="found", options={}), + HttpResponseFilter(action=ResponseAction.IGNORE, error_message_contains="found", config={}, options={}), {}, response_status.IGNORE, None, @@ -68,7 +68,7 @@ "test_403_dont_ignore_error_message", HTTPStatus.FORBIDDEN, None, - HttpResponseFilter(action=ResponseAction.IGNORE, error_message_contains="not_found", options={}), + HttpResponseFilter(action=ResponseAction.IGNORE, error_message_contains="not_found", config={}, options={}), {}, response_status.FAIL, None, @@ -78,7 +78,7 @@ "test_ignore_403", HTTPStatus.FORBIDDEN, None, - HttpResponseFilter(action=ResponseAction.IGNORE, http_codes={HTTPStatus.FORBIDDEN}, options={}), + HttpResponseFilter(action=ResponseAction.IGNORE, http_codes={HTTPStatus.FORBIDDEN}, config={}, options={}), {}, response_status.IGNORE, None, @@ -86,7 +86,7 @@ ( "test_403_with_predicate", HTTPStatus.FORBIDDEN, - HttpResponseFilter(action=ResponseAction.RETRY, predicate="{{ 'code' in response }}", options={}), + HttpResponseFilter(action=ResponseAction.RETRY, predicate="{{ 'code' in response }}", config={}, options={}), None, {}, ResponseStatus.retry(10), @@ -95,7 +95,7 @@ ( "test_403_with_predicate", HTTPStatus.FORBIDDEN, - HttpResponseFilter(action=ResponseAction.RETRY, predicate="{{ 'some_absent_field' in response }}", options={}), + HttpResponseFilter(action=ResponseAction.RETRY, predicate="{{ 'some_absent_field' in response }}", config={}, options={}), None, {}, response_status.FAIL, @@ -104,7 +104,7 @@ ( "test_200_fail_with_predicate", HTTPStatus.OK, - HttpResponseFilter(action=ResponseAction.FAIL, error_message_contains="found", options={}), + HttpResponseFilter(action=ResponseAction.FAIL, error_message_contains="found", config={}, options={}), None, {}, response_status.FAIL, @@ -113,7 +113,7 @@ ( "test_retry_403", HTTPStatus.FORBIDDEN, - HttpResponseFilter(action=ResponseAction.RETRY, http_codes={HTTPStatus.FORBIDDEN}, options={}), + HttpResponseFilter(action=ResponseAction.RETRY, http_codes={HTTPStatus.FORBIDDEN}, config={}, options={}), None, {}, ResponseStatus.retry(10), @@ -122,7 +122,7 @@ ( "test_200_fail_with_predicate_from_header", HTTPStatus.OK, - HttpResponseFilter(action=ResponseAction.FAIL, predicate="{{ headers['fail'] }}", options={}), + HttpResponseFilter(action=ResponseAction.FAIL, predicate="{{ headers['fail'] }}", config={}, options={}), None, {"fail": True}, response_status.FAIL, @@ -136,8 +136,8 @@ def test_default_error_handler( response_mock = create_response(http_code, headers=response_headers, json_body={"code": "1000", "error": "found"}) response_mock.ok = http_code < 400 response_filters = [f for f in [retry_response_filter, ignore_response_filter] if f] - error_handler = DefaultErrorHandler(response_filters=response_filters, backoff_strategies=backoff_strategy, options={}) - actual_should_retry = error_handler.should_retry(response_mock) + error_handler = DefaultErrorHandler(response_filters=response_filters, backoff_strategies=backoff_strategy, config={}, options={}) + actual_should_retry = error_handler.interpret_response(response_mock) assert actual_should_retry == should_retry if should_retry.action == ResponseAction.RETRY: assert actual_should_retry.retry_in == should_retry.retry_in @@ -146,19 +146,19 @@ def test_default_error_handler( def test_default_error_handler_attempt_count_increases(): status_code = 500 response_mock = create_response(status_code) - error_handler = DefaultErrorHandler(options={}) - actual_should_retry = error_handler.should_retry(response_mock) + error_handler = DefaultErrorHandler(config={}, options={}) + actual_should_retry = error_handler.interpret_response(response_mock) assert actual_should_retry == ResponseStatus.retry(10) assert actual_should_retry.retry_in == 10 # This is the same request, so the count should increase - actual_should_retry = error_handler.should_retry(response_mock) + actual_should_retry = error_handler.interpret_response(response_mock) assert actual_should_retry == ResponseStatus.retry(20) assert actual_should_retry.retry_in == 20 # This is a different request, so the count should not increase another_identical_request = create_response(status_code) - actual_should_retry = error_handler.should_retry(another_identical_request) + actual_should_retry = error_handler.interpret_response(another_identical_request) assert actual_should_retry == ResponseStatus.retry(10) assert actual_should_retry.retry_in == 10 diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py new file mode 100644 index 000000000000..b44a479c7d7b --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py @@ -0,0 +1,117 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +import pytest +from airbyte_cdk.sources.declarative.requesters.error_handlers import HttpResponseFilter +from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction +from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus + + +@pytest.mark.parametrize( + "action, http_codes, predicate, error_contains, back_off, error_message, response, expected_response_status", + [ + pytest.param( + ResponseAction.FAIL, + {501, 503}, + "", + "", + None, + "custom error message", + {"status_code": 503}, + ResponseStatus(response_action=ResponseAction.FAIL, error_message="custom error message"), + id="test_http_code_matches", + ), + pytest.param( + ResponseAction.IGNORE, + {403}, + "", + "", + None, + "", + {"status_code": 403}, + ResponseStatus(response_action=ResponseAction.IGNORE), + id="test_http_code_matches_ignore_action", + ), + pytest.param( + ResponseAction.RETRY, + {429}, + "", + "", + 30, + "", + {"status_code": 429}, + ResponseStatus(response_action=ResponseAction.RETRY, retry_in=30), + id="test_http_code_matches_retry_action", + ), + pytest.param( + ResponseAction.FAIL, + {}, + '{{ response.the_body == "do_i_match" }}', + "", + None, + "error message was: {{ response.failure }}", + {"status_code": 404, "json": {"the_body": "do_i_match", "failure": "i failed you"}}, + ResponseStatus(response_action=ResponseAction.FAIL, error_message="error message was: i failed you"), + id="test_predicate_matches_json", + ), + pytest.param( + ResponseAction.FAIL, + {}, + '{{ headers.the_key == "header_match" }}', + "", + None, + "error from header: {{ headers.warning }}", + {"status_code": 404, "headers": {"the_key": "header_match", "warning": "this failed"}}, + ResponseStatus(response_action=ResponseAction.FAIL, error_message="error from header: this failed"), + id="test_predicate_matches_headers", + ), + pytest.param( + ResponseAction.FAIL, + {}, + None, + "DENIED", + None, + "", + {"status_code": 403, "json": {"error": "REQUEST_DENIED"}}, + ResponseStatus(response_action=ResponseAction.FAIL), + id="test_predicate_matches_headers", + ), + pytest.param( + ResponseAction.FAIL, + {400, 404}, + '{{ headers.error == "invalid_input" or response.reason == "bad request"}}', + "", + None, + "", + {"status_code": 403, "headers": {"error", "authentication_error"}, "json": {"reason": "permission denied"}}, + None, + id="test_response_does_not_match_filter", + ), + ], +) +def test_matches(action, http_codes, predicate, error_contains, back_off, error_message, response, expected_response_status): + mock_response = MagicMock() + mock_response.status_code = response.get("status_code") + mock_response.headers = response.get("headers") + mock_response.json.return_value = response.get("json") + + response_filter = HttpResponseFilter( + action=action, + config={}, + options={}, + http_codes=http_codes, + predicate=predicate, + error_message_contains=error_contains, + error_message=error_message, + ) + + actual_response_status = response_filter.matches(mock_response, backoff_time=back_off or 10) + if expected_response_status: + assert actual_response_status.action == expected_response_status.action + assert actual_response_status.retry_in == expected_response_status.retry_in + assert actual_response_status.error_message == expected_response_status.error_message + else: + assert actual_response_status is None diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py index 8e51abb1a7be..6e9e6b8d4c95 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py @@ -8,22 +8,34 @@ @pytest.mark.parametrize( - "test_name, response_action, retry_in, expected_action, expected_backoff", + "test_name, response_action, retry_in, error_message, expected_action, expected_backoff, expected_message", [ - ("test_fail_with_backoff", ResponseAction.FAIL, 10, None, None), - ("test_success_no_backoff", ResponseAction.FAIL, None, ResponseAction.FAIL, None), - ("test_ignore_with_backoff", ResponseAction.IGNORE, 10, None, None), - ("test_success_no_backoff", ResponseAction.IGNORE, None, ResponseAction.IGNORE, None), - ("test_success_with_backoff", ResponseAction.SUCCESS, 10, None, None), - ("test_success_no_backoff", ResponseAction.SUCCESS, None, ResponseAction.SUCCESS, None), - ("test_retry_with_backoff", ResponseAction.RETRY, 10, ResponseAction.RETRY, 10), - ("test_retry_no_backoff", ResponseAction.RETRY, None, ResponseAction.RETRY, None), + ("test_fail_with_backoff", ResponseAction.FAIL, 10, "", None, None, ""), + ( + "test_success_no_backoff_error_message", + ResponseAction.FAIL, + None, + "custom error message", + ResponseAction.FAIL, + None, + "custom error message", + ), + ("test_ignore_with_backoff", ResponseAction.IGNORE, 10, "", None, None, ""), + ("test_success_no_backoff", ResponseAction.IGNORE, None, "", ResponseAction.IGNORE, None, ""), + ("test_success_with_backoff", ResponseAction.SUCCESS, 10, "", None, None, ""), + ("test_success_no_backoff", ResponseAction.SUCCESS, None, "", ResponseAction.SUCCESS, None, ""), + ("test_retry_with_backoff", ResponseAction.RETRY, 10, "", ResponseAction.RETRY, 10, ""), + ("test_retry_no_backoff", ResponseAction.RETRY, None, "", ResponseAction.RETRY, None, ""), ], ) -def test_response_status(test_name, response_action, retry_in, expected_action, expected_backoff): - if expected_action or expected_backoff: - response_status = ResponseStatus(response_action, retry_in) - assert response_status.action == expected_action and response_status.retry_in == expected_backoff +def test_response_status(test_name, response_action, retry_in, error_message, expected_action, expected_backoff, expected_message): + if expected_action or expected_backoff or expected_message: + response_status = ResponseStatus(response_action, retry_in, error_message) + assert ( + response_status.action == expected_action + and response_status.retry_in == expected_backoff + and response_status.error_message == expected_message + ) else: try: ResponseStatus(response_action, retry_in) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py index 789190dcf9d9..529b01db833a 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py @@ -29,10 +29,11 @@ def test_http_requester(): error_handler = MagicMock() max_retries = 10 - should_retry = True backoff_time = 1000 + response_status = MagicMock() + response_status.retry_in.return_value = 10 error_handler.max_retries = max_retries - error_handler.should_retry.return_value = should_retry + error_handler.interpret_response.return_value = response_status error_handler.backoff_time.return_value = backoff_time config = {"url": "https://airbyte.io"} @@ -59,7 +60,7 @@ def test_http_requester(): assert requester.get_request_params(stream_state={}, stream_slice=None, next_page_token=None) == request_params assert requester.get_request_body_data(stream_state={}, stream_slice=None, next_page_token=None) == request_body_data assert requester.get_request_body_json(stream_state={}, stream_slice=None, next_page_token=None) == request_body_json - assert requester.should_retry(requests.Response()) == should_retry + assert requester.interpret_response_status(requests.Response()) == response_status assert {} == requester.request_kwargs(stream_state={}, stream_slice=None, next_page_token=None) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index c8ca6d730719..ba801f8bcdd5 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -54,7 +54,7 @@ def test_simple_retriever_full(mock_http_stream): requester.get_method.return_value = http_method backoff_time = 60 should_retry = ResponseStatus.retry(backoff_time) - requester.should_retry.return_value = should_retry + requester.interpret_response_status.return_value = should_retry request_body_json = {"body": "json"} requester.request_body_json.return_value = request_body_json @@ -117,21 +117,34 @@ def test_simple_retriever_full(mock_http_stream): def test_should_retry(test_name, requester_response, expected_should_retry, expected_backoff_time): requester = MagicMock(use_cache=False) retriever = SimpleRetriever(name="stream_name", primary_key=primary_key, requester=requester, record_selector=MagicMock(), options={}) - requester.should_retry.return_value = requester_response + requester.interpret_response_status.return_value = requester_response assert retriever.should_retry(requests.Response()) == expected_should_retry if requester_response.action == ResponseAction.RETRY: assert retriever.backoff_time(requests.Response()) == expected_backoff_time @pytest.mark.parametrize( - "test_name, status_code, response_status, len_expected_records", + "test_name, status_code, response_status, len_expected_records, expected_error", [ - ("test_parse_response_fails_if_should_retry_is_fail", 404, response_status.FAIL, None), - ("test_parse_response_succeeds_if_should_retry_is_ok", 200, response_status.SUCCESS, 1), - ("test_parse_response_succeeds_if_should_retry_is_ignore", 404, response_status.IGNORE, 0), + ( + "test_parse_response_fails_if_should_retry_is_fail", + 404, + response_status.FAIL, + None, + ReadException("Request None failed with response "), + ), + ("test_parse_response_succeeds_if_should_retry_is_ok", 200, response_status.SUCCESS, 1, None), + ("test_parse_response_succeeds_if_should_retry_is_ignore", 404, response_status.IGNORE, 0, None), + ( + "test_parse_response_fails_with_custom_error_message", + 404, + ResponseStatus(response_action=ResponseAction.FAIL, error_message="Custom error message override"), + None, + ReadException("Custom error message override"), + ), ], ) -def test_parse_response(test_name, status_code, response_status, len_expected_records): +def test_parse_response(test_name, status_code, response_status, len_expected_records, expected_error): requester = MagicMock(use_cache=False) record_selector = MagicMock() record_selector.select_records.return_value = [{"id": 100}] @@ -140,13 +153,13 @@ def test_parse_response(test_name, status_code, response_status, len_expected_re ) response = requests.Response() response.status_code = status_code - requester.should_retry.return_value = response_status + requester.interpret_response_status.return_value = response_status if len_expected_records is None: try: retriever.parse_response(response, stream_state={}) assert False - except ReadException: - pass + except ReadException as actual_exception: + assert type(expected_error) is type(actual_exception) and expected_error.args == actual_exception.args else: records = retriever.parse_response(response, stream_state={}) assert len(records) == len_expected_records @@ -170,7 +183,7 @@ def test_backoff_time(test_name, response_action, retry_in, expected_backoff_tim name="stream_name", primary_key=primary_key, requester=requester, record_selector=record_selector, options={} ) if expected_backoff_time: - requester.should_retry.return_value = ResponseStatus(response_action, retry_in) + requester.interpret_response_status.return_value = ResponseStatus(response_action, retry_in) actual_backoff_time = retriever.backoff_time(response) assert expected_backoff_time == actual_backoff_time else: diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py index 3ed21485c3c0..eced8061f340 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py @@ -95,9 +95,14 @@ def test_update_cursor(test_name, stream_slice, expected_state): ), ] slicer = CartesianProductStreamSlicer(stream_slicers=stream_slicers, options={}) - slicer.update_cursor(stream_slice, None) - updated_state = slicer.get_stream_state() - assert expected_state == updated_state + + if expected_state: + slicer.update_cursor(stream_slice, None) + updated_state = slicer.get_stream_state() + assert expected_state == updated_state + else: + with pytest.raises(ValueError): + slicer.update_cursor(stream_slice, None) @pytest.mark.parametrize( @@ -169,9 +174,37 @@ def test_request_option( ], options={}, ) - slicer.update_cursor({"owner_resource": "customer", "repository": "airbyte"}, None) + stream_slice = {"owner_resource": "customer", "repository": "airbyte"} + + assert expected_req_params == slicer.get_request_params(stream_slice=stream_slice) + assert expected_headers == slicer.get_request_headers(stream_slice=stream_slice) + assert expected_body_json == slicer.get_request_body_json(stream_slice=stream_slice) + assert expected_body_data == slicer.get_request_body_data(stream_slice=stream_slice) + - assert expected_req_params == slicer.get_request_params() - assert expected_headers == slicer.get_request_headers() - assert expected_body_json == slicer.get_request_body_json() - assert expected_body_data == slicer.get_request_body_data() +def test_request_option_before_updating_cursor(): + stream_1_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, options={}, field_name="owner") + stream_2_request_option = RequestOption(inject_into=RequestOptionType.header, options={}, field_name="repo") + slicer = CartesianProductStreamSlicer( + stream_slicers=[ + ListStreamSlicer( + slice_values=["customer", "store", "subscription"], + cursor_field="owner_resource", + config={}, + request_option=stream_1_request_option, + options={}, + ), + ListStreamSlicer( + slice_values=["airbyte", "airbyte-cloud"], + cursor_field="repository", + config={}, + request_option=stream_2_request_option, + options={}, + ), + ], + options={}, + ) + assert {} == slicer.get_request_params() + assert {} == slicer.get_request_headers() + assert {} == slicer.get_request_body_json() + assert {} == slicer.get_request_body_data() diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_list_stream_slicer.py b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_list_stream_slicer.py index 1245a7c14ba0..73a8e8c1465b 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_list_stream_slicer.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_list_stream_slicer.py @@ -51,9 +51,13 @@ def test_list_stream_slicer(test_name, slice_values, cursor_field, expected_slic ) def test_update_cursor(test_name, stream_slice, last_record, expected_state): slicer = ListStreamSlicer(slice_values=slice_values, cursor_field=cursor_field, config={}, options={}) - slicer.update_cursor(stream_slice, last_record) - updated_state = slicer.get_stream_state() - assert expected_state == updated_state + if expected_state: + slicer.update_cursor(stream_slice, last_record) + updated_state = slicer.get_stream_state() + assert expected_state == updated_state + else: + with pytest.raises(ValueError): + slicer.update_cursor(stream_slice, last_record) @pytest.mark.parametrize( @@ -111,8 +115,24 @@ def test_request_option(test_name, request_option, expected_req_params, expected slicer = ListStreamSlicer(slice_values=slice_values, cursor_field=cursor_field, config={}, request_option=request_option, options={}) stream_slice = {cursor_field: "customer"} - slicer.update_cursor(stream_slice) - assert expected_req_params == slicer.get_request_params(stream_slice) - assert expected_headers == slicer.get_request_headers() - assert expected_body_json == slicer.get_request_body_json() - assert expected_body_data == slicer.get_request_body_data() + assert expected_req_params == slicer.get_request_params(stream_slice=stream_slice) + assert expected_headers == slicer.get_request_headers(stream_slice=stream_slice) + assert expected_body_json == slicer.get_request_body_json(stream_slice=stream_slice) + assert expected_body_data == slicer.get_request_body_data(stream_slice=stream_slice) + + +def test_request_option_before_updating_cursor(): + request_option = RequestOption(inject_into=RequestOptionType.request_parameter, options={}, field_name="owner_resource") + if request_option.inject_into == RequestOptionType.path: + try: + ListStreamSlicer(slice_values=slice_values, cursor_field=cursor_field, config={}, request_option=request_option, options={}) + assert False + except ValueError: + return + slicer = ListStreamSlicer(slice_values=slice_values, cursor_field=cursor_field, config={}, request_option=request_option, options={}) + stream_slice = {cursor_field: "customer"} + + assert {} == slicer.get_request_params(stream_slice) + assert {} == slicer.get_request_headers() + assert {} == slicer.get_request_body_json() + assert {} == slicer.get_request_body_data() diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_substream_slicer.py b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_substream_slicer.py index 6c131e60c5bb..5de520b40d5e 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_substream_slicer.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_substream_slicer.py @@ -66,7 +66,7 @@ def read_records( stream=MockStream([{}], [], "first_stream"), parent_key="id", stream_slice_field="first_stream_id", options={} ) ], - [{"first_stream_id": None, "parent_slice": {}}], + [], ), ( "test_single_parent_slices_with_records", @@ -94,7 +94,6 @@ def read_records( {"parent_slice": {"slice": "first"}, "first_stream_id": 0}, {"parent_slice": {"slice": "first"}, "first_stream_id": 1}, {"parent_slice": {"slice": "second"}, "first_stream_id": 2}, - {"parent_slice": {"slice": "third"}, "first_stream_id": None}, ], ), ( @@ -117,7 +116,6 @@ def read_records( {"parent_slice": {"slice": "first"}, "first_stream_id": 0}, {"parent_slice": {"slice": "first"}, "first_stream_id": 1}, {"parent_slice": {"slice": "second"}, "first_stream_id": 2}, - {"parent_slice": {"slice": "third"}, "first_stream_id": None}, {"parent_slice": {"slice": "second_parent"}, "second_stream_id": 10}, {"parent_slice": {"slice": "second_parent"}, "second_stream_id": 20}, ], @@ -258,9 +256,9 @@ def test_request_option( ], options={}, ) - slicer.update_cursor({"first_stream_id": "1234", "second_stream_id": "4567"}, None) + stream_slice = {"first_stream_id": "1234", "second_stream_id": "4567"} - assert expected_req_params == slicer.get_request_params() - assert expected_headers == slicer.get_request_headers() - assert expected_body_json == slicer.get_request_body_json() - assert expected_body_data == slicer.get_request_body_data() + assert expected_req_params == slicer.get_request_params(stream_slice=stream_slice) + assert expected_headers == slicer.get_request_headers(stream_slice=stream_slice) + assert expected_body_json == slicer.get_request_body_json(stream_slice=stream_slice) + assert expected_body_data == slicer.get_request_body_data(stream_slice=stream_slice) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py index 684a68c01561..cbf436b15d5c 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py @@ -497,6 +497,7 @@ def test_create_composite_error_handler(): - response_filters: - http_codes: [ 403 ] action: RETRY + error_message: "Retryable error received: {{ response.message }}" """ config = parser.parse(content) @@ -508,6 +509,7 @@ def test_create_composite_error_handler(): assert isinstance(component.error_handlers[0].response_filters[0], HttpResponseFilter) assert component.error_handlers[0].response_filters[0].predicate.condition == "{{ 'code' in response }}" assert component.error_handlers[1].response_filters[0].http_codes == [403] + assert component.error_handlers[1].response_filters[0].error_message.string == "Retryable error received: {{ response.message }}" assert isinstance(component, CompositeErrorHandler) diff --git a/airbyte-commons-temporal/build.gradle b/airbyte-commons-temporal/build.gradle index 3140b4272ba5..1f01642c9ab7 100644 --- a/airbyte-commons-temporal/build.gradle +++ b/airbyte-commons-temporal/build.gradle @@ -8,9 +8,7 @@ dependencies { implementation platform(libs.micronaut.bom) implementation libs.bundles.micronaut - - implementation 'io.temporal:temporal-sdk:1.8.1' - implementation 'io.temporal:temporal-serviceclient:1.8.1' + implementation libs.bundles.temporal testAnnotationProcessor platform(libs.micronaut.bom) testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor @@ -22,7 +20,7 @@ dependencies { implementation project(':airbyte-protocol:protocol-models') implementation project(':airbyte-worker-models') - testImplementation 'io.temporal:temporal-testing:1.8.1' + testImplementation libs.temporal.testing // Needed to be able to mock final class testImplementation 'org.mockito:mockito-inline:4.7.0' } diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index ab18c9b1a6c4..db543d0b11d0 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -370,12 +370,14 @@ public TemporalResponse submitSync(final long jobId, final i final IntegrationLauncherConfig sourceLauncherConfig = new IntegrationLauncherConfig() .withJobId(String.valueOf(jobId)) .withAttemptId((long) attempt) - .withDockerImage(config.getSourceDockerImage()); + .withDockerImage(config.getSourceDockerImage()) + .withProtocolVersion(config.getSourceProtocolVersion()); final IntegrationLauncherConfig destinationLauncherConfig = new IntegrationLauncherConfig() .withJobId(String.valueOf(jobId)) .withAttemptId((long) attempt) - .withDockerImage(config.getDestinationDockerImage()); + .withDockerImage(config.getDestinationDockerImage()) + .withProtocolVersion(config.getDestinationProtocolVersion()); final StandardSyncInput input = new StandardSyncInput() .withNamespaceDefinition(config.getNamespaceDefinition()) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/ConnectionManagerWorkflow.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/ConnectionManagerWorkflow.java index dff392109b07..7dc3a33acda5 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/ConnectionManagerWorkflow.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/ConnectionManagerWorkflow.java @@ -62,13 +62,6 @@ public interface ConnectionManagerWorkflow { @SignalMethod void resetConnectionAndSkipNextScheduling(); - /** - * If an activity fails the workflow will be stuck. This signal activity can be used to retry the - * activity. - */ - @SignalMethod - void retryFailedActivity(); - /** * Return the current state of the workflow. */ diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/state/WorkflowState.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/state/WorkflowState.java index e182d1cfd786..5f037ef1ca62 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/state/WorkflowState.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/state/WorkflowState.java @@ -32,7 +32,6 @@ public WorkflowState(final UUID id, final WorkflowStateChangedListener stateChan private final boolean resetConnection = false; @Deprecated private final boolean continueAsReset = false; - private boolean retryFailedActivity = false; private boolean quarantined = false; private boolean success = true; private boolean cancelledForReset = false; @@ -89,14 +88,6 @@ public void setFailed(final boolean failed) { this.failed = failed; } - public void setRetryFailedActivity(final boolean retryFailedActivity) { - final ChangedStateEvent event = new ChangedStateEvent( - StateField.RETRY_FAILED_ACTIVITY, - retryFailedActivity); - stateChangedListener.addEvent(id, event); - this.retryFailedActivity = retryFailedActivity; - } - public void setQuarantined(final boolean quarantined) { final ChangedStateEvent event = new ChangedStateEvent( StateField.QUARANTINED, @@ -146,7 +137,6 @@ public void reset() { this.setUpdated(false); this.setCancelled(false); this.setFailed(false); - this.setRetryFailedActivity(false); this.setSuccess(false); this.setQuarantined(false); this.setDoneWaiting(false); diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/state/listener/WorkflowStateChangedListener.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/state/listener/WorkflowStateChangedListener.java index cb302b422c26..881c73bb8203 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/state/listener/WorkflowStateChangedListener.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/state/listener/WorkflowStateChangedListener.java @@ -31,7 +31,6 @@ enum StateField { FAILED, RESET, CONTINUE_AS_RESET, - RETRY_FAILED_ACTIVITY, QUARANTINED, SUCCESS, CANCELLED_FOR_RESET, diff --git a/airbyte-commons-worker/build.gradle b/airbyte-commons-worker/build.gradle index 8f93185c9410..8051e8038d0d 100644 --- a/airbyte-commons-worker/build.gradle +++ b/airbyte-commons-worker/build.gradle @@ -10,7 +10,10 @@ dependencies { implementation libs.bundles.micronaut implementation 'io.fabric8:kubernetes-client:5.12.2' - implementation 'io.temporal:temporal-sdk:1.8.1' + implementation libs.guava + implementation (libs.temporal.sdk) { + exclude module: 'guava' + } implementation 'org.apache.ant:ant:1.10.10' implementation 'org.apache.commons:commons-text:1.10.0' implementation libs.bundles.datadog diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/FailureHelper.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/FailureHelper.java index e2e92d939010..d205894479c6 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/FailureHelper.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/FailureHelper.java @@ -227,6 +227,12 @@ public static FailureReason failureReasonFromWorkflowAndActivity( } } + public static FailureReason platformFailure(final Throwable t, final Long jobId, final Integer attemptNumber) { + return genericFailure(t, jobId, attemptNumber) + .withFailureOrigin(FailureOrigin.AIRBYTE_PLATFORM) + .withExternalMessage("Something went wrong within the airbyte platform"); + } + private static Metadata jobAndAttemptMetadata(final Long jobId, final Integer attemptNumber) { return new Metadata() .withAdditionalProperty(JOB_ID_METADATA_KEY, jobId) diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriter.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriter.java new file mode 100644 index 000000000000..b5d689966489 --- /dev/null +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriter.java @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.internal; + +import io.airbyte.protocol.models.AirbyteMessage; +import java.io.IOException; + +public interface AirbyteMessageBufferedWriter { + + void write(AirbyteMessage message) throws IOException; + + void flush() throws IOException; + + void close() throws IOException; + +} diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriterFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriterFactory.java new file mode 100644 index 000000000000..90c709da8c33 --- /dev/null +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriterFactory.java @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.internal; + +import java.io.BufferedWriter; + +public interface AirbyteMessageBufferedWriterFactory { + + AirbyteMessageBufferedWriter createWriter(BufferedWriter bufferedWriter); + +} diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/AirbyteMessageTracker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/AirbyteMessageTracker.java index 259d8b3f192c..aa4b348887ae 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/AirbyteMessageTracker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/AirbyteMessageTracker.java @@ -4,16 +4,21 @@ package io.airbyte.workers.internal; +import static io.airbyte.metrics.lib.ApmTraceConstants.WORKER_OPERATION_NAME; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Charsets; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; +import datadog.trace.api.Trace; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.json.Jsons; import io.airbyte.config.FailureReason; import io.airbyte.config.State; +import io.airbyte.protocol.models.AirbyteControlConnectorConfigMessage; +import io.airbyte.protocol.models.AirbyteControlMessage; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteRecordMessage; import io.airbyte.protocol.models.AirbyteStateMessage; @@ -98,6 +103,7 @@ protected AirbyteMessageTracker(final StateDeltaTracker stateDeltaTracker, this.stateAggregator = stateAggregator; } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public void acceptFromSource(final AirbyteMessage message) { logMessageAsJSON("source", message); @@ -106,10 +112,12 @@ public void acceptFromSource(final AirbyteMessage message) { case TRACE -> handleEmittedTrace(message.getTrace(), ConnectorType.SOURCE); case RECORD -> handleSourceEmittedRecord(message.getRecord()); case STATE -> handleSourceEmittedState(message.getState()); + case CONTROL -> handleEmittedOrchestratorMessage(message.getControl(), ConnectorType.SOURCE); default -> log.warn("Invalid message type for message: {}", message); } } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public void acceptFromDestination(final AirbyteMessage message) { logMessageAsJSON("destination", message); @@ -117,6 +125,7 @@ public void acceptFromDestination(final AirbyteMessage message) { switch (message.getType()) { case TRACE -> handleEmittedTrace(message.getTrace(), ConnectorType.DESTINATION); case STATE -> handleDestinationEmittedState(message.getState()); + case CONTROL -> handleEmittedOrchestratorMessage(message.getControl(), ConnectorType.DESTINATION); default -> log.warn("Invalid message type for message: {}", message); } } @@ -211,6 +220,30 @@ private void handleDestinationEmittedState(final AirbyteStateMessage stateMessag } } + /** + * When a connector signals that the platform should update persist an update + */ + private void handleEmittedOrchestratorMessage(final AirbyteControlMessage controlMessage, final ConnectorType connectorType) { + switch (controlMessage.getType()) { + case CONNECTOR_CONFIG -> handleEmittedOrchestratorConnectorConfig(controlMessage.getConnectorConfig(), connectorType); + default -> log.warn("Invalid orchestrator message type for message: {}", controlMessage); + } + } + + /** + * When a connector needs to update its configuration + */ + @SuppressWarnings("PMD") // until method is implemented + private void handleEmittedOrchestratorConnectorConfig(final AirbyteControlConnectorConfigMessage configMessage, + final ConnectorType connectorType) { + // TODO: Update config here + /** + * Pseudocode: for (key in configMessage.getConfig()) { validateIsReallyConfig(key); + * persistConfigChange(connectorType, key, configMessage.getConfig().get(key)); // nuance here for + * secret storage or not. May need to be async over API for replication orchestrator } + */ + } + /** * When a connector emits a trace message, check the type and call the correct function. If it is an * error trace message, add it to the list of errorTraceMessages for the connector type diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/AirbyteProtocolPredicate.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/AirbyteProtocolPredicate.java index bd61e714b670..d07104b88994 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/AirbyteProtocolPredicate.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/AirbyteProtocolPredicate.java @@ -4,7 +4,10 @@ package io.airbyte.workers.internal; +import static io.airbyte.metrics.lib.ApmTraceConstants.WORKER_OPERATION_NAME; + import com.fasterxml.jackson.databind.JsonNode; +import datadog.trace.api.Trace; import io.airbyte.protocol.models.AirbyteProtocolSchema; import io.airbyte.validation.json.JsonSchemaValidator; import java.util.function.Predicate; @@ -23,6 +26,7 @@ public AirbyteProtocolPredicate() { schema = JsonSchemaValidator.getSchema(AirbyteProtocolSchema.PROTOCOL.getFile(), "AirbyteMessage"); } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public boolean test(final JsonNode s) { return jsonSchemaValidator.test(schema, s); diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java index b866d9dbf740..fe01eeb2f0d4 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java @@ -4,8 +4,11 @@ package io.airbyte.workers.internal; +import static io.airbyte.metrics.lib.ApmTraceConstants.WORKER_OPERATION_NAME; + import com.google.common.base.Charsets; import com.google.common.base.Preconditions; +import datadog.trace.api.Trace; import io.airbyte.commons.io.IOs; import io.airbyte.commons.io.LineGobbler; import io.airbyte.commons.json.Jsons; @@ -39,25 +42,29 @@ public class DefaultAirbyteDestination implements AirbyteDestination { private final IntegrationLauncher integrationLauncher; private final AirbyteStreamFactory streamFactory; + private final AirbyteMessageBufferedWriterFactory messageWriterFactory; private final AtomicBoolean inputHasEnded = new AtomicBoolean(false); private Process destinationProcess = null; - private BufferedWriter writer = null; + private AirbyteMessageBufferedWriter writer = null; private Iterator messageIterator = null; private Integer exitValue = null; public DefaultAirbyteDestination(final IntegrationLauncher integrationLauncher) { - this(integrationLauncher, new DefaultAirbyteStreamFactory(CONTAINER_LOG_MDC_BUILDER)); + this(integrationLauncher, new DefaultAirbyteStreamFactory(CONTAINER_LOG_MDC_BUILDER), new DefaultAirbyteMessageBufferedWriterFactory()); } public DefaultAirbyteDestination(final IntegrationLauncher integrationLauncher, - final AirbyteStreamFactory streamFactory) { + final AirbyteStreamFactory streamFactory, + final AirbyteMessageBufferedWriterFactory messageWriterFactory) { this.integrationLauncher = integrationLauncher; this.streamFactory = streamFactory; + this.messageWriterFactory = messageWriterFactory; } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public void start(final WorkerDestinationConfig destinationConfig, final Path jobRoot) throws IOException, WorkerException { Preconditions.checkState(destinationProcess == null); @@ -72,21 +79,22 @@ public void start(final WorkerDestinationConfig destinationConfig, final Path jo // stdout logs are logged elsewhere since stdout also contains data LineGobbler.gobble(destinationProcess.getErrorStream(), LOGGER::error, "airbyte-destination", CONTAINER_LOG_MDC_BUILDER); - writer = new BufferedWriter(new OutputStreamWriter(destinationProcess.getOutputStream(), Charsets.UTF_8)); + writer = messageWriterFactory.createWriter(new BufferedWriter(new OutputStreamWriter(destinationProcess.getOutputStream(), Charsets.UTF_8))); messageIterator = streamFactory.create(IOs.newBufferedReader(destinationProcess.getInputStream())) .filter(message -> message.getType() == Type.STATE || message.getType() == Type.TRACE) .iterator(); } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public void accept(final AirbyteMessage message) throws IOException { Preconditions.checkState(destinationProcess != null && !inputHasEnded.get()); - writer.write(Jsons.serialize(message)); - writer.newLine(); + writer.write(message); } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public void notifyEndOfInput() throws IOException { Preconditions.checkState(destinationProcess != null && !inputHasEnded.get()); @@ -96,6 +104,7 @@ public void notifyEndOfInput() throws IOException { inputHasEnded.set(true); } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public void close() throws Exception { if (destinationProcess == null) { @@ -116,6 +125,7 @@ public void close() throws Exception { } } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public void cancel() throws Exception { LOGGER.info("Attempting to cancel destination process..."); @@ -129,6 +139,7 @@ public void cancel() throws Exception { } } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public boolean isFinished() { Preconditions.checkState(destinationProcess != null); @@ -139,6 +150,7 @@ public boolean isFinished() { return !messageIterator.hasNext() && !destinationProcess.isAlive(); } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public int getExitValue() { Preconditions.checkState(destinationProcess != null, "Destination process is null, cannot retrieve exit value."); @@ -151,6 +163,7 @@ public int getExitValue() { return exitValue; } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public Optional attemptRead() { Preconditions.checkState(destinationProcess != null); diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriter.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriter.java new file mode 100644 index 000000000000..9343d5b0db3b --- /dev/null +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriter.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.internal; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.AirbyteMessage; +import java.io.BufferedWriter; +import java.io.IOException; + +public class DefaultAirbyteMessageBufferedWriter implements AirbyteMessageBufferedWriter { + + protected final BufferedWriter writer; + + public DefaultAirbyteMessageBufferedWriter(final BufferedWriter writer) { + this.writer = writer; + } + + @Override + public void write(final AirbyteMessage message) throws IOException { + writer.write(Jsons.serialize(message)); + writer.newLine(); + } + + @Override + public void flush() throws IOException { + writer.flush(); + } + + @Override + public void close() throws IOException { + writer.close(); + } + +} diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriterFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriterFactory.java new file mode 100644 index 000000000000..7c544a084cde --- /dev/null +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriterFactory.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.internal; + +import java.io.BufferedWriter; + +public class DefaultAirbyteMessageBufferedWriterFactory implements AirbyteMessageBufferedWriterFactory { + + @Override + public AirbyteMessageBufferedWriter createWriter(BufferedWriter writer) { + return new DefaultAirbyteMessageBufferedWriter(writer); + } + +} diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java index 8a94d3a05599..9da89aa0b75f 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java @@ -4,8 +4,11 @@ package io.airbyte.workers.internal; +import static io.airbyte.metrics.lib.ApmTraceConstants.WORKER_OPERATION_NAME; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import datadog.trace.api.Trace; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.io.IOs; import io.airbyte.commons.io.LineGobbler; @@ -50,8 +53,11 @@ public class DefaultAirbyteSource implements AirbyteSource { private final boolean logConnectorMessages = new EnvVariableFeatureFlags().logConnectorMessages(); public DefaultAirbyteSource(final IntegrationLauncher integrationLauncher) { - this(integrationLauncher, new DefaultAirbyteStreamFactory(CONTAINER_LOG_MDC_BUILDER), - new HeartbeatMonitor(HEARTBEAT_FRESH_DURATION)); + this(integrationLauncher, new DefaultAirbyteStreamFactory(CONTAINER_LOG_MDC_BUILDER)); + } + + public DefaultAirbyteSource(final IntegrationLauncher integrationLauncher, final AirbyteStreamFactory streamFactory) { + this(integrationLauncher, streamFactory, new HeartbeatMonitor(HEARTBEAT_FRESH_DURATION)); } @VisibleForTesting @@ -63,6 +69,7 @@ public DefaultAirbyteSource(final IntegrationLauncher integrationLauncher) { this.heartbeatMonitor = heartbeatMonitor; } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public void start(final WorkerSourceConfig sourceConfig, final Path jobRoot) throws Exception { Preconditions.checkState(sourceProcess == null); @@ -85,6 +92,7 @@ public void start(final WorkerSourceConfig sourceConfig, final Path jobRoot) thr .iterator(); } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public boolean isFinished() { Preconditions.checkState(sourceProcess != null); @@ -96,6 +104,7 @@ public boolean isFinished() { return !messageIterator.hasNext() && !sourceProcess.isAlive(); } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public int getExitValue() throws IllegalStateException { Preconditions.checkState(sourceProcess != null, "Source process is null, cannot retrieve exit value."); @@ -108,6 +117,7 @@ public int getExitValue() throws IllegalStateException { return exitValue; } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public Optional attemptRead() { Preconditions.checkState(sourceProcess != null); @@ -115,6 +125,7 @@ public Optional attemptRead() { return Optional.ofNullable(messageIterator.hasNext() ? messageIterator.next() : null); } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public void close() throws Exception { if (sourceProcess == null) { @@ -134,6 +145,7 @@ public void close() throws Exception { } } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public void cancel() throws Exception { LOGGER.info("Attempting to cancel source process..."); diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java index c8bd56efa0ab..21ae3f239313 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java @@ -4,7 +4,10 @@ package io.airbyte.workers.internal; +import static io.airbyte.metrics.lib.ApmTraceConstants.WORKER_OPERATION_NAME; + import com.fasterxml.jackson.databind.JsonNode; +import datadog.trace.api.Trace; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.logging.MdcScope; import io.airbyte.metrics.lib.MetricClientFactory; @@ -52,6 +55,7 @@ public DefaultAirbyteStreamFactory(final MdcScope.Builder containerLogMdcBuilder this.containerLogMdcBuilder = containerLogMdcBuilder; } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public Stream create(final BufferedReader bufferedReader) { final var metricClient = MetricClientFactory.getMetricClient(); diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/EmptyAirbyteSource.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/EmptyAirbyteSource.java index 9bbeb99be12a..ad46f140cf75 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/EmptyAirbyteSource.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/EmptyAirbyteSource.java @@ -4,6 +4,9 @@ package io.airbyte.workers.internal; +import static io.airbyte.metrics.lib.ApmTraceConstants.WORKER_OPERATION_NAME; + +import datadog.trace.api.Trace; import io.airbyte.commons.json.Jsons; import io.airbyte.config.ResetSourceConfiguration; import io.airbyte.config.StateType; @@ -50,6 +53,7 @@ public EmptyAirbyteSource(final boolean useStreamCapableState) { this.useStreamCapableState = useStreamCapableState; } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public void start(final WorkerSourceConfig workerSourceConfig, final Path jobRoot) throws Exception { @@ -105,16 +109,19 @@ public void start(final WorkerSourceConfig workerSourceConfig, final Path jobRoo } // always finished. it has no data to send. + @Trace(operationName = WORKER_OPERATION_NAME) @Override public boolean isFinished() { return hasEmittedState.get(); } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public int getExitValue() { return 0; } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public Optional attemptRead() { if (!isStarted) { @@ -134,11 +141,13 @@ public Optional attemptRead() { } } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public void close() throws Exception { // no op. } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public void cancel() throws Exception { // no op. diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/StateDeltaTracker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/StateDeltaTracker.java index 480b4678f076..beab1e0b63e1 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/StateDeltaTracker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/StateDeltaTracker.java @@ -4,7 +4,10 @@ package io.airbyte.workers.internal; +import static io.airbyte.metrics.lib.ApmTraceConstants.WORKER_OPERATION_NAME; + import com.google.common.annotations.VisibleForTesting; +import datadog.trace.api.Trace; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; @@ -72,6 +75,7 @@ public StateDeltaTracker(final long memoryLimitBytes) { * @throws StateDeltaTrackerException thrown when the memory footprint of stateDeltas exceeds * available capacity. */ + @Trace(operationName = WORKER_OPERATION_NAME) public void addState(final int stateHash, final Map streamIndexToRecordCount) throws StateDeltaTrackerException { synchronized (this) { final int size = STATE_HASH_BYTES + (streamIndexToRecordCount.size() * BYTES_PER_STREAM); @@ -104,6 +108,7 @@ public void addState(final int stateHash, final Map streamIndexToRe * @throws StateDeltaTrackerException thrown when committed counts can no longer be reliably * computed. */ + @Trace(operationName = WORKER_OPERATION_NAME) public void commitStateHash(final int stateHash) throws StateDeltaTrackerException { synchronized (this) { if (capacityExceeded) { @@ -139,6 +144,7 @@ public void commitStateHash(final int stateHash) throws StateDeltaTrackerExcepti } } + @Trace(operationName = WORKER_OPERATION_NAME) public Map getStreamToCommittedRecords() { return streamToCommittedRecords; } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriter.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriter.java new file mode 100644 index 000000000000..9463b89e8ae5 --- /dev/null +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriter.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.internal; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.protocol.AirbyteMessageVersionedMigrator; +import io.airbyte.commons.protocol.serde.AirbyteMessageSerializer; +import io.airbyte.protocol.models.AirbyteMessage; +import java.io.BufferedWriter; +import java.io.IOException; + +public class VersionedAirbyteMessageBufferedWriter extends DefaultAirbyteMessageBufferedWriter { + + private final AirbyteMessageSerializer serializer; + private final AirbyteMessageVersionedMigrator migrator; + + public VersionedAirbyteMessageBufferedWriter(final BufferedWriter writer, + final AirbyteMessageSerializer serializer, + final AirbyteMessageVersionedMigrator migrator) { + super(writer); + this.serializer = serializer; + this.migrator = migrator; + } + + @Override + public void write(final AirbyteMessage message) throws IOException { + final T downgradedMessage = migrator.downgrade(convert(message)); + writer.write(serializer.serialize(downgradedMessage)); + writer.newLine(); + } + + // TODO remove this conversion once we migrated default AirbyteMessage to be from a versioned + // namespace + private io.airbyte.protocol.models.v0.AirbyteMessage convert(final AirbyteMessage message) { + return Jsons.object(Jsons.jsonNode(message), io.airbyte.protocol.models.v0.AirbyteMessage.class); + } + +} diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriterFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriterFactory.java new file mode 100644 index 000000000000..3b2a2c8f0a56 --- /dev/null +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriterFactory.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.internal; + +import io.airbyte.commons.protocol.AirbyteMessageSerDeProvider; +import io.airbyte.commons.protocol.AirbyteMessageVersionedMigratorFactory; +import io.airbyte.commons.version.Version; +import java.io.BufferedWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class VersionedAirbyteMessageBufferedWriterFactory implements AirbyteMessageBufferedWriterFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(VersionedAirbyteMessageBufferedWriterFactory.class); + + private final AirbyteMessageSerDeProvider serDeProvider; + private final AirbyteMessageVersionedMigratorFactory migratorFactory; + private final Version protocolVersion; + + public VersionedAirbyteMessageBufferedWriterFactory(final AirbyteMessageSerDeProvider serDeProvider, + final AirbyteMessageVersionedMigratorFactory migratorFactory, + final Version protocolVersion) { + this.serDeProvider = serDeProvider; + this.migratorFactory = migratorFactory; + this.protocolVersion = protocolVersion; + } + + @Override + public AirbyteMessageBufferedWriter createWriter(BufferedWriter bufferedWriter) { + final boolean needMigration = !protocolVersion.getMajorVersion().equals(migratorFactory.getMostRecentVersion().getMajorVersion()); + LOGGER.info( + "Writing messages to protocol version {}{}", + protocolVersion.serialize(), + needMigration ? ", messages will be downgraded from protocol version " + migratorFactory.getMostRecentVersion().serialize() : ""); + return new VersionedAirbyteMessageBufferedWriter<>( + bufferedWriter, + serDeProvider.getSerializer(protocolVersion).orElseThrow(), + migratorFactory.getVersionedMigrator(protocolVersion)); + } + +} diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactory.java index b49190dc8658..fe4a88d56605 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactory.java @@ -4,8 +4,11 @@ package io.airbyte.workers.internal; +import static io.airbyte.metrics.lib.ApmTraceConstants.WORKER_OPERATION_NAME; + import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Preconditions; +import datadog.trace.api.Trace; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.logging.MdcScope; import io.airbyte.commons.protocol.AirbyteMessageSerDeProvider; @@ -74,6 +77,7 @@ public VersionedAirbyteStreamFactory(final AirbyteMessageSerDeProvider serDeProv * If detectVersion is set to true, it will decide which protocol version to use from the content of * the stream rather than the one passed from the constructor. */ + @Trace(operationName = WORKER_OPERATION_NAME) @SneakyThrows @Override public Stream create(final BufferedReader bufferedReader) { @@ -88,6 +92,12 @@ public Stream create(final BufferedReader bufferedReader) { initializeForProtocolVersion(fallbackVersion); } } + + final boolean needMigration = !protocolVersion.getMajorVersion().equals(migratorFactory.getMostRecentVersion().getMajorVersion()); + logger.info( + "Reading messages from protocol version {}{}", + protocolVersion.serialize(), + needMigration ? ", messages will be upgraded to protocol version " + migratorFactory.getMostRecentVersion().serialize() : ""); return super.create(bufferedReader); } @@ -123,7 +133,7 @@ private Optional detectVersion(final BufferedReader bufferedReader) thr } bufferedReader.reset(); return Optional.empty(); - } catch (IOException e) { + } catch (final IOException e) { logger.warn( "Protocol version detection failed, it is likely than the connector sent more than {}B without an complete SPEC message." + " A SPEC message that is too long could be the root cause here.", @@ -156,7 +166,7 @@ protected Stream toAirbyteMessage(final JsonNode json) { try { final io.airbyte.protocol.models.v0.AirbyteMessage message = migrator.upgrade(deserializer.deserialize(json)); return Stream.of(convert(message)); - } catch (RuntimeException e) { + } catch (final RuntimeException e) { logger.warn("Failed to upgrade a message from version {}: {}", protocolVersion, Jsons.serialize(json), e); return Stream.empty(); } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/state_aggregator/SingleStateAggregator.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/state_aggregator/SingleStateAggregator.java index 0cfe422ea1f7..7eaca78a662a 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/state_aggregator/SingleStateAggregator.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/state_aggregator/SingleStateAggregator.java @@ -4,6 +4,9 @@ package io.airbyte.workers.internal.state_aggregator; +import static io.airbyte.metrics.lib.ApmTraceConstants.WORKER_OPERATION_NAME; + +import datadog.trace.api.Trace; import io.airbyte.commons.json.Jsons; import io.airbyte.config.State; import io.airbyte.protocol.models.AirbyteStateMessage; @@ -14,11 +17,13 @@ class SingleStateAggregator implements StateAggregator { AirbyteStateMessage state; + @Trace(operationName = WORKER_OPERATION_NAME) @Override public void ingest(final AirbyteStateMessage stateMessage) { state = stateMessage; } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public State getAggregated() { if (state.getType() == null || state.getType() == AirbyteStateType.LEGACY) { diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/state_aggregator/StreamStateAggregator.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/state_aggregator/StreamStateAggregator.java index 4d3247b2549d..dfe046eaf2c9 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/state_aggregator/StreamStateAggregator.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/state_aggregator/StreamStateAggregator.java @@ -4,6 +4,9 @@ package io.airbyte.workers.internal.state_aggregator; +import static io.airbyte.metrics.lib.ApmTraceConstants.WORKER_OPERATION_NAME; + +import datadog.trace.api.Trace; import io.airbyte.commons.json.Jsons; import io.airbyte.config.State; import io.airbyte.protocol.models.AirbyteStateMessage; @@ -15,6 +18,7 @@ class StreamStateAggregator implements StateAggregator { Map aggregatedState = new HashMap<>(); + @Trace(operationName = WORKER_OPERATION_NAME) @Override public void ingest(final AirbyteStateMessage stateMessage) { /** @@ -27,6 +31,7 @@ public void ingest(final AirbyteStateMessage stateMessage) { aggregatedState.put(stateMessage.getStream().getStreamDescriptor(), stateMessage); } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public State getAggregated() { diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java index 405293663946..7f8542f9b116 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java @@ -4,13 +4,20 @@ package io.airbyte.workers.process; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.DOCKER_IMAGE_KEY; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ID_KEY; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ROOT_KEY; +import static io.airbyte.metrics.lib.ApmTraceConstants.WORKER_OPERATION_NAME; + import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import datadog.trace.api.Trace; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.config.ResourceRequirements; import io.airbyte.config.WorkerEnvConstants; +import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.workers.exception.WorkerException; import java.nio.file.Path; import java.util.Collections; @@ -63,8 +70,10 @@ public AirbyteIntegrationLauncher(final String jobId, this.featureFlags = new EnvVariableFeatureFlags(); } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public Process spec(final Path jobRoot) throws WorkerException { + ApmTraceUtils.addTagsToTrace(Map.of(JOB_ID_KEY, jobId, JOB_ROOT_KEY, jobRoot, DOCKER_IMAGE_KEY, imageName)); return processFactory.create( SPEC_JOB, jobId, @@ -81,8 +90,10 @@ public Process spec(final Path jobRoot) throws WorkerException { "spec"); } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public Process check(final Path jobRoot, final String configFilename, final String configContents) throws WorkerException { + ApmTraceUtils.addTagsToTrace(Map.of(JOB_ID_KEY, jobId, JOB_ROOT_KEY, jobRoot, DOCKER_IMAGE_KEY, imageName)); return processFactory.create( CHECK_JOB, jobId, @@ -100,8 +111,10 @@ public Process check(final Path jobRoot, final String configFilename, final Stri CONFIG, configFilename); } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public Process discover(final Path jobRoot, final String configFilename, final String configContents) throws WorkerException { + ApmTraceUtils.addTagsToTrace(Map.of(JOB_ID_KEY, jobId, JOB_ROOT_KEY, jobRoot, DOCKER_IMAGE_KEY, imageName)); return processFactory.create( DISCOVER_JOB, jobId, @@ -119,6 +132,7 @@ public Process discover(final Path jobRoot, final String configFilename, final S CONFIG, configFilename); } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public Process read(final Path jobRoot, final String configFilename, @@ -128,6 +142,7 @@ public Process read(final Path jobRoot, final String stateFilename, final String stateContents) throws WorkerException { + ApmTraceUtils.addTagsToTrace(Map.of(JOB_ID_KEY, jobId, JOB_ROOT_KEY, jobRoot, DOCKER_IMAGE_KEY, imageName)); final List arguments = Lists.newArrayList( "read", CONFIG, configFilename, @@ -161,6 +176,7 @@ public Process read(final Path jobRoot, arguments.toArray(new String[arguments.size()])); } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public Process write(final Path jobRoot, final String configFilename, @@ -168,6 +184,7 @@ public Process write(final Path jobRoot, final String catalogFilename, final String catalogContents) throws WorkerException { + ApmTraceUtils.addTagsToTrace(Map.of(JOB_ID_KEY, jobId, JOB_ROOT_KEY, jobRoot, DOCKER_IMAGE_KEY, imageName)); final Map files = ImmutableMap.of( configFilename, configContents, catalogFilename, catalogContents); diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java index 03c9bbdefc71..c433f922ce50 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java @@ -4,12 +4,20 @@ package io.airbyte.workers.sync; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.CONNECTION_ID_KEY; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ID_KEY; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ROOT_KEY; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.PROCESS_EXIT_VALUE_KEY; +import static io.airbyte.metrics.lib.ApmTraceConstants.WORKER_OPERATION_NAME; + import com.google.common.base.Stopwatch; +import datadog.trace.api.Trace; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.lang.Exceptions; import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.temporal.sync.OrchestratorConstants; import io.airbyte.config.ResourceRequirements; +import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.workers.ContainerOrchestratorConfig; import io.airbyte.workers.Worker; @@ -91,6 +99,7 @@ public LauncherWorker(final UUID connectionId, this.temporalUtils = temporalUtils; } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public OUTPUT run(final INPUT input, final Path jobRoot) throws WorkerException { final AtomicBoolean isCanceled = new AtomicBoolean(false); @@ -132,6 +141,8 @@ public OUTPUT run(final INPUT input, final Path jobRoot) throws WorkerException podName, mainContainerInfo); + ApmTraceUtils.addTagsToTrace(Map.of(CONNECTION_ID_KEY, connectionId, JOB_ID_KEY, jobRunConfig.getJobId(), JOB_ROOT_KEY, jobRoot)); + // Use the configuration to create the process. process = new AsyncOrchestratorPodProcess( kubePodInfo, @@ -165,6 +176,7 @@ public OUTPUT run(final INPUT input, final Path jobRoot) throws WorkerException fileMap, portMap); } catch (final KubernetesClientException e) { + ApmTraceUtils.addExceptionToTrace(e); throw new WorkerException( "Failed to create pod " + podName + ", pre-existing pod exists which didn't advance out of the NOT_STARTED state.", e); } @@ -174,18 +186,24 @@ public OUTPUT run(final INPUT input, final Path jobRoot) throws WorkerException process.waitFor(); if (cancelled.get()) { - throw new CancellationException(); + final CancellationException e = new CancellationException(); + ApmTraceUtils.addExceptionToTrace(e); + throw e; } final int asyncProcessExitValue = process.exitValue(); if (asyncProcessExitValue != 0) { - throw new WorkerException("Orchestrator process exited with non-zero exit code: " + asyncProcessExitValue); + final WorkerException e = new WorkerException("Orchestrator process exited with non-zero exit code: " + asyncProcessExitValue); + ApmTraceUtils.addTagsToTrace(Map.of(PROCESS_EXIT_VALUE_KEY, asyncProcessExitValue)); + ApmTraceUtils.addExceptionToTrace(e); + throw e; } final var output = process.getOutput(); return output.map(s -> Jsons.deserialize(s, outputClass)).orElse(null); } catch (final Exception e) { + ApmTraceUtils.addExceptionToTrace(e); if (cancelled.get()) { try { log.info("Destroying process due to cancellation."); @@ -231,7 +249,9 @@ private void killRunningPodsForConnection() { if (runningPods.isEmpty()) { log.info("Successfully deleted all running pods for the connection!"); } else { - throw new RuntimeException("Unable to delete pods: " + getPodNames(runningPods).toString()); + final RuntimeException e = new RuntimeException("Unable to delete pods: " + getPodNames(runningPods).toString()); + ApmTraceUtils.addExceptionToTrace(e); + throw e; } } @@ -250,6 +270,7 @@ private List getNonTerminalPodsWithLabels() { .collect(Collectors.toList()); } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public void cancel() { cancelled.set(true); diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/DefaultAirbyteDestinationTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/DefaultAirbyteDestinationTest.java index a9464b477e2c..cf599ad972a7 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/DefaultAirbyteDestinationTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/DefaultAirbyteDestinationTest.java @@ -79,6 +79,7 @@ class DefaultAirbyteDestinationTest { private IntegrationLauncher integrationLauncher; private Process process; private AirbyteStreamFactory streamFactory; + private AirbyteMessageBufferedWriterFactory messageWriterFactory; private ByteArrayOutputStream outputStream; @BeforeEach @@ -105,6 +106,7 @@ void setup() throws IOException, WorkerException { when(process.getInputStream()).thenReturn(inputStream); streamFactory = noop -> MESSAGES.stream(); + messageWriterFactory = new DefaultAirbyteMessageBufferedWriterFactory(); } @AfterEach @@ -120,7 +122,7 @@ void tearDown() throws IOException { @SuppressWarnings("BusyWait") @Test void testSuccessfulLifecycle() throws Exception { - final AirbyteDestination destination = new DefaultAirbyteDestination(integrationLauncher, streamFactory); + final AirbyteDestination destination = new DefaultAirbyteDestination(integrationLauncher, streamFactory, messageWriterFactory); destination.start(DESTINATION_CONFIG, jobRoot); final AirbyteMessage recordMessage = AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, "blue"); @@ -159,7 +161,7 @@ void testSuccessfulLifecycle() throws Exception { @Test void testTaggedLogs() throws Exception { - final AirbyteDestination destination = new DefaultAirbyteDestination(integrationLauncher, streamFactory); + final AirbyteDestination destination = new DefaultAirbyteDestination(integrationLauncher, streamFactory, messageWriterFactory); destination.start(DESTINATION_CONFIG, jobRoot); final AirbyteMessage recordMessage = AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, "blue"); diff --git a/airbyte-config/config-models/src/main/resources/types/ActorCatalogFetchEvent.yaml b/airbyte-config/config-models/src/main/resources/types/ActorCatalogFetchEvent.yaml index bd2b5206c62b..3bb598c7bd59 100644 --- a/airbyte-config/config-models/src/main/resources/types/ActorCatalogFetchEvent.yaml +++ b/airbyte-config/config-models/src/main/resources/types/ActorCatalogFetchEvent.yaml @@ -25,3 +25,6 @@ properties: type: string connectorVersion: type: string + createdAt: + type: integer + format: int64 diff --git a/airbyte-config/config-models/src/main/resources/types/JobResetConnectionConfig.yaml b/airbyte-config/config-models/src/main/resources/types/JobResetConnectionConfig.yaml index 62e15dfe3b72..160cdd4067d7 100644 --- a/airbyte-config/config-models/src/main/resources/types/JobResetConnectionConfig.yaml +++ b/airbyte-config/config-models/src/main/resources/types/JobResetConnectionConfig.yaml @@ -30,6 +30,10 @@ properties: destinationDockerImage: description: Image name of the destination with tag. type: string + destinationProtocolVersion: + description: Airbyte Protocol Version of the destination + type: object + existingJavaType: io.airbyte.commons.version.Version operationSequence: description: Sequence of configurations of operations to apply as part of the sync type: array diff --git a/airbyte-config/config-models/src/main/resources/types/JobSyncConfig.yaml b/airbyte-config/config-models/src/main/resources/types/JobSyncConfig.yaml index 317ad0a89394..f6b9af129a08 100644 --- a/airbyte-config/config-models/src/main/resources/types/JobSyncConfig.yaml +++ b/airbyte-config/config-models/src/main/resources/types/JobSyncConfig.yaml @@ -36,9 +36,17 @@ properties: sourceDockerImage: description: Image name of the source with tag. type: string + sourceProtocolVersion: + description: Airbyte Protocol Version of the source + type: object + existingJavaType: io.airbyte.commons.version.Version destinationDockerImage: description: Image name of the destination with tag. type: string + destinationProtocolVersion: + description: Airbyte Protocol Version of the destination + type: object + existingJavaType: io.airbyte.commons.version.Version sourceResourceRequirements: type: object description: optional resource requirements to use in source container - this is used instead of `resourceRequirements` for the source container diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index da59d93e9068..2e0f1c33861a 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -650,6 +650,54 @@ public List listDestinationConnection() throws JsonValida return persistence.listConfigs(ConfigSchema.DESTINATION_CONNECTION, DestinationConnection.class); } + /** + * Returns all destinations for a workspace. Does not contain secrets. + * + * @param workspaceId - id of the workspace + * @return destinations + * @throws IOException - you never know when you IO + */ + public List listWorkspaceDestinationConnection(UUID workspaceId) throws IOException { + final Result result = database.query(ctx -> ctx.select(asterisk()) + .from(ACTOR) + .where(ACTOR.ACTOR_TYPE.eq(ActorType.destination)) + .and(ACTOR.WORKSPACE_ID.eq(workspaceId)) + .andNot(ACTOR.TOMBSTONE).fetch()); + return result.stream().map(DbConverter::buildDestinationConnection).collect(Collectors.toList()); + } + + /** + * Returns all active sources using a definition + * + * @param definitionId - id for the definition + * @return sources + * @throws IOException + */ + public List listSourcesForDefinition(UUID definitionId) throws IOException { + final Result result = database.query(ctx -> ctx.select(asterisk()) + .from(ACTOR) + .where(ACTOR.ACTOR_TYPE.eq(ActorType.source)) + .and(ACTOR.ACTOR_DEFINITION_ID.eq(definitionId)) + .andNot(ACTOR.TOMBSTONE).fetch()); + return result.stream().map(DbConverter::buildSourceConnection).collect(Collectors.toList()); + } + + /** + * Returns all active destinations using a definition + * + * @param definitionId - id for the definition + * @return destinations + * @throws IOException + */ + public List listDestinationsForDefinition(UUID definitionId) throws IOException { + final Result result = database.query(ctx -> ctx.select(asterisk()) + .from(ACTOR) + .where(ACTOR.ACTOR_TYPE.eq(ActorType.destination)) + .and(ACTOR.ACTOR_DEFINITION_ID.eq(definitionId)) + .andNot(ACTOR.TOMBSTONE).fetch()); + return result.stream().map(DbConverter::buildDestinationConnection).collect(Collectors.toList()); + } + public StandardSync getStandardSync(final UUID connectionId) throws JsonValidationException, IOException, ConfigNotFoundException { return persistence.getConfig(ConfigSchema.STANDARD_SYNC, connectionId.toString(), StandardSync.class); } @@ -997,8 +1045,8 @@ public Map getMostRecentActorCatalogFetchEventForS return database.query(ctx -> ctx.fetch( """ - select actor_catalog_id, actor_id from - (select actor_catalog_id, actor_id, rank() over (partition by actor_id order by created_at desc) as creation_order_rank + select actor_catalog_id, actor_id, created_at from + (select actor_catalog_id, actor_id, created_at, rank() over (partition by actor_id order by created_at desc) as creation_order_rank from public.actor_catalog_fetch_event ) table_with_rank where creation_order_rank = 1; diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java index 7083412db7f7..b0c941bac8cc 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java @@ -768,6 +768,9 @@ private void writeStandardWorkspace(final List configs, final .set(WORKSPACE.NOTIFICATIONS, JSONB.valueOf(Jsons.serialize(standardWorkspace.getNotifications()))) .set(WORKSPACE.FIRST_SYNC_COMPLETE, standardWorkspace.getFirstCompletedSync()) .set(WORKSPACE.FEEDBACK_COMPLETE, standardWorkspace.getFeedbackDone()) + .set(WORKSPACE.GEOGRAPHY, Enums.toEnum( + standardWorkspace.getDefaultGeography().value(), + io.airbyte.db.instance.configs.jooq.generated.enums.GeographyType.class).orElseThrow()) .set(WORKSPACE.UPDATED_AT, timestamp) .set(WORKSPACE.WEBHOOK_OPERATION_CONFIGS, standardWorkspace.getWebhookOperationConfigs() == null ? null : JSONB.valueOf(Jsons.serialize(standardWorkspace.getWebhookOperationConfigs()))) @@ -791,6 +794,9 @@ private void writeStandardWorkspace(final List configs, final .set(WORKSPACE.FEEDBACK_COMPLETE, standardWorkspace.getFeedbackDone()) .set(WORKSPACE.CREATED_AT, timestamp) .set(WORKSPACE.UPDATED_AT, timestamp) + .set(WORKSPACE.GEOGRAPHY, Enums.toEnum( + standardWorkspace.getDefaultGeography().value(), + io.airbyte.db.instance.configs.jooq.generated.enums.GeographyType.class).orElseThrow()) .set(WORKSPACE.WEBHOOK_OPERATION_CONFIGS, standardWorkspace.getWebhookOperationConfigs() == null ? null : JSONB.valueOf(Jsons.serialize(standardWorkspace.getWebhookOperationConfigs()))) .execute(); diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java index ea4548d9f78a..96a705dddb90 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java @@ -38,6 +38,8 @@ import io.airbyte.config.WorkspaceServiceAccount; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; +import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -199,7 +201,8 @@ public static ActorCatalog buildActorCatalog(final Record record) { public static ActorCatalogFetchEvent buildActorCatalogFetchEvent(final Record record) { return new ActorCatalogFetchEvent() .withActorId(record.get(ACTOR_CATALOG_FETCH_EVENT.ACTOR_ID)) - .withActorCatalogId(record.get(ACTOR_CATALOG_FETCH_EVENT.ACTOR_CATALOG_ID)); + .withActorCatalogId(record.get(ACTOR_CATALOG_FETCH_EVENT.ACTOR_CATALOG_ID)) + .withCreatedAt(record.get(ACTOR_CATALOG_FETCH_EVENT.CREATED_AT, LocalDateTime.class).toEpochSecond(ZoneOffset.UTC)); } public static WorkspaceServiceAccount buildWorkspaceServiceAccount(final Record record) { diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java index 552355066b5e..77c8b231a738 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java @@ -158,6 +158,22 @@ void testWorkspaceCountConnectionsDeprecated() throws IOException { assertEquals(1, configRepository.countConnectionsForWorkspace(workspaceId)); } + @Test + void testFetchActorsUsingDefinition() throws IOException { + UUID destinationDefinitionId = MockData.publicDestinationDefinition().getDestinationDefinitionId(); + UUID sourceDefinitionId = MockData.publicSourceDefinition().getSourceDefinitionId(); + final List destinationConnections = configRepository.listDestinationsForDefinition( + destinationDefinitionId); + final List sourceConnections = configRepository.listSourcesForDefinition( + sourceDefinitionId); + + assertThat(destinationConnections) + .containsExactlyElementsOf(MockData.destinationConnections().stream().filter(d -> d.getDestinationDefinitionId().equals( + destinationDefinitionId) && !d.getTombstone()).collect(Collectors.toList())); + assertThat(sourceConnections).containsExactlyElementsOf(MockData.sourceConnections().stream().filter(d -> d.getSourceDefinitionId().equals( + sourceDefinitionId) && !d.getTombstone()).collect(Collectors.toList())); + } + @Test void testSimpleInsertActorCatalog() throws IOException, JsonValidationException, SQLException { @@ -306,6 +322,15 @@ void testListWorkspaceSources() throws IOException { assertThat(sources).hasSameElementsAs(expectedSources); } + @Test + void testListWorkspaceDestinations() throws IOException { + UUID workspaceId = MockData.standardWorkspaces().get(0).getWorkspaceId(); + final List expectedDestinations = MockData.destinationConnections().stream() + .filter(destination -> destination.getWorkspaceId().equals(workspaceId)).collect(Collectors.toList()); + final List destinations = configRepository.listWorkspaceDestinationConnection(workspaceId); + assertThat(destinations).hasSameElementsAs(expectedDestinations); + } + @Test void testSourceDefinitionGrants() throws IOException { final UUID workspaceId = MockData.standardWorkspaces().get(0).getWorkspaceId(); diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java index 2b6050b7d56a..4397890a90cd 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java @@ -161,7 +161,7 @@ public static List standardWorkspaces() { .withNotifications(Collections.singletonList(notification)) .withFirstCompletedSync(true) .withFeedbackDone(true) - .withDefaultGeography(Geography.AUTO) + .withDefaultGeography(Geography.US) .withWebhookOperationConfigs(Jsons.jsonNode( new WebhookOperationConfigs().withWebhookConfigs(List.of(new WebhookConfig().withId(WEBHOOK_CONFIG_ID).withName("name"))))); diff --git a/airbyte-config/init/src/main/resources/icons/clockify.svg b/airbyte-config/init/src/main/resources/icons/clockify.svg new file mode 100644 index 000000000000..562fbab20a7e --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/clockify.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/airbyte-config/init/src/main/resources/icons/meilisearch.svg b/airbyte-config/init/src/main/resources/icons/meilisearch.svg index 85fecca5aab3..f43fc2e56dd7 100644 --- a/airbyte-config/init/src/main/resources/icons/meilisearch.svg +++ b/airbyte-config/init/src/main/resources/icons/meilisearch.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/airbyte-config/init/src/main/resources/icons/nasa.svg b/airbyte-config/init/src/main/resources/icons/nasa.svg new file mode 100644 index 000000000000..e776a4be47d1 --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/nasa.svg @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-config/init/src/main/resources/icons/workable.svg b/airbyte-config/init/src/main/resources/icons/workable.svg new file mode 100644 index 000000000000..10c4da2ac370 --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/workable.svg @@ -0,0 +1,23 @@ + + + + + + + + + diff --git a/airbyte-config/init/src/main/resources/icons/yugabytedb.svg b/airbyte-config/init/src/main/resources/icons/yugabytedb.svg new file mode 100644 index 000000000000..0a493b99787b --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/yugabytedb.svg @@ -0,0 +1,409 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 780c0dc5312f..e6561b8854ed 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -100,7 +100,7 @@ - destinationDefinitionId: 68f351a7-2745-4bef-ad7f-996b8e51bb8c name: ElasticSearch dockerRepository: airbyte/destination-elasticsearch - dockerImageTag: 0.1.5 + dockerImageTag: 0.1.6 documentationUrl: https://docs.airbyte.com/integrations/destinations/elasticsearch icon: elasticsearch.svg releaseStage: alpha @@ -182,14 +182,14 @@ - name: MeiliSearch destinationDefinitionId: af7c921e-5892-4ff2-b6c1-4a5ab258fb7e dockerRepository: airbyte/destination-meilisearch - dockerImageTag: 0.2.13 + dockerImageTag: 1.0.0 documentationUrl: https://docs.airbyte.com/integrations/destinations/meilisearch icon: meilisearch.svg releaseStage: alpha - name: MongoDB destinationDefinitionId: 8b746512-8c2e-6ac1-4adc-b59faafd473c dockerRepository: airbyte/destination-mongodb - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.8 documentationUrl: https://docs.airbyte.com/integrations/destinations/mongodb icon: mongodb.svg releaseStage: alpha @@ -238,7 +238,7 @@ - name: Redshift destinationDefinitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc dockerRepository: airbyte/destination-redshift - dockerImageTag: 0.3.50 + dockerImageTag: 0.3.51 documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift icon: redshift.svg resourceRequirements: @@ -257,7 +257,7 @@ - name: S3 destinationDefinitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362 dockerRepository: airbyte/destination-s3 - dockerImageTag: 0.3.16 + dockerImageTag: 0.3.17 documentationUrl: https://docs.airbyte.com/integrations/destinations/s3 icon: s3.svg resourceRequirements: @@ -329,3 +329,16 @@ documentationUrl: https://docs.airbyte.com/integrations/destinations/tidb icon: tidb.svg releaseStage: alpha +- name: Typesense + destinationDefinitionId: 36be8dc6-9851-49af-b776-9d4c30e4ab6a + dockerRepository: airbyte/destination-typesense + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/destinations/typesense + releaseStage: alpha +- name: YugabyteDB + destinationDefinitionId: 2300fdcf-a532-419f-9f24-a014336e7966 + dockerRepository: airbyte/destination-yugabytedb + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/destinations/yugabytedb + icon: yugabytedb.svg + releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 1f71e6f53c4c..12f732b856a6 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -1700,7 +1700,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-elasticsearch:0.1.5" +- dockerImage: "airbyte/destination-elasticsearch:0.1.6" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/elasticsearch" connectionSpecification: @@ -3167,16 +3167,16 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-meilisearch:0.2.13" +- dockerImage: "airbyte/destination-meilisearch:1.0.0" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/meilisearch" connectionSpecification: $schema: "http://json-schema.org/draft-07/schema#" - title: "MeiliSearch Destination Spec" + title: "Destination Meilisearch" type: "object" required: - "host" - additionalProperties: true + additionalProperties: false properties: host: title: "Host" @@ -3196,7 +3196,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-mongodb:0.1.7" +- dockerImage: "airbyte/destination-mongodb:0.1.8" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/mongodb" connectionSpecification: @@ -3328,6 +3328,107 @@ type: "string" airbyte_secret: true order: 2 + tunnel_method: + type: "object" + title: "SSH Tunnel Method" + description: "Whether to initiate an SSH tunnel before connecting to the\ + \ database, and if so, which kind of authentication to use." + oneOf: + - title: "No Tunnel" + required: + - "tunnel_method" + properties: + tunnel_method: + description: "No ssh tunnel needed to connect to database" + type: "string" + const: "NO_TUNNEL" + order: 0 + - title: "SSH Key Authentication" + required: + - "tunnel_method" + - "tunnel_host" + - "tunnel_port" + - "tunnel_user" + - "ssh_key" + properties: + tunnel_method: + description: "Connect through a jump server tunnel host using username\ + \ and ssh key" + type: "string" + const: "SSH_KEY_AUTH" + order: 0 + tunnel_host: + title: "SSH Tunnel Jump Server Host" + description: "Hostname of the jump server host that allows inbound\ + \ ssh tunnel." + type: "string" + order: 1 + tunnel_port: + title: "SSH Connection Port" + description: "Port on the proxy/jump server that accepts inbound ssh\ + \ connections." + type: "integer" + minimum: 0 + maximum: 65536 + default: 22 + examples: + - "22" + order: 2 + tunnel_user: + title: "SSH Login Username" + description: "OS-level username for logging into the jump server host." + type: "string" + order: 3 + ssh_key: + title: "SSH Private Key" + description: "OS-level user account ssh key credentials in RSA PEM\ + \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )" + type: "string" + airbyte_secret: true + multiline: true + order: 4 + - title: "Password Authentication" + required: + - "tunnel_method" + - "tunnel_host" + - "tunnel_port" + - "tunnel_user" + - "tunnel_user_password" + properties: + tunnel_method: + description: "Connect through a jump server tunnel host using username\ + \ and password authentication" + type: "string" + const: "SSH_PASSWORD_AUTH" + order: 0 + tunnel_host: + title: "SSH Tunnel Jump Server Host" + description: "Hostname of the jump server host that allows inbound\ + \ ssh tunnel." + type: "string" + order: 1 + tunnel_port: + title: "SSH Connection Port" + description: "Port on the proxy/jump server that accepts inbound ssh\ + \ connections." + type: "integer" + minimum: 0 + maximum: 65536 + default: 22 + examples: + - "22" + order: 2 + tunnel_user: + title: "SSH Login Username" + description: "OS-level username for logging into the jump server host" + type: "string" + order: 3 + tunnel_user_password: + title: "Password" + description: "OS-level password for logging into the jump server host" + type: "string" + airbyte_secret: true + order: 4 supportsIncremental: true supportsNormalization: false supportsDBT: false @@ -4400,7 +4501,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-redshift:0.3.50" +- dockerImage: "airbyte/destination-redshift:0.3.51" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/redshift" connectionSpecification: @@ -4664,7 +4765,7 @@ supported_destination_sync_modes: - "append" - "overwrite" -- dockerImage: "airbyte/destination-s3:0.3.16" +- dockerImage: "airbyte/destination-s3:0.3.17" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/s3" connectionSpecification: @@ -6044,3 +6145,117 @@ supported_destination_sync_modes: - "overwrite" - "append" +- dockerImage: "airbyte/destination-typesense:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/destinations/typesense" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Destination Typesense" + type: "object" + required: + - "api_key" + - "host" + additionalProperties: false + properties: + api_key: + title: "API Key" + type: "string" + description: "Typesense API Key" + order: 0 + host: + title: "Host" + type: "string" + description: "Hostname of the Typesense instance without protocol." + order: 1 + port: + title: "Port" + type: "string" + description: "Port of the Typesense instance. Ex: 8108, 80, 443. Default\ + \ is 443" + order: 2 + protocol: + title: "Protocol" + type: "string" + description: "Protocol of the Typesense instance. Ex: http or https. Default\ + \ is https" + order: 3 + batch_size: + title: "Batch size" + type: "string" + description: "How many documents should be imported together. Default 1000" + order: 4 + supportsIncremental: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: + - "overwrite" + - "append" +- dockerImage: "airbyte/destination-yugabytedb:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/destinations/yugabytedb" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Yugabytedb destination spec" + type: "object" + required: + - "host" + - "port" + - "username" + - "database" + - "schema" + additionalProperties: true + properties: + host: + title: "Host" + description: "The Hostname of the database." + type: "string" + order: 0 + port: + title: "Port" + description: "The Port of the database." + type: "integer" + minimum: 0 + maximum: 65536 + default: 3306 + examples: + - "3306" + order: 1 + database: + title: "Database" + description: "Name of the database." + type: "string" + order: 2 + username: + title: "Username" + description: "The Username which is used to access the database." + type: "string" + order: 3 + schema: + title: "Default Schema" + description: "The default schema tables are written to if the source does\ + \ not specify a namespace. The usual value for this field is \"public\"\ + ." + type: "string" + examples: + - "public" + default: "public" + order: 3 + password: + title: "Password" + description: "The Password associated with the username." + type: "string" + airbyte_secret: true + order: 4 + jdbc_url_params: + description: "Additional properties to pass to the JDBC URL string when\ + \ connecting to the database formatted as 'key=value' pairs separated\ + \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)." + title: "JDBC URL Params" + type: "string" + order: 5 + supportsIncremental: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: + - "overwrite" + - "append" diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 46a0fe2a1498..fa7b5504c0c2 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1,3 +1,10 @@ +- name: ActiveCampaign + sourceDefinitionId: 9f32dab3-77cb-45a1-9d33-347aa5fbe363 + dockerRepository: airbyte/source-activecampaign + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/activecampaign + sourceType: api + releaseStage: alpha - name: Adjust sourceDefinitionId: d3b7fa46-111b-419a-998a-d7f046f6d66d dockerRepository: airbyte/source-adjust @@ -9,7 +16,7 @@ - name: Airtable sourceDefinitionId: 14c6e7ea-97ed-4f5e-a7b5-25e9a80b8212 dockerRepository: airbyte/source-airtable - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.com/integrations/sources/airtable icon: airtable.svg sourceType: api @@ -17,7 +24,7 @@ - name: AlloyDB for PostgreSQL sourceDefinitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 dockerRepository: airbyte/source-alloydb - dockerImageTag: 1.0.15 + dockerImageTag: 1.0.16 documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb icon: alloydb.svg sourceType: database @@ -93,6 +100,20 @@ icon: asana.svg sourceType: api releaseStage: beta +- name: Ashby + sourceDefinitionId: 4e8c9fa0-3634-499b-b948-11581b5c3efa + dockerRepository: airbyte/source-ashby + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/ashby + sourceType: api + releaseStage: alpha +- name: Auth0 + sourceDefinitionId: 6c504e48-14aa-4221-9a72-19cf5ff1ae78 + dockerRepository: airbyte/source-auth0 + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/auth0 + sourceType: api + releaseStage: alpha - name: Azure Table Storage sourceDefinitionId: 798ae795-5189-42b6-b64e-3cb91db93338 dockerRepository: airbyte/source-azure-table @@ -197,6 +218,13 @@ icon: cockroachdb.svg sourceType: database releaseStage: alpha +- name: Coin API + sourceDefinitionId: 919984ef-53a2-479b-8ffe-9c1ddb9fc3f3 + dockerRepository: airbyte/source-coin-api + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/coin-api + sourceType: api + releaseStage: alpha - name: Commercetools sourceDefinitionId: 008b2e26-11a3-11ec-82a8-0242ac130003 dockerRepository: airbyte/source-commercetools @@ -213,6 +241,13 @@ icon: confluence.svg sourceType: api releaseStage: alpha +- name: ConvertKit + sourceDefinitionId: be9ee02f-6efe-4970-979b-95f797a37188 + dockerRepository: airbyte/source-convertkit + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/convertkit + sourceType: api + releaseStage: alpha - name: Courier sourceDefinitionId: 0541b2cd-2367-4986-b5f1-b79ff55439e4 dockerRepository: airbyte/source-courier @@ -220,6 +255,14 @@ documentationUrl: https://docs.airbyte.com/integrations/sources/courier sourceType: api releaseStage: alpha +- name: Clockify + sourceDefinitionId: e71aae8a-5143-11ed-bdc3-0242ac120002 + dockerRepository: airbyte/source-clockify + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/clockify + icon: clockify.svg + sourceType: api + releaseStage: alpha - name: Customer.io sourceDefinitionId: c47d6804-8b98-449f-970a-5ddb5cb5d7aa dockerRepository: farosai/airbyte-customer-io-source @@ -286,7 +329,7 @@ - name: Facebook Marketing sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c dockerRepository: airbyte/source-facebook-marketing - dockerImageTag: 0.2.69 + dockerImageTag: 0.2.70 documentationUrl: https://docs.airbyte.com/integrations/sources/facebook-marketing icon: facebook.svg sourceType: api @@ -317,7 +360,7 @@ - name: File sourceDefinitionId: 778daa7c-feaf-4db6-96f3-70fd645acc77 dockerRepository: airbyte/source-file - dockerImageTag: 0.2.26 + dockerImageTag: 0.2.28 documentationUrl: https://docs.airbyte.com/integrations/sources/file icon: file.svg sourceType: file @@ -424,7 +467,7 @@ - name: Google Search Console sourceDefinitionId: eb4c9e00-db83-4d63-a386-39cfa91012a8 dockerRepository: airbyte/source-google-search-console - dockerImageTag: 0.1.17 + dockerImageTag: 0.1.18 documentationUrl: https://docs.airbyte.com/integrations/sources/google-search-console icon: googlesearchconsole.svg sourceType: api @@ -437,6 +480,14 @@ icon: google-sheets.svg sourceType: file releaseStage: generally_available +- name: Google Webfonts + sourceDefinitionId: a68fbcde-b465-4ab3-b2a6-b0590a875835 + dockerRepository: airbyte/source-google-webfonts + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/google-webfonts + icon: googleworkpace.svg + sourceType: api + releaseStage: alpha - name: Google Workspace Admin Reports sourceDefinitionId: ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734 dockerRepository: airbyte/source-google-workspace-admin-reports @@ -448,7 +499,7 @@ - name: Greenhouse sourceDefinitionId: 59f1e50a-331f-4f09-b3e8-2e8d4d355f44 dockerRepository: airbyte/source-greenhouse - dockerImageTag: 0.2.11 + dockerImageTag: 0.3.0 documentationUrl: https://docs.airbyte.com/integrations/sources/greenhouse icon: greenhouse.svg sourceType: api @@ -532,7 +583,7 @@ - name: Iterable sourceDefinitionId: 2e875208-0c0b-4ee4-9e92-1cb3156ea799 dockerRepository: airbyte/source-iterable - dockerImageTag: 0.1.20 + dockerImageTag: 0.1.21 documentationUrl: https://docs.airbyte.com/integrations/sources/iterable icon: iterable.svg sourceType: api @@ -586,7 +637,7 @@ - name: Lever Hiring sourceDefinitionId: 3981c999-bd7d-4afc-849b-e53dea90c948 dockerRepository: airbyte/source-lever-hiring - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.com/integrations/sources/lever-hiring icon: leverhiring.svg sourceType: api @@ -615,6 +666,13 @@ icon: linnworks.svg sourceType: api releaseStage: alpha +- name: Lokalise + sourceDefinitionId: 45e0b135-615c-40ac-b38e-e65b0944197f + dockerRepository: airbyte/source-lokalise + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/lokalise + sourceType: api + releaseStage: alpha - name: Looker sourceDefinitionId: 00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c dockerRepository: airbyte/source-looker @@ -631,6 +689,27 @@ icon: mailchimp.svg sourceType: api releaseStage: generally_available +- name: Mailjet Mail + sourceDefinitionId: 56582331-5de2-476b-b913-5798de77bbdf + dockerRepository: airbyte/source-mailjet-mail + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/mailjet-mail + sourceType: api + releaseStage: alpha +- name: Mailjet SMS + sourceDefinitionId: 6ec2acea-7fd1-4378-b403-41a666e0c028 + dockerRepository: airbyte/source-mailjet-sms + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/mailjet-sms + sourceType: api + releaseStage: alpha +- name: MailerLite + sourceDefinitionId: dc3b9003-2432-4e93-a7f4-4620b0f14674 + dockerRepository: airbyte/source-mailerlite + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/mailerlite + sourceType: api + releaseStage: alpha - name: Mailgun sourceDefinitionId: 5b9cb09e-1003-4f9c-983d-5779d1b2cd51 dockerRepository: airbyte/source-mailgun @@ -658,7 +737,7 @@ - name: Microsoft SQL Server (MSSQL) sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 dockerRepository: airbyte/source-mssql - dockerImageTag: 0.4.23 + dockerImageTag: 0.4.24 documentationUrl: https://docs.airbyte.com/integrations/sources/mssql icon: mssql.svg sourceType: database @@ -706,17 +785,31 @@ - name: MySQL sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad dockerRepository: airbyte/source-mysql - dockerImageTag: 1.0.7 + dockerImageTag: 1.0.8 documentationUrl: https://docs.airbyte.com/integrations/sources/mysql icon: mysql.svg sourceType: database releaseStage: beta +- name: NASA + sourceDefinitionId: 1a8667d7-7978-43cd-ba4d-d32cbd478971 + dockerRepository: airbyte/source-nasa + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/nasa + icon: nasa.svg + sourceType: api + releaseStage: alpha - name: Netsuite sourceDefinitionId: 4f2f093d-ce44-4121-8118-9d13b7bfccd0 dockerRepository: airbyte/source-netsuite dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.com/integrations/sources/netsuite - # icon: notion.svg + sourceType: api + releaseStage: alpha +- name: News API + sourceDefinitionId: df38991e-f35b-4af2-996d-36817f614587 + dockerRepository: airbyte/source-news-api + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/news-api sourceType: api releaseStage: alpha - name: Notion @@ -774,8 +867,15 @@ icon: orbit.svg sourceType: api releaseStage: alpha -- sourceDefinitionId: 3490c201-5d95-4783-b600-eaf07a4c7787 - name: Outreach +- name: Oura + sourceDefinitionId: 2bf6c581-bec5-4e32-891d-de33036bd631 + dockerRepository: airbyte/source-oura + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/oura + sourceType: api + releaseStage: alpha +- name: Outreach + sourceDefinitionId: 3490c201-5d95-4783-b600-eaf07a4c7787 dockerRepository: airbyte/source-outreach dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.com/integrations/sources/outreach @@ -864,7 +964,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 1.0.19 + dockerImageTag: 1.0.21 documentationUrl: https://docs.airbyte.com/integrations/sources/postgres icon: postgresql.svg sourceType: database @@ -933,6 +1033,13 @@ icon: retently.svg sourceType: api releaseStage: alpha +- name: RD Station Marketing + sourceDefinitionId: fb141f29-be2a-450b-a4f2-2cd203a00f84 + dockerRepository: airbyte/source-rd-station-marketing + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/rd-station-marketing + sourceType: api + releaseStage: alpha - name: RKI Covid sourceDefinitionId: d78e5de0-aa44-4744-aa4f-74c818ccfe19 dockerRepository: airbyte/source-rki-covid @@ -943,7 +1050,7 @@ - name: S3 sourceDefinitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2 dockerRepository: airbyte/source-s3 - dockerImageTag: 0.1.24 + dockerImageTag: 0.1.25 documentationUrl: https://docs.airbyte.com/integrations/sources/s3 icon: s3.svg sourceType: file @@ -1028,6 +1135,13 @@ icon: snowflake.svg sourceType: database releaseStage: alpha +- name: Sonar Cloud + sourceDefinitionId: 3ab1d7d0-1577-4ab9-bcc4-1ff6a4c2c9f2 + dockerRepository: airbyte/source-sonar-cloud + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/sonar-cloud + sourceType: api + releaseStage: alpha - name: Square sourceDefinitionId: 77225a51-cd15-4a13-af02-65816bd0ecf4 dockerRepository: airbyte/source-square @@ -1108,10 +1222,18 @@ icon: trelllo.svg sourceType: api releaseStage: alpha +- name: TVMaze Schedule + sourceDefinitionId: bd14b08f-9f43-400f-b2b6-7248b5c72561 + dockerRepository: airbyte/source-tvmaze-schedule + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/tvmaze-schedule + icon: trelllo.svg + sourceType: api + releaseStage: alpha - name: Twilio sourceDefinitionId: b9dc6155-672e-42ea-b10d-9f1f1fb95ab1 dockerRepository: airbyte/source-twilio - dockerImageTag: 0.1.12 + dockerImageTag: 0.1.13 documentationUrl: https://docs.airbyte.com/integrations/sources/twilio icon: twilio.svg sourceType: api @@ -1178,6 +1300,13 @@ icon: woocommerce.svg sourceType: api releaseStage: alpha +- name: Workable + sourceDefinitionId: ef3c99c6-9e90-43c8-9517-926cfd978517 + dockerRepository: airbyte/source-workable + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/workable + sourceType: api + releaseStage: alpha - name: Wrike sourceDefinitionId: 9c13f986-a13b-4988-b808-4705badf71c2 dockerRepository: airbyte/source-wrike @@ -1194,6 +1323,14 @@ icon: zendesk.svg sourceType: api releaseStage: generally_available +- name: Zendesk Sell + sourceDefinitionId: 982eaa4c-bba1-4cce-a971-06a41f700b8c + dockerRepository: airbyte/source-zendesk-sell + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-sell + icon: zendesk.svg + sourceType: api + releaseStage: alpha - name: Zendesk Sunshine sourceDefinitionId: 325e0640-e7b3-4e24-b823-3361008f603f dockerRepository: airbyte/source-zendesk-sunshine @@ -1241,14 +1378,6 @@ icon: sentry.svg sourceType: api releaseStage: generally_available -- name: Zoom - sourceDefinitionId: aea2fd0d-377d-465e-86c0-4fdc4f688e51 - dockerRepository: airbyte/source-zoom-singer - dockerImageTag: 0.2.4 - documentationUrl: https://docs.airbyte.com/integrations/sources/zoom - icon: zoom.svg - sourceType: api - releaseStage: alpha - name: Zuora sourceDefinitionId: 3dc3037c-5ce8-4661-adc2-f7a9e3c5ece5 dockerRepository: airbyte/source-zuora @@ -1278,6 +1407,13 @@ documentationUrl: https://docs.airbyte.com/integrations/sources/sftp sourceType: file releaseStage: alpha +- name: SFTP Bulk + sourceDefinitionId: 31e3242f-dee7-4cdc-a4b8-8e06c5458517 + dockerRepository: airbyte/source-sftp-bulk + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/sftp-bulk + sourceType: file + releaseStage: alpha - name: Firebolt sourceDefinitionId: 6f2ac653-8623-43c4-8950-19218c7caf3d dockerRepository: airbyte/source-firebolt @@ -1292,6 +1428,13 @@ documentationUrl: https://docs.airbyte.com/integrations/sources/elasticsearch sourceType: api releaseStage: alpha +- name: Waiteraid + sourceDefinitionId: 03a53b13-794a-4d6b-8544-3b36ed8f3ce4 + dockerRepository: airbyte/source-waiteraid + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/waiteraid + sourceType: api + releaseStage: alpha - name: Yandex Metrica sourceDefinitionId: 7865dce4-2211-4f6a-88e5-9d0fe161afe7 dockerRepository: airbyte/source-yandex-metrica @@ -1299,3 +1442,11 @@ documentationUrl: https://docs.airbyte.com/integrations/sources/yandex-metrica sourceType: api releaseStage: alpha +- name: Zoom + sourceDefinitionId: cbfd9856-1322-44fb-bcf1-0b39b7a8e92e + dockerRepository: airbyte/source-zoom + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/zoom + sourceType: api + icon: zoom.svg + releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 3706bf8b0fa5..3071dd229099 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1,6 +1,28 @@ # This file is generated by io.airbyte.config.specs.SeedConnectorSpecGenerator. # Do NOT edit this file directly. See generator class for more details. --- +- dockerImage: "airbyte/source-activecampaign:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/activecampaign" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Activecampaign Spec" + type: "object" + required: + - "api_key" + - "account_username" + additionalProperties: true + properties: + api_key: + type: "string" + description: "API Key" + airbyte_secret: true + account_username: + type: "string" + description: "Account Username" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-adjust:0.1.0" spec: documentationUrl: "https://raw.githubusercontent.com/appchoose/airbyte/feature/source-adjust/docs/integrations/sources/adjust.md" @@ -156,7 +178,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-airtable:0.1.2" +- dockerImage: "airbyte/source-airtable:0.1.3" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/airtable" connectionSpecification: @@ -167,7 +189,6 @@ - "api_key" - "base_id" - "tables" - additionalProperties: false properties: api_key: type: "string" @@ -198,7 +219,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-alloydb:1.0.15" +- dockerImage: "airbyte/source-alloydb:1.0.16" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: @@ -1333,6 +1354,118 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" +- dockerImage: "airbyte/source-ashby:0.1.0" + spec: + documentationUrl: "https://developers.ashbyhq.com/reference/introduction" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Ashby Spec" + type: "object" + required: + - "api_key" + - "start_date" + additionalProperties: true + properties: + api_key: + type: "string" + title: "Ashby API key" + description: "The Ashby API Key, see doc here." + airbyte_secret: true + start_date: + type: "string" + title: "Start date" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ + \ data before this date will not be replicated." + examples: + - "2017-01-25T00:00:00Z" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-auth0:0.1.0" + spec: + documentationUrl: "https://auth0.com/docs/api/management/v2/" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Auth0 Management API Spec" + type: "object" + required: + - "base_url" + - "credentials" + additionalProperties: true + properties: + base_url: + type: "string" + title: "Base URL" + examples: + - "https://dev-yourOrg.us.auth0.com/" + description: "The Authentication API is served over HTTPS. All URLs referenced\ + \ in the documentation have the following base `https://YOUR_DOMAIN`" + credentials: + title: "Authentication Method" + type: "object" + oneOf: + - type: "object" + title: "OAuth2 Confidential Application" + required: + - "auth_type" + - "client_id" + - "client_secret" + - "audience" + properties: + auth_type: + type: "string" + title: "Authentication Method" + const: "oauth2_confidential_application" + order: 0 + client_id: + title: "Client ID" + description: "Your application's Client ID. You can find this value\ + \ on the application's\ + \ settings tab after you login the admin portal." + type: "string" + examples: + - "Client_ID" + client_secret: + title: "Client Secret" + description: "Your application's Client Secret. You can find this\ + \ value on the application's settings tab after you login the admin portal." + type: "string" + examples: + - "Client_Secret" + airbyte_secret: true + audience: + title: "Audience" + description: "The audience for the token, which is your API. You can\ + \ find this in the Identifier field on your API's settings tab" + type: "string" + examples: + - "https://dev-yourOrg.us.auth0.com/api/v2/" + - type: "object" + title: "OAuth2 Access Token" + required: + - "access_token" + - "auth_type" + properties: + auth_type: + type: "string" + title: "Authentication Method" + const: "oauth2_access_token" + order: 0 + access_token: + title: "OAuth2 Access Token" + description: "Also called API Access Token The access token used to call the Auth0 Management\ + \ API Token. It's a JWT that contains specific grant permissions\ + \ knowns as scopes." + type: "string" + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-azure-table:0.1.3" spec: documentationUrl: "https://docsurl.com" @@ -2090,6 +2223,69 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-coin-api:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/coin-api" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Coin API Spec" + type: "object" + required: + - "api_key" + - "environment" + - "symbol_id" + - "period" + - "start_date" + properties: + api_key: + type: "string" + description: "API Key" + airbyte_secret: true + order: 0 + environment: + type: "string" + description: "The environment to use. Either sandbox or production.\n" + enum: + - "sandbox" + - "production" + default: "sandbox" + order: 1 + symbol_id: + type: "string" + description: "The symbol ID to use. See the documentation for a list.\n\ + https://docs.coinapi.io/#list-all-symbols-get\n" + order: 2 + period: + type: "string" + description: "The period to use. See the documentation for a list. https://docs.coinapi.io/#list-all-periods-get" + examples: + - "5SEC" + - "2MTH" + start_date: + type: "string" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$" + description: "The start date in ISO 8601 format." + examples: + - "2019-01-01T00:00:00" + end_date: + type: "string" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$" + description: "The end date in ISO 8601 format. If not supplied, data will\ + \ be returned\nfrom the start date to the current time, or when the count\ + \ of result\nelements reaches its limit.\n" + examples: + - "2019-01-01T00:00:00" + limit: + type: "integer" + description: "The maximum number of elements to return. If not supplied,\ + \ the default\nis 100. For numbers larger than 100, each 100 items is\ + \ counted as one\nrequest for pricing purposes. Maximum value is 100000.\n" + minimum: 1 + maximum: 100000 + default: 100 + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-commercetools:0.1.0" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/commercetools" @@ -2169,6 +2365,24 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-convertkit:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/convertkit" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Convertkit Spec" + type: "object" + required: + - "api_secret" + additionalProperties: true + properties: + api_secret: + type: "string" + description: "API Secret" + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-courier:0.1.0" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/courier" @@ -2187,6 +2401,31 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-clockify:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/clockify" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Clockify Spec" + type: "object" + required: + - "workspace_id" + - "api_key" + additionalProperties: true + properties: + workspace_id: + title: "Workspace Id" + description: "WorkSpace Id" + type: "string" + api_key: + title: "API Key" + description: "You can get your api access_key here This API is Case Sensitive." + type: "string" + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "farosai/airbyte-customer-io-source:0.1.23" spec: documentationUrl: "https://docs.faros.ai" @@ -2593,7 +2832,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-facebook-marketing:0.2.69" +- dockerImage: "airbyte/source-facebook-marketing:0.2.70" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/facebook-marketing" changelogUrl: "https://docs.airbyte.com/integrations/sources/facebook-marketing" @@ -2776,6 +3015,7 @@ - "social_spend" - "spend" - "total_postbacks" + - "total_postbacks_detailed" - "unique_actions" - "unique_clicks" - "unique_conversions" @@ -2832,12 +3072,15 @@ - "hourly_stats_aggregated_by_audience_time_zone" - "image_asset" - "impression_device" + - "is_conversion_id_modeled" - "link_url_asset" + - "mmm" - "place_page_id" - "platform_position" - "product_id" - "publisher_platform" - "region" + - "skan_campaign_id" - "skan_conversion_id" - "title_asset" - "video_asset" @@ -3123,7 +3366,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-file:0.2.26" +- dockerImage: "airbyte/source-file:0.2.28" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/file" connectionSpecification: @@ -4296,7 +4539,7 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" -- dockerImage: "airbyte/source-google-search-console:0.1.17" +- dockerImage: "airbyte/source-google-search-console:0.1.18" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/google-search-console" connectionSpecification: @@ -4520,6 +4763,34 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" +- dockerImage: "airbyte/source-google-webfonts:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/google-webfonts" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Google Webfonts Spec" + type: "object" + required: + - "api_key" + additionalProperties: true + properties: + api_key: + type: "string" + description: "API key is required to access google apis, For getting your's\ + \ goto google console and generate api key for Webfonts" + airbyte_secret: true + sort: + type: "string" + description: "Optional, to find how to sort" + prettyPrint: + type: "string" + description: "Optional, boolean type" + alt: + type: "string" + description: "Optional, Available params- json, media, proto" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-google-workspace-admin-reports:0.1.8" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/google-workspace-admin-reports" @@ -4554,7 +4825,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-greenhouse:0.2.11" +- dockerImage: "airbyte/source-greenhouse:0.3.0" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/greenhouse" connectionSpecification: @@ -5192,7 +5463,7 @@ oauthFlowInitParameters: [] oauthFlowOutputParameters: - - "access_token" -- dockerImage: "airbyte/source-iterable:0.1.20" +- dockerImage: "airbyte/source-iterable:0.1.21" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/iterable" connectionSpecification: @@ -5741,7 +6012,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-lever-hiring:0.1.2" +- dockerImage: "airbyte/source-lever-hiring:0.1.3" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/lever-hiring" changelogUrl: "https://docs.airbyte.com/integrations/sources/lever-hiring#changelog" @@ -5767,9 +6038,6 @@ auth_type: type: "string" const: "Client" - enum: - - "Client" - default: "Client" order: 0 client_id: title: "Client ID" @@ -5780,16 +6048,26 @@ type: "string" description: "The Client Secret of your Lever Hiring developer application." airbyte_secret: true - option_title: - type: "string" - title: "Credentials Title" - description: "OAuth Credentials" - const: "OAuth Credentials" refresh_token: type: "string" title: "Refresh Token" description: "The token for obtaining new access token." airbyte_secret: true + - type: "object" + title: "Authenticate via Lever (Api Key)" + required: + - "api_key" + properties: + auth_type: + type: "string" + const: "Api Key" + order: 0 + api_key: + title: "Api key" + type: "string" + description: "The Api Key of your Lever Hiring account." + airbyte_secret: true + order: 1 start_date: order: 0 type: "string" @@ -6069,6 +6347,32 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-lokalise:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/lokalise" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Lokalise Spec" + type: "object" + required: + - "api_key" + - "project_id" + additionalProperties: true + properties: + api_key: + title: "API Key" + type: "string" + description: "Lokalise API Key with read-access. Available at Profile settings\ + \ > API tokens. See here." + airbyte_secret: true + project_id: + title: "Project Id" + type: "string" + description: "Lokalise project ID. Available at Project Settings > General." + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-looker:0.2.7" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/looker" @@ -6208,58 +6512,139 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-mailgun:0.1.0" +- dockerImage: "airbyte/source-mailjet-mail:0.1.0" spec: - documentationUrl: "https://docs.airbyte.com/integrations/sources/mailgun" + documentationUrl: "https://docs.airbyte.com/integrations/sources/mailjet-mail" connectionSpecification: $schema: "http://json-schema.org/draft-07/schema#" - title: "Source Mailgun Spec" + title: "Mailjet Mail Spec" type: "object" required: - - "private_key" + - "api_key" + - "api_key_secret" additionalProperties: true properties: - private_key: - type: "string" - airbyte_secret: true - description: "Primary account API key to access your Mailgun data." - title: "Private API Key" - domain_region: + api_key: + title: "API Key" type: "string" - description: "Domain region code. 'EU' or 'US' are possible values. The\ - \ default is 'US'." - title: "Domain Region Code" - start_date: - title: "Replication Start Date" - description: "UTC date and time in the format 2020-10-01 00:00:00. Any data\ - \ before this date will not be replicated. If omitted, defaults to 3 days\ - \ ago." - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}$" - examples: - - "2020-10-01 00:00:00" + description: "Your API Key. See here." + api_key_secret: + title: "API Secret Key" type: "string" + description: "Your API Secret Key. See here." + airbyte_secret: true supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-marketo:0.1.11" +- dockerImage: "airbyte/source-mailjet-sms:0.1.0" spec: - documentationUrl: "https://docs.airbyte.com/integrations/sources/marketo" + documentationUrl: "https://docs.airbyte.com/integrations/sources/mailjet-sms" connectionSpecification: $schema: "http://json-schema.org/draft-07/schema#" - title: "Source Marketo Spec" + title: "Mailjet Sms Spec" type: "object" required: - - "domain_url" - - "client_id" - - "client_secret" - - "start_date" + - "token" additionalProperties: true properties: - domain_url: - title: "Domain URL" + token: + title: "Access Token" type: "string" - order: 3 - description: "Your Marketo Base URL. See here." + airbyte_secret: true + start_date: + title: "Start date" + type: "integer" + description: "Retrieve SMS messages created after the specified timestamp.\ + \ Required format - Unix timestamp." + pattern: "^[0-9]*$" + examples: + - 1666261656 + end_date: + title: "End date" + type: "integer" + description: "Retrieve SMS messages created before the specified timestamp.\ + \ Required format - Unix timestamp." + pattern: "^[0-9]*$" + examples: + - 1666281656 + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-mailerlite:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/mailerlite" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Mailerlite Spec" + type: "object" + required: + - "api_token" + additionalProperties: true + properties: + api_token: + type: "string" + description: "Your API Token. See here." + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-mailgun:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/mailgun" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Source Mailgun Spec" + type: "object" + required: + - "private_key" + additionalProperties: true + properties: + private_key: + type: "string" + airbyte_secret: true + description: "Primary account API key to access your Mailgun data." + title: "Private API Key" + domain_region: + type: "string" + description: "Domain region code. 'EU' or 'US' are possible values. The\ + \ default is 'US'." + title: "Domain Region Code" + start_date: + title: "Replication Start Date" + description: "UTC date and time in the format 2020-10-01 00:00:00. Any data\ + \ before this date will not be replicated. If omitted, defaults to 3 days\ + \ ago." + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}$" + examples: + - "2020-10-01 00:00:00" + type: "string" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-marketo:0.1.11" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/marketo" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Source Marketo Spec" + type: "object" + required: + - "domain_url" + - "client_id" + - "client_secret" + - "start_date" + additionalProperties: true + properties: + domain_url: + title: "Domain URL" + type: "string" + order: 3 + description: "Your Marketo Base URL. See the docs for info on how to obtain this." examples: - "https://000-AAA-000.mktorest.com" @@ -6330,7 +6715,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mssql:0.4.23" +- dockerImage: "airbyte/source-mssql:0.4.24" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/mssql" connectionSpecification: @@ -7160,7 +7545,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mysql:1.0.7" +- dockerImage: "airbyte/source-mysql:1.0.8" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/mysql" connectionSpecification: @@ -7504,6 +7889,60 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-nasa:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/nasa-apod" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "NASA spec" + type: "object" + required: + - "api_key" + properties: + api_key: + type: "string" + description: "API access key used to retrieve data from the NASA APOD API." + airbyte_secret: true + concept_tags: + type: "boolean" + default: false + description: "Indicates whether concept tags should be returned with the\ + \ rest of the response. The concept tags are not necessarily included\ + \ in the explanation, but rather derived from common search tags that\ + \ are associated with the description text. (Better than just pure text\ + \ search.) Defaults to False." + count: + type: "integer" + minimum: 1 + maximum: 100 + description: "A positive integer, no greater than 100. If this is specified\ + \ then `count` randomly chosen images will be returned in a JSON array.\ + \ Cannot be used in conjunction with `date` or `start_date` and `end_date`." + start_date: + type: "string" + description: "Indicates the start of a date range. All images in the range\ + \ from `start_date` to `end_date` will be returned in a JSON array. Must\ + \ be after 1995-06-16, the first day an APOD picture was posted. There\ + \ are no images for tomorrow available through this API." + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + examples: + - "2022-10-20" + end_date: + type: "string" + description: "Indicates that end of a date range. If `start_date` is specified\ + \ without an `end_date` then `end_date` defaults to the current date." + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + examples: + - "2022-10-20" + thumbs: + type: "boolean" + default: false + description: "Indicates whether the API should return a thumbnail image\ + \ URL for video files. If set to True, the API returns URL of video thumbnail.\ + \ If an APOD is not a video, this parameter is ignored." + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-netsuite:0.1.1" spec: documentationUrl: "https://docsurl.com" @@ -7584,6 +8023,187 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-news-api:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/news-api" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "News Api Spec" + type: "object" + required: + - "api_key" + - "country" + - "category" + - "sort_by" + additionalProperties: true + properties: + api_key: + type: "string" + description: "API Key" + airbyte_secret: true + order: 0 + search_query: + type: "string" + description: "Search query. See https://newsapi.org/docs/endpoints/everything\ + \ for \ninformation.\n" + examples: + - "+bitcoin OR +crypto" + - "sunak AND (truss OR johnson)" + order: 1 + search_in: + type: "array" + description: "Where to apply search query. Possible values are: title, description,\n\ + content.\n" + items: + type: "string" + enum: + - "title" + - "description" + - "content" + order: 2 + sources: + type: "array" + description: "Identifiers (maximum 20) for the news sources or blogs you\ + \ want\nheadlines from. Use the `/sources` endpoint to locate these\n\ + programmatically or look at the sources index:\nhttps://newsapi.com/sources.\ + \ Will override both country and category.\n" + items: + type: "string" + order: 3 + domains: + type: "array" + description: "A comma-seperated string of domains (eg bbc.co.uk, techcrunch.com,\n\ + engadget.com) to restrict the search to.\n" + items: + type: "string" + order: 4 + exclude_domains: + type: "array" + description: "A comma-seperated string of domains (eg bbc.co.uk, techcrunch.com,\n\ + engadget.com) to remove from the results.\n" + items: + type: "string" + order: 5 + start_date: + type: "string" + description: "A date and optional time for the oldest article allowed. This\ + \ should\nbe in ISO 8601 format (e.g. 2021-01-01 or 2021-01-01T12:00:00).\n" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2})?$" + order: 6 + end_date: + type: "string" + description: "A date and optional time for the newest article allowed. This\ + \ should\nbe in ISO 8601 format (e.g. 2021-01-01 or 2021-01-01T12:00:00).\n" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2})?$" + order: 7 + language: + type: "string" + description: "The 2-letter ISO-639-1 code of the language you want to get\ + \ headlines\nfor. Possible options: ar de en es fr he it nl no pt ru se\ + \ ud zh.\n" + enum: + - "ar" + - "de" + - "en" + - "es" + - "fr" + - "he" + - "it" + - "nl" + - false + - "pt" + - "ru" + - "se" + - "ud" + - "zh" + order: 8 + country: + type: "string" + description: "The 2-letter ISO 3166-1 code of the country you want to get\ + \ headlines\nfor. You can't mix this with the sources parameter.\n" + enum: + - "ae" + - "ar" + - "at" + - "au" + - "be" + - "bg" + - "br" + - "ca" + - "ch" + - "cn" + - "co" + - "cu" + - "cz" + - "de" + - "eg" + - "fr" + - "gb" + - "gr" + - "hk" + - "hu" + - "id" + - "ie" + - "il" + - "in" + - "it" + - "jp" + - "kr" + - "lt" + - "lv" + - "ma" + - "mx" + - "my" + - "ng" + - "nl" + - false + - "nz" + - "ph" + - "pl" + - "pt" + - "ro" + - "rs" + - "ru" + - "sa" + - "se" + - "sg" + - "si" + - "sk" + - "th" + - "tr" + - "tw" + - "ua" + - "us" + - "ve" + - "za" + default: "us" + order: 9 + category: + type: "string" + description: "The category you want to get top headlines for." + enum: + - "business" + - "entertainment" + - "general" + - "health" + - "science" + - "sports" + - "technology" + default: "business" + order: 10 + sort_by: + type: "string" + description: "The order to sort the articles in. Possible options: relevancy,\n\ + popularity, publishedAt.\n" + enum: + - "relevancy" + - "popularity" + - "publishedAt" + default: "publishedAt" + order: 11 + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-notion:0.1.10" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/notion" @@ -8285,6 +8905,36 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-oura:0.1.0" + spec: + documentationUrl: "https://docsurl.com" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Oura Spec" + type: "object" + required: + - "api_key" + additionalProperties: true + properties: + api_key: + type: "string" + description: "API Key" + airbyte_secret: true + order: 0 + start_datetime: + type: "string" + description: "Start datetime to sync from. Default is current UTC datetime\ + \ minus 1\nday.\n" + pattern: "^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(Z|[+-](?:2[0-3]|[01][0-9]):[0-5][0-9])?$" + order: 1 + end_datetime: + type: "string" + description: "End datetime to sync until. Default is current UTC datetime." + pattern: "^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(Z|[+-](?:2[0-3]|[01][0-9]):[0-5][0-9])?$" + order: 2 + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-outreach:0.1.2" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/outreach" @@ -8792,7 +9442,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:1.0.19" +- dockerImage: "airbyte/source-postgres:1.0.21" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: @@ -9612,6 +10262,70 @@ path_in_connector_config: - "credentials" - "client_secret" +- dockerImage: "airbyte/source-rd-station-marketing:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/rd-station-marketing" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "RD Station Marketing Spec" + type: "object" + required: + - "start_date" + additionalProperties: true + properties: + authorization: + type: "object" + title: "Authentication Type" + description: "Choose one of the possible authorization method" + oneOf: + - title: "Sign in via RD Station (OAuth)" + type: "object" + required: + - "auth_type" + properties: + auth_type: + type: "string" + const: "Client" + order: 0 + client_id: + title: "Client ID" + type: "string" + description: "The Client ID of your RD Station developer application." + airbyte_secret: true + client_secret: + title: "Client Secret" + type: "string" + description: "The Client Secret of your RD Station developer application" + airbyte_secret: true + refresh_token: + title: "Refresh Token" + type: "string" + description: "The token for obtaining the new access token." + airbyte_secret: true + start_date: + title: "Start Date" + description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ + \ data before this date will not be replicated. When specified and not\ + \ None, then stream will behave as incremental" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + examples: + - "2017-01-25T00:00:00Z" + type: "string" + supportsIncremental: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] + authSpecification: + auth_type: "oauth2.0" + oauth2Specification: + rootObject: + - "authorization" + - "0" + oauthFlowInitParameters: + - - "client_id" + - - "client_secret" + oauthFlowOutputParameters: + - - "refresh_token" - dockerImage: "airbyte/source-rki-covid:0.1.1" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/rki-covid" @@ -9632,7 +10346,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-s3:0.1.24" +- dockerImage: "airbyte/source-s3:0.1.25" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/s3" changelogUrl: "https://docs.airbyte.com/integrations/sources/s3" @@ -10798,6 +11512,56 @@ path_in_connector_config: - "credentials" - "client_secret" +- dockerImage: "airbyte/source-sonar-cloud:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/sonar-cloud" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Sonar Cloud Spec" + type: "object" + required: + - "user_token" + - "organization" + - "component_keys" + additionalProperties: true + properties: + user_token: + title: "User Token" + type: "string" + description: "Your User Token. See here. The token is case sensitive." + airbyte_secret: true + organization: + title: "Organization" + type: "string" + description: "Organization key. See here." + examples: + - "airbyte" + component_keys: + title: "Component Keys" + type: "array" + description: "Comma-separated list of component keys." + examples: + - "airbyte-ws-order" + - "airbyte-ws-checkout" + start_date: + title: "Start date" + type: "string" + description: "To retrieve issues created after the given date (inclusive)." + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + examples: + - "YYYY-MM-DD" + end_date: + title: "End date" + type: "string" + description: "To retrieve issues created before the given date (inclusive)." + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + examples: + - "YYYY-MM-DD" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-square:0.1.4" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/square" @@ -11610,7 +12374,48 @@ oauthFlowOutputParameters: - - "token" - - "key" -- dockerImage: "airbyte/source-twilio:0.1.12" +- dockerImage: "airbyte/source-tvmaze-schedule:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/tvmaze-schedule" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "TVMaze Schedule Spec" + type: "object" + required: + - "start_date" + - "domestic_schedule_country_code" + additionalProperties: true + properties: + start_date: + type: "string" + description: "Start date for TV schedule retrieval. May be in the future." + order: 0 + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + end_date: + type: "string" + description: "End date for TV schedule retrieval. May be in the future.\ + \ Optional.\n" + order: 1 + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + domestic_schedule_country_code: + type: "string" + description: "Country code for domestic TV schedule retrieval." + examples: + - "US" + - "GB" + web_schedule_country_code: + type: "string" + description: "ISO 3166-1 country code for web TV schedule retrieval. Leave\ + \ blank for\nall countries plus global web channels (e.g. Netflix). Alternatively,\n\ + set to 'global' for just global web channels.\n" + examples: + - "US" + - "GB" + - "global" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-twilio:0.1.13" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/twilio" connectionSpecification: @@ -11919,6 +12724,40 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-workable:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/workable" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Workable API Spec" + type: "object" + required: + - "api_key" + - "account_subdomain" + - "start_date" + additionalProperties: true + properties: + api_key: + title: "API Key" + type: "string" + description: "Your Workable API Key. See here." + airbyte_secret: true + account_subdomain: + title: "Account Subdomain" + type: "string" + description: "Your Workable account subdomain, e.g. https://your_account_subdomain.workable.com." + start_date: + title: "Start Date" + type: "string" + description: "Get data that was created since this date (format: YYYYMMDDTHHMMSSZ)." + pattern: "^[0-9]{8}T[0-9]{6}Z$" + examples: + - "20150708T115616Z" + - "20221115T225616Z" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-wrike:0.1.0" spec: documentationUrl: "https://docsurl.com" @@ -12080,6 +12919,26 @@ path_in_connector_config: - "credentials" - "client_secret" +- dockerImage: "airbyte/source-zendesk-sell:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/zendesk-sell" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Source Zendesk Sell Spec" + type: "object" + required: + - "api_token" + properties: + api_token: + title: "API token" + type: "string" + description: "The API token for authenticating to Zendesk Sell" + examples: + - "f23yhd630otl94y85a8bf384958473pto95847fd006da49382716or937ruw059" + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-zendesk-sunshine:0.1.1" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/zendesk_sunshine" @@ -12545,26 +13404,6 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-zoom-singer:0.2.4" - spec: - documentationUrl: "https://docs.airbyte.com/integrations/sources/zoom" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Source Zoom Singer Spec" - type: "object" - required: - - "jwt" - additionalProperties: false - properties: - jwt: - title: "JWT Token" - type: "string" - description: "Zoom JWT Token. See the docs for more information on how to obtain this key." - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] - dockerImage: "airbyte/source-zuora:0.1.3" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/zuora" @@ -12852,6 +13691,113 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-sftp-bulk:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/source/ftp" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "FTP Source Spec" + type: "object" + required: + - "username" + - "host" + - "port" + - "stream_name" + - "start_date" + - "folder_path" + additionalProperties: true + properties: + username: + title: "User Name" + description: "The server user" + type: "string" + order: 0 + password: + title: "Password" + description: "OS-level password for logging into the jump server host" + type: "string" + airbyte_secret: true + order: 1 + private_key: + title: "Private key" + description: "The private key" + type: "string" + multiline: true + order: 1 + host: + title: "Host Address" + description: "The server host address" + type: "string" + examples: + - "www.host.com" + - "192.0.2.1" + order: 1 + port: + title: "Port" + description: "The server port" + type: "integer" + default: 22 + examples: + - "22" + order: 2 + stream_name: + title: "Stream name" + description: "The name of the stream or table you want to create" + type: "string" + examples: + - "ftp_contacts" + order: 1 + file_type: + title: "File type" + description: "The file type you want to sync. Currently only 'csv' and 'json'\ + \ files are supported." + type: "string" + default: "csv" + enum: + - "csv" + - "json" + order: 4 + examples: + - "csv" + - "json" + folder_path: + title: "Folder Path (Optional)" + description: "The directory to search files for sync" + type: "string" + default: "" + examples: + - "/logs/2022" + order: 5 + file_pattern: + title: "File Pattern (Optional)" + description: "The regular expression to specify files for sync in a chosen\ + \ Folder Path" + type: "string" + default: "" + examples: + - "log-([0-9]{4})([0-9]{2})([0-9]{2}) - This will filter files which `log-yearmmdd`" + order: 6 + file_most_recent: + title: "Most recent file (Optional)" + description: "Sync only the most recent file for the configured folder path\ + \ and file pattern" + type: "boolean" + default: false + order: 7 + start_date: + type: "string" + title: "Start Date" + format: "date-time" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + examples: + - "2017-01-25T00:00:00Z" + description: "The date from which you'd like to replicate data for all incremental\ + \ streams, in the format YYYY-MM-DDT00:00:00Z. All data generated after\ + \ this date will be replicated." + order: 8 + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-firebolt:0.1.0" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/firebolt" @@ -12971,6 +13917,40 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-waiteraid:0.1.0" + spec: + documentationUrl: "https://docsurl.com" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Waiteraid Spec" + type: "object" + required: + - "start_date" + - "auth_hash" + - "restid" + additionalProperties: true + properties: + start_date: + title: "Start Date" + type: "string" + description: "Start getting data from that date." + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + examples: + - "YYYY-MM-DD" + auth_hash: + title: "Authentication Hash" + type: "string" + description: "Your WaiterAid API key, obtained from API request with Username\ + \ and Password" + airbyte_secret: true + restid: + title: "Restaurant ID" + type: "string" + description: "Your WaiterAid restaurant id from API request to getRestaurants" + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-yandex-metrica:0.1.0" spec: documentationUrl: "https://docsurl.com" @@ -13016,3 +13996,21 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-zoom:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/zoom" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Zoom Spec" + type: "object" + required: + - "jwt_token" + additionalProperties: true + properties: + jwt_token: + type: "string" + description: "JWT Token" + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] diff --git a/airbyte-container-orchestrator/build.gradle b/airbyte-container-orchestrator/build.gradle index 3a96f895137d..907a6937a00b 100644 --- a/airbyte-container-orchestrator/build.gradle +++ b/airbyte-container-orchestrator/build.gradle @@ -13,6 +13,7 @@ dependencies { implementation project(':airbyte-api') implementation project(':airbyte-config:config-models') implementation project(':airbyte-config:config-persistence') + implementation project(':airbyte-commons-protocol') implementation project(':airbyte-commons-temporal') implementation project(':airbyte-commons-worker') implementation project(':airbyte-config:init') diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java index 22f82426b494..b26012ebba95 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java @@ -8,6 +8,12 @@ import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.logging.LoggingHelper; import io.airbyte.commons.logging.MdcScope; +import io.airbyte.commons.protocol.AirbyteMessageMigrator; +import io.airbyte.commons.protocol.AirbyteMessageSerDeProvider; +import io.airbyte.commons.protocol.AirbyteMessageVersionedMigratorFactory; +import io.airbyte.commons.protocol.migrations.AirbyteMessageMigrationV0; +import io.airbyte.commons.protocol.serde.AirbyteMessageV0Deserializer; +import io.airbyte.commons.protocol.serde.AirbyteMessageV0Serializer; import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.temporal.sync.OrchestratorConstants; import io.airbyte.config.Configs; @@ -32,6 +38,7 @@ import java.io.IOException; import java.net.InetAddress; import java.nio.file.Path; +import java.util.List; import java.util.Map; import java.util.Optional; import lombok.extern.slf4j.Slf4j; @@ -113,7 +120,10 @@ private void runInternal(final DefaultAsyncStateManager asyncStateManager) { final WorkerConfigs workerConfigs = new WorkerConfigs(configs); final ProcessFactory processFactory = getProcessBuilderFactory(configs, workerConfigs); - final JobOrchestrator jobOrchestrator = getJobOrchestrator(configs, workerConfigs, processFactory, application, featureFlags); + final AirbyteMessageSerDeProvider serDeProvider = getAirbyteMessageSerDeProvider(); + final AirbyteMessageVersionedMigratorFactory migratorFactory = getAirbyteMessageVersionedMigratorFactory(); + final JobOrchestrator jobOrchestrator = + getJobOrchestrator(configs, workerConfigs, processFactory, application, featureFlags, serDeProvider, migratorFactory); if (jobOrchestrator == null) { throw new IllegalStateException("Could not find job orchestrator for application: " + application); @@ -201,9 +211,12 @@ private static JobOrchestrator getJobOrchestrator(final Configs configs, final WorkerConfigs workerConfigs, final ProcessFactory processFactory, final String application, - final FeatureFlags featureFlags) { + final FeatureFlags featureFlags, + final AirbyteMessageSerDeProvider serDeProvider, + final AirbyteMessageVersionedMigratorFactory migratorFactory) { return switch (application) { - case ReplicationLauncherWorker.REPLICATION -> new ReplicationJobOrchestrator(configs, processFactory, featureFlags); + case ReplicationLauncherWorker.REPLICATION -> new ReplicationJobOrchestrator(configs, processFactory, featureFlags, serDeProvider, + migratorFactory); case NormalizationLauncherWorker.NORMALIZATION -> new NormalizationJobOrchestrator(configs, processFactory); case DbtLauncherWorker.DBT -> new DbtJobOrchestrator(configs, workerConfigs, processFactory); case AsyncOrchestratorPodProcess.NO_OP -> new NoOpOrchestrator(); @@ -241,4 +254,23 @@ private static ProcessFactory getProcessBuilderFactory(final Configs configs, fi } } + // Create AirbyteMessageSerDeProvider + // This should be replaced by a simple injection once we migrated the orchestrator to the micronaut + private static AirbyteMessageSerDeProvider getAirbyteMessageSerDeProvider() { + final AirbyteMessageSerDeProvider serDeProvider = new AirbyteMessageSerDeProvider( + List.of(new AirbyteMessageV0Deserializer()), + List.of(new AirbyteMessageV0Serializer())); + serDeProvider.initialize(); + return serDeProvider; + } + + // Create AirbyteMessageVersionedMigratorFactory + // This should be replaced by a simple injection once we migrated the orchestrator to the micronaut + private static AirbyteMessageVersionedMigratorFactory getAirbyteMessageVersionedMigratorFactory() { + final AirbyteMessageMigrator messageMigrator = new AirbyteMessageMigrator( + List.of(new AirbyteMessageMigrationV0())); + messageMigrator.initialize(); + return new AirbyteMessageVersionedMigratorFactory(messageMigrator); + } + } diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ReplicationJobOrchestrator.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ReplicationJobOrchestrator.java index c33c7ccabc80..a695ae705230 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ReplicationJobOrchestrator.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ReplicationJobOrchestrator.java @@ -12,7 +12,10 @@ import datadog.trace.api.Trace; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.protocol.AirbyteMessageSerDeProvider; +import io.airbyte.commons.protocol.AirbyteMessageVersionedMigratorFactory; import io.airbyte.commons.temporal.TemporalUtils; +import io.airbyte.commons.version.Version; import io.airbyte.config.Configs; import io.airbyte.config.ReplicationOutput; import io.airbyte.config.StandardSyncInput; @@ -30,10 +33,14 @@ import io.airbyte.workers.general.ReplicationWorker; import io.airbyte.workers.internal.AirbyteMessageTracker; import io.airbyte.workers.internal.AirbyteSource; +import io.airbyte.workers.internal.AirbyteStreamFactory; import io.airbyte.workers.internal.DefaultAirbyteDestination; import io.airbyte.workers.internal.DefaultAirbyteSource; +import io.airbyte.workers.internal.DefaultAirbyteStreamFactory; import io.airbyte.workers.internal.EmptyAirbyteSource; import io.airbyte.workers.internal.NamespacingMapper; +import io.airbyte.workers.internal.VersionedAirbyteMessageBufferedWriterFactory; +import io.airbyte.workers.internal.VersionedAirbyteStreamFactory; import io.airbyte.workers.process.AirbyteIntegrationLauncher; import io.airbyte.workers.process.IntegrationLauncher; import io.airbyte.workers.process.KubePodProcess; @@ -50,13 +57,19 @@ public class ReplicationJobOrchestrator implements JobOrchestrator runJob() throws Exception { final AirbyteSource airbyteSource = WorkerConstants.RESET_JOB_SOURCE_DOCKER_IMAGE_STUB.equals(sourceLauncherConfig.getDockerImage()) ? new EmptyAirbyteSource( featureFlags.useStreamCapableState()) - : new DefaultAirbyteSource(sourceLauncher); + : new DefaultAirbyteSource(sourceLauncher, getStreamFactory(sourceLauncherConfig.getProtocolVersion())); MetricClientFactory.initialize(MetricEmittingApps.WORKER); final MetricClient metricClient = MetricClientFactory.getMetricClient(); @@ -119,7 +132,8 @@ public Optional runJob() throws Exception { Math.toIntExact(jobRunConfig.getAttemptId()), airbyteSource, new NamespacingMapper(syncInput.getNamespaceDefinition(), syncInput.getNamespaceFormat(), syncInput.getPrefix()), - new DefaultAirbyteDestination(destinationLauncher), + new DefaultAirbyteDestination(destinationLauncher, getStreamFactory(destinationLauncherConfig.getProtocolVersion()), + new VersionedAirbyteMessageBufferedWriterFactory(serDeProvider, migratorFactory, destinationLauncherConfig.getProtocolVersion())), new AirbyteMessageTracker(), new RecordSchemaValidator(WorkerUtils.mapStreamNamesToSchemas(syncInput)), metricReporter); @@ -132,4 +146,10 @@ public Optional runJob() throws Exception { return Optional.of(Jsons.serialize(replicationOutput)); } + private AirbyteStreamFactory getStreamFactory(final Version protocolVersion) { + return protocolVersion != null + ? new VersionedAirbyteStreamFactory<>(serDeProvider, migratorFactory, protocolVersion) + : new DefaultAirbyteStreamFactory(); + } + } diff --git a/airbyte-cron/build.gradle b/airbyte-cron/build.gradle index 6ff678e4b087..47c1c939a2ed 100644 --- a/airbyte-cron/build.gradle +++ b/airbyte-cron/build.gradle @@ -6,8 +6,7 @@ dependencies { implementation 'com.auth0:java-jwt:3.19.2' implementation 'io.fabric8:kubernetes-client:5.12.2' implementation 'io.sentry:sentry:6.3.1' - implementation 'io.temporal:temporal-sdk:1.8.1' - implementation 'io.temporal:temporal-serviceclient:1.8.1' + implementation libs.bundles.temporal implementation libs.bundles.datadog implementation project(':airbyte-api') diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DatabaseDriver.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DatabaseDriver.java index 0bd19d2e196e..d8d4fcd1e91a 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DatabaseDriver.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DatabaseDriver.java @@ -18,7 +18,8 @@ public enum DatabaseDriver { ORACLE("oracle.jdbc.OracleDriver", "jdbc:oracle:thin:@%s:%d/%s"), POSTGRESQL("org.postgresql.Driver", "jdbc:postgresql://%s:%d/%s"), REDSHIFT("com.amazon.redshift.jdbc.Driver", "jdbc:redshift://%s:%d/%s"), - SNOWFLAKE("net.snowflake.client.jdbc.SnowflakeDriver", "jdbc:snowflake://%s/"); + SNOWFLAKE("net.snowflake.client.jdbc.SnowflakeDriver", "jdbc:snowflake://%s/"), + YUGABYTEDB("com.yugabyte.Driver", "jdbc:yugabytedb://%s:%d/%s"); private final String driverClassName; private final String urlFormatString; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BaseS3Destination.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BaseS3Destination.java index 075faba07235..0119ad41511f 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BaseS3Destination.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BaseS3Destination.java @@ -42,9 +42,8 @@ public AirbyteConnectionStatus check(final JsonNode config) { try { final S3DestinationConfig destinationConfig = configFactory.getS3DestinationConfig(config, storageProvider()); final AmazonS3 s3Client = destinationConfig.getS3Client(); - final S3StorageOperations storageOperations = new S3StorageOperations(nameTransformer, s3Client, destinationConfig); - S3BaseChecks.attemptS3WriteAndDelete(storageOperations, destinationConfig, destinationConfig.getBucketName()); + S3BaseChecks.testIAMUserHasListObjectPermission(s3Client, destinationConfig.getBucketName()); S3BaseChecks.testSingleUpload(s3Client, destinationConfig.getBucketName(), destinationConfig.getBucketPath()); S3BaseChecks.testMultipartUpload(s3Client, destinationConfig.getBucketName(), destinationConfig.getBucketPath()); diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3BaseChecks.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3BaseChecks.java index 1dc91a13ba8e..b5f2037e842b 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3BaseChecks.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3BaseChecks.java @@ -9,6 +9,7 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.ListObjectsRequest; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory; import java.io.IOException; import java.io.PrintWriter; @@ -37,10 +38,8 @@ public static void attemptS3WriteAndDelete(final S3StorageOperations storageOper public static void testSingleUpload(final AmazonS3 s3Client, final String bucketName, final String bucketPath) { LOGGER.info("Started testing if all required credentials assigned to user for single file uploading"); - if (bucketPath.endsWith("/")) { - throw new RuntimeException("Bucket Path should not end with /"); - } - final String testFile = bucketPath + "/" + "test_" + System.currentTimeMillis(); + final var prefix = bucketPath.endsWith("/") ? bucketPath : bucketPath + "/"; + final String testFile = prefix + "test_" + System.currentTimeMillis(); try { s3Client.putObject(bucketName, testFile, "this is a test file"); } finally { @@ -51,10 +50,8 @@ public static void testSingleUpload(final AmazonS3 s3Client, final String bucket public static void testMultipartUpload(final AmazonS3 s3Client, final String bucketName, final String bucketPath) throws IOException { LOGGER.info("Started testing if all required credentials assigned to user for multipart upload"); - if (bucketPath.endsWith("/")) { - throw new RuntimeException("Bucket Path should not end with /"); - } - final String testFile = bucketPath + "/" + "test_" + System.currentTimeMillis(); + final var prefix = bucketPath.endsWith("/") ? bucketPath : bucketPath + "/"; + final String testFile = prefix + "test_" + System.currentTimeMillis(); final StreamTransferManager manager = StreamTransferManagerFactory.create(bucketName, testFile, s3Client).get(); boolean success = false; try (final MultiPartOutputStream outputStream = manager.getMultiPartOutputStreams().get(0); @@ -96,18 +93,35 @@ static void attemptS3WriteAndDelete(final S3StorageOperations storageOperations, final S3DestinationConfig s3Config, final String bucketPath, final AmazonS3 s3) { - final var prefix = bucketPath.isEmpty() ? "" : bucketPath + (bucketPath.endsWith("/") ? "" : "/"); + final String prefix; + if (Strings.isNullOrEmpty(bucketPath)) { + prefix = ""; + } else if (bucketPath.endsWith("/")) { + prefix = bucketPath; + } else { + prefix = bucketPath + "/"; + } + final String outputTableName = prefix + "_airbyte_connection_test_" + UUID.randomUUID().toString().replaceAll("-", ""); attemptWriteAndDeleteS3Object(storageOperations, s3Config, outputTableName, s3); } + /** + * Runs some permissions checks: 1. Check whether the bucket exists; create it if not 2. Check + * whether s3://bucketName/bucketPath/ exists; create it (with empty contents) if not. (if + * bucketPath is null/empty-string, then skip this step) 3. Attempt to create and delete + * s3://bucketName/outputTableName 4. Attempt to list all objects in the bucket + */ private static void attemptWriteAndDeleteS3Object(final S3StorageOperations storageOperations, final S3DestinationConfig s3Config, final String outputTableName, final AmazonS3 s3) { final var s3Bucket = s3Config.getBucketName(); + final var bucketPath = s3Config.getBucketPath(); - storageOperations.createBucketObjectIfNotExists(s3Bucket); + if (!Strings.isNullOrEmpty(bucketPath)) { + storageOperations.createBucketObjectIfNotExists(bucketPath); + } s3.putObject(s3Bucket, outputTableName, "check-content"); testIAMUserHasListObjectPermission(s3, s3Bucket); s3.deleteObject(s3Bucket, outputTableName); diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java index a4d6370cc02d..599463435142 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java @@ -93,17 +93,23 @@ public String getBucketObjectPath(final String namespace, final String streamNam .replaceAll("/+", "/")); } + /** + * Create a directory object at the specified location. Creates the bucket if necessary. + * + * @param objectPath The directory to create. Must be a nonempty string. + */ @Override public void createBucketObjectIfNotExists(final String objectPath) { final String bucket = s3Config.getBucketName(); + final String folderPath = objectPath.endsWith("/") ? objectPath : objectPath + "/"; if (!doesBucketExist(bucket)) { LOGGER.info("Bucket {} does not exist; creating...", bucket); s3Client.createBucket(bucket); LOGGER.info("Bucket {} has been created.", bucket); } - if (!s3Client.doesObjectExist(bucket, objectPath)) { + if (!s3Client.doesObjectExist(bucket, folderPath)) { LOGGER.info("Storage Object {}/{} does not exist in bucket; creating...", bucket, objectPath); - s3Client.putObject(bucket, objectPath.endsWith("/") ? objectPath : objectPath + "/", ""); + s3Client.putObject(bucket, folderPath, ""); LOGGER.info("Storage Object {}/{} has been created in bucket.", bucket, objectPath); } } diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetWriter.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetWriter.java index 3104e3093e83..1a5bd5cc877d 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetWriter.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetWriter.java @@ -40,8 +40,10 @@ public class S3ParquetWriter extends BaseS3Writer implements DestinationFileWrit private final AvroRecordFactory avroRecordFactory; private final Schema schema; private final String outputFilename; + // object key = / private final String objectKey; - private final String gcsFileLocation; + // full file path = s3://// + private final String fullFilePath; public S3ParquetWriter(final S3DestinationConfig config, final AmazonS3 s3Client, @@ -61,14 +63,10 @@ public S3ParquetWriter(final S3DestinationConfig config, .build()); objectKey = String.join("/", outputPrefix, outputFilename); + fullFilePath = String.format("s3a://%s/%s", config.getBucketName(), objectKey); + LOGGER.info("Full S3 path for stream '{}': {}", stream.getName(), fullFilePath); - LOGGER.info("Full S3 path for stream '{}': s3://{}/{}", stream.getName(), config.getBucketName(), objectKey); - gcsFileLocation = String.format("s3a://%s/%s/%s", config.getBucketName(), outputPrefix, outputFilename); - - final URI uri = new URI( - String.format("s3a://%s/%s/%s", config.getBucketName(), outputPrefix, outputFilename)); - final Path path = new Path(uri); - + final Path path = new Path(new URI(fullFilePath)); final S3ParquetFormatConfig formatConfig = (S3ParquetFormatConfig) config.getFormatConfig(); final Configuration hadoopConfig = getHadoopConfig(config); this.parquetWriter = AvroParquetWriter.builder(HadoopOutputFile.fromPath(path, hadoopConfig)) @@ -137,7 +135,7 @@ public String getOutputPath() { @Override public String getFileLocation() { - return gcsFileLocation; + return fullFilePath; } @Override diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/S3BaseChecksTest.java b/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/S3BaseChecksTest.java new file mode 100644 index 000000000000..6f08faa683e3 --- /dev/null +++ b/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/S3BaseChecksTest.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.s3; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.startsWith; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import io.airbyte.integrations.destination.s3.util.S3NameTransformer; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentMatchers; + +public class S3BaseChecksTest { + + private AmazonS3 s3Client; + + @BeforeEach + public void setup() { + s3Client = mock(AmazonS3.class); + when(s3Client.doesObjectExist(anyString(), eq(""))).thenThrow(new IllegalArgumentException("Object path must not be empty")); + when(s3Client.putObject(anyString(), eq(""), anyString())).thenThrow(new IllegalArgumentException("Object path must not be empty")); + } + + @Test + public void attemptWriteAndDeleteS3Object_should_createSpecificFiles() { + S3DestinationConfig config = new S3DestinationConfig( + null, + "test_bucket", + "test/bucket/path", + null, + null, + null, + null, + s3Client); + S3StorageOperations operations = new S3StorageOperations(new S3NameTransformer(), s3Client, config); + when(s3Client.doesObjectExist("test_bucket", "test/bucket/path/")).thenReturn(false); + + S3BaseChecks.attemptS3WriteAndDelete(operations, config, "test/bucket/path"); + + verify(s3Client).putObject("test_bucket", "test/bucket/path/", ""); + verify(s3Client).putObject(eq("test_bucket"), startsWith("test/bucket/path/_airbyte_connection_test_"), anyString()); + verify(s3Client).listObjects(ArgumentMatchers.argThat(request -> "test_bucket".equals(request.getBucketName()))); + verify(s3Client).deleteObject(eq("test_bucket"), startsWith("test/bucket/path/_airbyte_connection_test_")); + } + + @Test + public void attemptWriteAndDeleteS3Object_should_skipDirectoryCreateIfRootPath() { + S3DestinationConfig config = new S3DestinationConfig( + null, + "test_bucket", + "", + null, + null, + null, + null, + s3Client); + S3StorageOperations operations = new S3StorageOperations(new S3NameTransformer(), s3Client, config); + + S3BaseChecks.attemptS3WriteAndDelete(operations, config, ""); + + verify(s3Client, never()).putObject("test_bucket", "", ""); + verify(s3Client).putObject(eq("test_bucket"), startsWith("_airbyte_connection_test_"), anyString()); + verify(s3Client).listObjects(ArgumentMatchers.argThat(request -> "test_bucket".equals(request.getBucketName()))); + verify(s3Client).deleteObject(eq("test_bucket"), startsWith("_airbyte_connection_test_")); + } + + @Test + public void attemptWriteAndDeleteS3Object_should_skipDirectoryCreateIfNullPath() { + S3DestinationConfig config = new S3DestinationConfig( + null, + "test_bucket", + null, + null, + null, + null, + null, + s3Client); + S3StorageOperations operations = new S3StorageOperations(new S3NameTransformer(), s3Client, config); + + S3BaseChecks.attemptS3WriteAndDelete(operations, config, null); + + verify(s3Client, never()).putObject("test_bucket", "", ""); + verify(s3Client).putObject(eq("test_bucket"), startsWith("_airbyte_connection_test_"), anyString()); + verify(s3Client).listObjects(ArgumentMatchers.argThat(request -> "test_bucket".equals(request.getBucketName()))); + verify(s3Client).deleteObject(eq("test_bucket"), startsWith("_airbyte_connection_test_")); + } + +} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshTunnel.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshTunnel.java index 0647f775ea34..261699be2168 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshTunnel.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshTunnel.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Preconditions; -import io.airbyte.commons.exceptions.ConnectionErrorException; import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; @@ -303,7 +302,7 @@ public void close() { * @see loadKeyPairs() */ - KeyPair getPrivateKeyPair() throws IOException, GeneralSecurityException, ConnectionErrorException { + KeyPair getPrivateKeyPair() throws IOException, GeneralSecurityException { final String validatedKey = validateKey(); final var keyPairs = SecurityUtils .getKeyPairResourceParser() @@ -312,7 +311,7 @@ KeyPair getPrivateKeyPair() throws IOException, GeneralSecurityException, Connec if (keyPairs != null && keyPairs.iterator().hasNext()) { return keyPairs.iterator().next(); } - throw new ConnectionErrorException("Unable to load private key pairs, verify key pairs are properly inputted"); + throw new RuntimeException("Unable to load private key pairs, verify key pairs are properly inputted"); } private String validateKey() { diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedDestination.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedDestination.java index 2e1735bdc18a..56b489d8eaca 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedDestination.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedDestination.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.commons.exceptions.ConnectionErrorException; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.integrations.base.AirbyteMessageConsumer; @@ -19,7 +18,6 @@ import io.airbyte.protocol.models.ConnectorSpecification; import java.util.List; import java.util.function.Consumer; -import org.apache.sshd.common.SshException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,7 +65,7 @@ public AirbyteConnectionStatus check(final JsonNode config) throws Exception { try { return (endPointKey != null) ? SshTunnel.sshWrap(config, endPointKey, delegate::check) : SshTunnel.sshWrap(config, hostKey, portKey, delegate::check); - } catch (final SshException | ConnectionErrorException e) { + } catch (final RuntimeException e) { final String sshErrorMessage = "Could not connect with provided SSH configuration. Error: " + e.getMessage(); AirbyteTraceMessageUtility.emitConfigErrorTrace(e, sshErrorMessage); return new AirbyteConnectionStatus() diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedSource.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedSource.java index eec5220391c5..1c955ab09f65 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedSource.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedSource.java @@ -5,7 +5,6 @@ package io.airbyte.integrations.base.ssh; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.exceptions.ConnectionErrorException; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; import io.airbyte.integrations.base.AirbyteTraceMessageUtility; @@ -17,7 +16,6 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; import java.util.List; -import org.apache.sshd.common.SshException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +41,7 @@ public ConnectorSpecification spec() throws Exception { public AirbyteConnectionStatus check(final JsonNode config) throws Exception { try { return SshTunnel.sshWrap(config, hostKey, portKey, delegate::check); - } catch (final SshException | ConnectionErrorException e) { + } catch (final RuntimeException e) { final String sshErrorMessage = "Could not connect with provided SSH configuration. Error: " + e.getMessage(); AirbyteTraceMessageUtility.emitConfigErrorTrace(e, sshErrorMessage); return new AirbyteConnectionStatus() diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/catalog_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/catalog_processor.py index 00901f2e6fca..2cbfe09394e4 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/catalog_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/catalog_processor.py @@ -9,7 +9,7 @@ from typing import Any, Dict, List, Set import yaml -from airbyte_cdk.models import DestinationSyncMode, SyncMode +from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode, SyncMode from normalization.destination_type import DestinationType from normalization.transform_catalog import dbt_macro from normalization.transform_catalog.destination_name_transformer import DestinationNameTransformer diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py index d0e40cf04dcb..b46d8e390c81 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py @@ -8,7 +8,7 @@ from enum import Enum from typing import Any, Dict, List, Optional, Tuple, Union -from airbyte_cdk.models import DestinationSyncMode, SyncMode +from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode, SyncMode from jinja2 import Template from normalization.destination_type import DestinationType from normalization.transform_catalog import dbt_macro diff --git a/airbyte-integrations/bases/source-acceptance-test/.coveragerc b/airbyte-integrations/bases/source-acceptance-test/.coveragerc index 11957a0e6499..2cda014ccf5e 100644 --- a/airbyte-integrations/bases/source-acceptance-test/.coveragerc +++ b/airbyte-integrations/bases/source-acceptance-test/.coveragerc @@ -1,8 +1,8 @@ [report] # show lines missing coverage show_missing = true -# coverage 64% measured on 62303a85def89450d2e46573a3d96cd326f2e921 (2022-08-09) +# coverage 74% measured on 4977ac2c527f03c15ce0094cfd48f6104a0fd82f (2022-10-26) # This number should probably never be adjusted down, only up i.e: we should only ever increase our test coverage -fail_under = 64 +fail_under = 74 skip_covered = true skip_empty = true diff --git a/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md b/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md index 389270e9a22c..c8b79bcb78d8 100644 --- a/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md +++ b/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 0.2.15 +Make `expect_records` mandatory in `high` `test_strictness_level`. [#18497](https://github.com/airbytehq/airbyte/pull/18497/). + +## 0.2.14 +Fail basic read in `high` `test_strictness_level` if no `bypass_reason` is set on empty_streams. [#18425](https://github.com/airbytehq/airbyte/pull/18425/). + +## 0.2.13 +Fail tests in `high` `test_strictness_level` if all tests are not configured. [#18414](https://github.com/airbytehq/airbyte/pull/18414/). + ## 0.2.12 Declare `bypass_reason` field in test configuration. [#18364](https://github.com/airbytehq/airbyte/pull/18364). diff --git a/airbyte-integrations/bases/source-acceptance-test/Dockerfile b/airbyte-integrations/bases/source-acceptance-test/Dockerfile index 21c9e87828cc..96225d466ae4 100644 --- a/airbyte-integrations/bases/source-acceptance-test/Dockerfile +++ b/airbyte-integrations/bases/source-acceptance-test/Dockerfile @@ -33,7 +33,7 @@ COPY pytest.ini setup.py ./ COPY source_acceptance_test ./source_acceptance_test RUN pip install . -LABEL io.airbyte.version=0.2.12 +LABEL io.airbyte.version=0.2.15 LABEL io.airbyte.name=airbyte/source-acceptance-test ENTRYPOINT ["python", "-m", "pytest", "-p", "source_acceptance_test.plugin", "-r", "fEsx"] diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/base.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/base.py index f29e537cb8ee..bb7b6488c7f0 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/base.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/base.py @@ -5,6 +5,7 @@ import inflection import pytest +from source_acceptance_test.config import Config @pytest.mark.usefixtures("inputs") @@ -16,3 +17,5 @@ def config_key(cls): if class_name.startswith("Test"): class_name = class_name[len("Test") :] return inflection.underscore(class_name) + + MANDATORY_FOR_TEST_STRICTNESS_LEVELS = [Config.TestStrictnessLevel.high] diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py index f73d81c0de12..5bd5307e4fe9 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py @@ -73,7 +73,8 @@ class ExpectedRecordsConfig(BaseModel): class Config: extra = "forbid" - path: Path = Field(description="File with expected records") + bypass_reason: Optional[str] = Field(description="Reason why this test is bypassed.") + path: Optional[Path] = Field(description="File with expected records") extra_fields: bool = Field(False, description="Allow records to have other fields") exact_order: bool = Field(False, description="Ensure that records produced in exact same order") extra_records: bool = Field( @@ -92,11 +93,29 @@ def validate_extra_records(cls, extra_records, values): raise ValueError("extra_records must be off if extra_fields enabled") return extra_records + @validator("path", always=True) + def no_bypass_reason_when_path_is_set(cls, path, values): + if path and values.get("bypass_reason"): + raise ValueError("You can't set a bypass_reason if a path is set") + if not path and not values.get("bypass_reason"): + raise ValueError("A path or a bypass_reason must be set") + return path + + +class EmptyStreamConfiguration(BaseConfig): + name: str + bypass_reason: Optional[str] = Field(default=None, description="Reason why this stream is considered empty.") + + def __hash__(self): # make it hashable + return hash((type(self),) + tuple(self.__dict__.values())) + class BasicReadTestConfig(BaseConfig): config_path: str = config_path configured_catalog_path: Optional[str] = configured_catalog_path - empty_streams: Set[str] = Field(default_factory=set, description="We validate that all streams has records. These are exceptions") + empty_streams: Set[EmptyStreamConfiguration] = Field( + default_factory=set, description="We validate that all streams has records. These are exceptions" + ) expect_records: Optional[ExpectedRecordsConfig] = Field(description="Expected records from the read") validate_schema: bool = Field(True, description="Ensure that records match the schema of the corresponding stream") # TODO: remove this field after https://github.com/airbytehq/airbyte/issues/8312 is done @@ -206,6 +225,11 @@ def migrate_legacy_to_current_config(legacy_config: dict) -> dict: migrated_config["acceptance_tests"] = {} for test_name, test_configs in legacy_config["tests"].items(): migrated_config["acceptance_tests"][test_name] = {"tests": test_configs} + for basic_read_tests in migrated_config["acceptance_tests"].get("basic_read", {}).get("tests", []): + if "empty_streams" in basic_read_tests: + basic_read_tests["empty_streams"] = [ + {"name": empty_stream_name} for empty_stream_name in basic_read_tests.get("empty_streams", []) + ] return migrated_config @root_validator(pre=True) diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py index ac0f79760c1d..72a00b883405 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py @@ -10,7 +10,7 @@ from logging import Logger from pathlib import Path from subprocess import STDOUT, check_output, run -from typing import Any, List, MutableMapping, Optional +from typing import Any, List, MutableMapping, Optional, Set import pytest from airbyte_cdk.models import ( @@ -24,7 +24,8 @@ ) from docker import errors from source_acceptance_test.base import BaseTest -from source_acceptance_test.config import Config +from source_acceptance_test.config import Config, EmptyStreamConfiguration +from source_acceptance_test.tests import TestBasicRead from source_acceptance_test.utils import ConnectorRunner, SecretDict, filter_output, load_config, load_yaml_or_json_path @@ -167,14 +168,62 @@ def pull_docker_image(acceptance_test_config) -> None: pytest.exit(f"Docker image `{image_name}` not found, please check your {config_filename} file", returncode=1) -@pytest.fixture(name="expected_records") -def expected_records_fixture(inputs, base_path) -> List[AirbyteRecordMessage]: - expect_records = getattr(inputs, "expect_records") - if not expect_records: - return [] - - with open(str(base_path / getattr(expect_records, "path"))) as f: - return [AirbyteRecordMessage.parse_raw(line) for line in f] +@pytest.fixture(name="empty_streams") +def empty_streams_fixture(inputs, test_strictness_level) -> Set[EmptyStreamConfiguration]: + empty_streams = getattr(inputs, "empty_streams", set()) + if test_strictness_level is Config.TestStrictnessLevel.high and empty_streams: + all_empty_streams_have_bypass_reasons = all([bool(empty_stream.bypass_reason) for empty_stream in inputs.empty_streams]) + if not all_empty_streams_have_bypass_reasons: + pytest.fail("A bypass_reason must be filled in for all empty streams when test_strictness_level is set to high.") + return empty_streams + + +@pytest.fixture(name="expected_records_by_stream") +def expected_records_by_stream_fixture( + test_strictness_level: Config.TestStrictnessLevel, + configured_catalog: ConfiguredAirbyteCatalog, + empty_streams: Set[EmptyStreamConfiguration], + inputs, + base_path, +) -> MutableMapping[str, List[MutableMapping]]: + def enforce_high_strictness_level_rules(expect_records_config, configured_catalog, empty_streams, records_by_stream) -> Optional[str]: + error_prefix = "High strictness level error: " + if expect_records_config is None: + pytest.fail(error_prefix + "expect_records must be configured for the basic_read test.") + elif expect_records_config.path: + not_seeded_streams = find_not_seeded_streams(configured_catalog, empty_streams, records_by_stream) + if not_seeded_streams: + pytest.fail( + error_prefix + + f"{', '.join(not_seeded_streams)} streams are declared in the catalog but do not have expected records. Please add expected records to {expect_records_config.path} or declare these streams in empty_streams." + ) + + expect_records_config = inputs.expect_records + + expected_records_by_stream = {} + if expect_records_config: + if expect_records_config.path: + expected_records_file_path = str(base_path / expect_records_config.path) + with open(expected_records_file_path, "r") as f: + all_records = [AirbyteRecordMessage.parse_raw(line) for line in f] + expected_records_by_stream = TestBasicRead.group_by_stream(all_records) + + if test_strictness_level is Config.TestStrictnessLevel.high: + enforce_high_strictness_level_rules(expect_records_config, configured_catalog, empty_streams, expected_records_by_stream) + return expected_records_by_stream + + +def find_not_seeded_streams( + configured_catalog: ConfiguredAirbyteCatalog, + empty_streams: Set[EmptyStreamConfiguration], + records_by_stream: MutableMapping[str, List[MutableMapping]], +) -> Set[str]: + stream_names_in_catalog = set([configured_stream.stream.name for configured_stream in configured_catalog.streams]) + empty_streams_names = set([stream.name for stream in empty_streams]) + expected_record_stream_names = set(records_by_stream.keys()) + expected_seeded_stream_names = stream_names_in_catalog - empty_streams_names + + return expected_seeded_stream_names - expected_record_stream_names @pytest.fixture(name="cached_schemas", scope="session") diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/plugin.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/plugin.py index 74c432caa01a..52bf61f0366a 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/plugin.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/plugin.py @@ -3,12 +3,16 @@ # +from enum import Enum from pathlib import Path -from typing import List +from typing import Callable, List, Tuple, Type import pytest from _pytest.config import Config from _pytest.config.argparsing import Parser +from source_acceptance_test.base import BaseTest +from source_acceptance_test.config import Config as AcceptanceTestConfig +from source_acceptance_test.config import GenericTestConfig from source_acceptance_test.utils import diff_dicts, load_config HERE = Path(__file__).parent.absolute() @@ -34,36 +38,75 @@ def pytest_addoption(parser): ) +class TestAction(Enum): + PARAMETRIZE = 1 + SKIP = 2 + FAIL = 3 + + def pytest_generate_tests(metafunc): """Hook function to customize test discovery and parametrization. - It does two things: - 1. skip test class if its name omitted in the config file (or it has no inputs defined) - 2. parametrize each test with inputs from config file. - - For example config file contains this: - tests: - test_suite1: - - input1: value1 - input2: value2 - - input1: value3 - input2: value4 - test_suite2: [] - - Hook function will skip test_suite2 and test_suite3, but parametrize test_suite1 with two sets of inputs. + It parametrizes, skips or fails a discovered test according the test configuration. """ if "inputs" in metafunc.fixturenames: - config_key = metafunc.cls.config_key() - test_name = f"{metafunc.cls.__name__}.{metafunc.function.__name__}" - config = load_config(metafunc.config.getoption("--acceptance-test-config")) - if not hasattr(config.acceptance_tests, config_key) or not getattr(config.acceptance_tests, config_key): - pytest.skip(f"Skipping {test_name} because not found in the config") + test_config_key = metafunc.cls.config_key() + global_config = load_config(metafunc.config.getoption("--acceptance-test-config")) + test_configuration: GenericTestConfig = getattr(global_config.acceptance_tests, test_config_key, None) + test_action, reason = parametrize_skip_or_fail( + metafunc.cls, metafunc.function, global_config.test_strictness_level, test_configuration + ) + + if test_action == TestAction.PARAMETRIZE: + metafunc.parametrize("inputs", test_configuration.tests) + if test_action == TestAction.SKIP: + pytest.skip(reason) + if test_action == TestAction.FAIL: + pytest.fail(reason) + + +def parametrize_skip_or_fail( + TestClass: Type[BaseTest], + test_function: Callable, + global_test_mode: AcceptanceTestConfig.TestStrictnessLevel, + test_configuration: GenericTestConfig, +) -> Tuple[TestAction, str]: + """Use the current test strictness level and test configuration to determine if the discovered test should be parametrized, skipped or failed. + We parametrize a test if: + - the configuration declares tests. + We skip a test if: + - the configuration does not declare tests and: + - the current test mode allows this test to be skipped. + - Or a bypass_reason is declared in the test configuration. + We fail a test if: + - the configuration does not declare the test but the discovered test is declared as mandatory for the current test strictness level. + Args: + TestClass (Type[BaseTest]): The discovered test class + test_function (Callable): The discovered test function + global_test_mode (AcceptanceTestConfig.TestStrictnessLevel): The global test strictness level (from the global configuration object) + test_configuration (GenericTestConfig): The current test configuration. + + Returns: + Tuple[TestAction, str]: The test action the execution should take and the reason why. + """ + test_name = f"{TestClass.__name__}.{test_function.__name__}" + test_mode_can_skip_this_test = global_test_mode not in TestClass.MANDATORY_FOR_TEST_STRICTNESS_LEVELS + skipping_reason_prefix = f"Skipping {test_name}: " + default_skipping_reason = skipping_reason_prefix + "not found in the config." + + if test_configuration is None: + if test_mode_can_skip_this_test: + return TestAction.SKIP, default_skipping_reason else: - test_inputs = getattr(config.acceptance_tests, config_key).tests - if not test_inputs: - pytest.skip(f"Skipping {test_name} because no inputs provided") - - metafunc.parametrize("inputs", test_inputs) + return ( + TestAction.FAIL, + f"{test_name} failed: it was not configured but must be according to the current {global_test_mode} test strictness level.", + ) + else: + if test_configuration.tests is not None: + return TestAction.PARAMETRIZE, f"Parametrize {test_name}: tests are configured." + else: + return TestAction.SKIP, skipping_reason_prefix + test_configuration.bypass_reason def pytest_collection_modifyitems(config, items): diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py index 3cdfc2987d7f..08b5f5717f4d 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py @@ -385,13 +385,14 @@ def _validate_empty_streams(self, records, configured_catalog, allowed_empty_str """ Only certain streams allowed to be empty """ + allowed_empty_stream_names = set([allowed_empty_stream.name for allowed_empty_stream in allowed_empty_streams]) counter = Counter(record.stream for record in records) all_streams = set(stream.stream.name for stream in configured_catalog.streams) streams_with_records = set(counter.keys()) streams_without_records = all_streams - streams_with_records - streams_without_records = streams_without_records - allowed_empty_streams + streams_without_records = streams_without_records - allowed_empty_stream_names assert not streams_without_records, f"All streams should return some records, streams without records: {streams_without_records}" def _validate_field_appears_at_least_once_in_stream(self, records: List, schema: Dict): @@ -434,14 +435,17 @@ def _validate_field_appears_at_least_once(self, records: List, configured_catalo assert not stream_name_to_empty_fields_mapping, msg def _validate_expected_records( - self, records: List[AirbyteRecordMessage], expected_records: List[AirbyteRecordMessage], flags, detailed_logger: Logger + self, + records: List[AirbyteRecordMessage], + expected_records_by_stream: MutableMapping[str, List[MutableMapping]], + flags, + detailed_logger: Logger, ): """ We expect some records from stream to match expected_records, partially or fully, in exact or any order. """ actual_by_stream = self.group_by_stream(records) - expected_by_stream = self.group_by_stream(expected_records) - for stream_name, expected in expected_by_stream.items(): + for stream_name, expected in expected_records_by_stream.items(): actual = actual_by_stream.get(stream_name, []) detailed_logger.info(f"Actual records for stream {stream_name}:") detailed_logger.log_json_list(actual) @@ -463,7 +467,7 @@ def test_read( connector_config, configured_catalog, inputs: BasicReadTestConfig, - expected_records: List[AirbyteRecordMessage], + expected_records_by_stream: MutableMapping[str, List[MutableMapping]], docker_runner: ConnectorRunner, detailed_logger, ): @@ -486,9 +490,12 @@ def test_read( if inputs.validate_data_points: self._validate_field_appears_at_least_once(records=records, configured_catalog=configured_catalog) - if expected_records: + if expected_records_by_stream: self._validate_expected_records( - records=records, expected_records=expected_records, flags=inputs.expect_records, detailed_logger=detailed_logger + records=records, + expected_records_by_stream=expected_records_by_stream, + flags=inputs.expect_records, + detailed_logger=detailed_logger, ) def test_airbyte_trace_message_on_failure(self, connector_config, inputs: BasicReadTestConfig, docker_runner: ConnectorRunner): diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_config.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_config.py index 938791090943..907db03ae8eb 100644 --- a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_config.py +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_config.py @@ -197,3 +197,18 @@ def test_config_parsing(self, raw_config, expected_output_config, expected_error def test_legacy_config_migration(self, legacy_config, expected_parsed_config): assert config.Config.is_legacy(legacy_config) assert config.Config.parse_obj(legacy_config) == expected_parsed_config + + +class TestExpectedRecordsConfig: + @pytest.mark.parametrize( + "path, bypass_reason, expectation", + [ + pytest.param("my_path", None, does_not_raise()), + pytest.param(None, "Good bypass reason", does_not_raise()), + pytest.param(None, None, pytest.raises(ValidationError)), + pytest.param("my_path", "Good bypass reason", pytest.raises(ValidationError)), + ], + ) + def test_bypass_reason_behavior(self, path, bypass_reason, expectation): + with expectation: + config.ExpectedRecordsConfig(path=path, bypass_reason=bypass_reason) diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_core.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_core.py index eec175467915..8082a6e02e27 100644 --- a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_core.py +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_core.py @@ -236,22 +236,30 @@ def test_additional_properties_is_true(discovered_catalog, expectation): @pytest.mark.parametrize( - "schema, record, should_fail", + "schema, record, expectation", [ - ({"type": "object"}, {"aa": 23}, False), - ({"type": "object"}, {}, False), - ({"type": "object", "properties": {"created": {"type": "string"}}}, {"aa": 23}, True), - ({"type": "object", "properties": {"created": {"type": "string"}}}, {"created": "23"}, False), - ({"type": "object", "properties": {"created": {"type": "string"}}}, {"root": {"created": "23"}}, True), + ({"type": "object"}, {"aa": 23}, does_not_raise()), + ({"type": "object"}, {}, does_not_raise()), + ( + {"type": "object", "properties": {"created": {"type": "string"}}}, + {"aa": 23}, + pytest.raises(AssertionError, match="should have some fields mentioned by json schema"), + ), + ({"type": "object", "properties": {"created": {"type": "string"}}}, {"created": "23"}, does_not_raise()), + ( + {"type": "object", "properties": {"created": {"type": "string"}}}, + {"root": {"created": "23"}}, + pytest.raises(AssertionError, match="should have some fields mentioned by json schema"), + ), # Recharge shop stream case ( {"type": "object", "properties": {"shop": {"type": ["null", "object"]}, "store": {"type": ["null", "object"]}}}, {"shop": {"a": "23"}, "store": {"b": "23"}}, - False, + does_not_raise(), ), ], ) -def test_read(schema, record, should_fail): +def test_read(schema, record, expectation): catalog = ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( @@ -267,10 +275,7 @@ def test_read(schema, record, should_fail): AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data=record, emitted_at=111)) ] t = _TestBasicRead() - if should_fail: - with pytest.raises(AssertionError, match="should have some fields mentioned by json schema"): - t.test_read(None, catalog, input_config, [], docker_runner_mock, MagicMock()) - else: + with expectation: t.test_read(None, catalog, input_config, [], docker_runner_mock, MagicMock()) diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_global_fixtures.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_global_fixtures.py new file mode 100644 index 000000000000..cdd8b006b3c6 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_global_fixtures.py @@ -0,0 +1,167 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import json + +import pytest +from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode +from source_acceptance_test import conftest +from source_acceptance_test.config import BasicReadTestConfig, Config, EmptyStreamConfiguration, ExpectedRecordsConfig + + +@pytest.mark.parametrize( + "test_strictness_level, basic_read_test_config, expect_test_failure", + [ + pytest.param( + Config.TestStrictnessLevel.low, + BasicReadTestConfig(config_path="config_path", empty_streams={EmptyStreamConfiguration(name="my_empty_stream")}), + False, + id="[LOW test strictness level] Empty streams can be declared without bypass_reason.", + ), + pytest.param( + Config.TestStrictnessLevel.low, + BasicReadTestConfig( + config_path="config_path", empty_streams={EmptyStreamConfiguration(name="my_empty_stream", bypass_reason="good reason")} + ), + False, + id="[LOW test strictness level] Empty streams can be declared with a bypass_reason.", + ), + pytest.param( + Config.TestStrictnessLevel.high, + BasicReadTestConfig(config_path="config_path", empty_streams={EmptyStreamConfiguration(name="my_empty_stream")}), + True, + id="[HIGH test strictness level] Empty streams can't be declared without bypass_reason.", + ), + pytest.param( + Config.TestStrictnessLevel.high, + BasicReadTestConfig( + config_path="config_path", empty_streams={EmptyStreamConfiguration(name="my_empty_stream", bypass_reason="good reason")} + ), + False, + id="[HIGH test strictness level] Empty streams can be declared with a bypass_reason.", + ), + ], +) +def test_empty_streams_fixture(mocker, test_strictness_level, basic_read_test_config, expect_test_failure): + mocker.patch.object(conftest.pytest, "fail") + # Pytest prevents fixture to be directly called. Using __wrapped__ allows us to call the actual function before it's been wrapped by the decorator. + assert conftest.empty_streams_fixture.__wrapped__(basic_read_test_config, test_strictness_level) == basic_read_test_config.empty_streams + if expect_test_failure: + conftest.pytest.fail.assert_called_once() + else: + conftest.pytest.fail.assert_not_called() + + +TEST_AIRBYTE_STREAM_A = AirbyteStream(name="test_stream_a", json_schema={"k": "v"}, supported_sync_modes=[SyncMode.full_refresh]) +TEST_AIRBYTE_STREAM_B = AirbyteStream(name="test_stream_b", json_schema={"k": "v"}, supported_sync_modes=[SyncMode.full_refresh]) +TEST_AIRBYTE_STREAM_C = AirbyteStream(name="test_stream_c", json_schema={"k": "v"}, supported_sync_modes=[SyncMode.full_refresh]) + +TEST_CONFIGURED_AIRBYTE_STREAM_A = ConfiguredAirbyteStream( + stream=TEST_AIRBYTE_STREAM_A, + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, +) + +TEST_CONFIGURED_AIRBYTE_STREAM_B = ConfiguredAirbyteStream( + stream=TEST_AIRBYTE_STREAM_B, + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, +) + +TEST_CONFIGURED_AIRBYTE_STREAM_C = ConfiguredAirbyteStream( + stream=TEST_AIRBYTE_STREAM_C, + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, +) + +TEST_CONFIGURED_CATALOG = ConfiguredAirbyteCatalog( + streams=[TEST_CONFIGURED_AIRBYTE_STREAM_A, TEST_CONFIGURED_AIRBYTE_STREAM_B, TEST_CONFIGURED_AIRBYTE_STREAM_C] +) + + +@pytest.mark.parametrize( + "test_strictness_level, configured_catalog, empty_streams, expected_records, expected_records_config, should_fail", + [ + pytest.param( + Config.TestStrictnessLevel.high, + TEST_CONFIGURED_CATALOG, + set(), + [], + None, + True, + id="High strictness level: No expected records configuration -> Failing", + ), + pytest.param( + Config.TestStrictnessLevel.high, + TEST_CONFIGURED_CATALOG, + {EmptyStreamConfiguration(name="test_stream_b"), EmptyStreamConfiguration(name="test_stream_c")}, + [{"stream": "test_stream_a", "data": {"k": "foo"}, "emitted_at": 1634387507000}], + ExpectedRecordsConfig(path="expected_records.json"), + False, + id="High strictness level: test_stream_b and test_stream_c are declared as empty streams, expected records only contains test_stream_a record -> Not failing", + ), + pytest.param( + Config.TestStrictnessLevel.high, + TEST_CONFIGURED_CATALOG, + set(), + [{"stream": "test_stream_a", "data": {"k": "foo"}, "emitted_at": 1634387507000}], + ExpectedRecordsConfig(path="expected_records.json"), + True, + id="High strictness level: test_stream_b and test_stream_c are not declared as empty streams, expected records only contains test_stream_a record -> Failing", + ), + pytest.param( + Config.TestStrictnessLevel.high, + TEST_CONFIGURED_CATALOG, + {EmptyStreamConfiguration(name="test_stream_b")}, + [{"stream": "test_stream_a", "data": {"k": "foo"}, "emitted_at": 1634387507000}], + ExpectedRecordsConfig(path="expected_records.json"), + True, + id="High strictness level: test_stream_b is declared as an empty stream, test_stream_c is not declared as empty streams, expected records only contains test_stream_a record -> Failing", + ), + pytest.param( + Config.TestStrictnessLevel.high, + TEST_CONFIGURED_CATALOG, + set(), + [], + ExpectedRecordsConfig(bypass_reason="A good reason to not have expected records"), + False, + id="High strictness level: Expected records configuration with bypass_reason -> Not failing", + ), + pytest.param( + Config.TestStrictnessLevel.low, + TEST_CONFIGURED_CATALOG, + set(), + [], + None, + False, + id="Low strictness level, no empty stream, no expected records -> Not failing", + ), + pytest.param( + Config.TestStrictnessLevel.low, + TEST_CONFIGURED_CATALOG, + set(), + [{"stream": "test_stream_a", "data": {"k": "foo"}, "emitted_at": 1634387507000}], + ExpectedRecordsConfig(path="expected_records.json"), + False, + id="Low strictness level, no empty stream, incomplete expected records -> Not failing", + ), + ], +) +def test_expected_records_by_stream_fixture( + tmp_path, mocker, test_strictness_level, configured_catalog, empty_streams, expected_records, expected_records_config, should_fail +): + mocker.patch.object(conftest.pytest, "fail") + + base_path = tmp_path + with open(f"{base_path}/expected_records.json", "w") as expected_records_file: + for record in expected_records: + expected_records_file.write(json.dumps(record) + "\n") + + inputs = BasicReadTestConfig(config_path="", empty_streams=empty_streams, expect_records=expected_records_config) + + conftest.expected_records_by_stream_fixture.__wrapped__(test_strictness_level, configured_catalog, empty_streams, inputs, base_path) + if should_fail: + conftest.pytest.fail.assert_called_once() + else: + conftest.pytest.fail.assert_not_called() diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_plugin.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_plugin.py new file mode 100644 index 000000000000..854cc782c64f --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_plugin.py @@ -0,0 +1,122 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import pytest +from source_acceptance_test import config, plugin + +HIGH_TEST_STRICTNESS_LEVEL = config.Config.TestStrictnessLevel.high +LOW_TEST_STRICTNESS_LEVEL = config.Config.TestStrictnessLevel.low + +PARAMETRIZE_ACTION = plugin.TestAction.PARAMETRIZE +SKIP_ACTION = plugin.TestAction.SKIP +FAIL_ACTION = plugin.TestAction.FAIL + + +class MyTestClass: + def dumb_test_function(self): + assert 2 > 1 + + +@pytest.mark.parametrize( + "parametrize_skip_or_fail_return", + [(PARAMETRIZE_ACTION, "parametrize reason"), (SKIP_ACTION, "skip reason"), (FAIL_ACTION, "fail_reason")], +) +def test_pytest_generate_tests(mocker, parametrize_skip_or_fail_return): + test_config = config.Config( + connector_image="foo", + acceptance_tests=config.AcceptanceTestConfigurations(spec=config.GenericTestConfig(tests=[config.SpecTestConfig()])), + ) + mocker.patch.object(plugin.pytest, "skip") + mocker.patch.object(plugin.pytest, "fail") + mocker.patch.object(plugin, "parametrize_skip_or_fail", mocker.Mock(return_value=parametrize_skip_or_fail_return)) + mocker.patch.object(plugin, "load_config", mocker.Mock(return_value=test_config)) + metafunc_mock = mocker.Mock( + fixturenames=["inputs"], + function=mocker.Mock(__name__="test_function"), + cls=mocker.Mock(config_key=mocker.Mock(return_value="spec"), __name__="MyTest"), + ) + plugin.pytest_generate_tests(metafunc_mock) + action, reason = parametrize_skip_or_fail_return + if action == PARAMETRIZE_ACTION: + metafunc_mock.parametrize.assert_called_once_with("inputs", test_config.acceptance_tests.spec.tests) + if action == SKIP_ACTION: + plugin.pytest.skip.assert_called_once_with(reason) + if action == FAIL_ACTION: + plugin.pytest.fail.assert_called_once_with(reason) + + +@pytest.mark.parametrize( + "TestClass, test_class_MANDATORY_FOR_TEST_STRICTNESS_LEVELS, global_test_mode, test_configuration, expected_action, expected_reason", + [ + pytest.param( + MyTestClass, + (HIGH_TEST_STRICTNESS_LEVEL), + HIGH_TEST_STRICTNESS_LEVEL, + None, + FAIL_ACTION, + "MyTestClass.dumb_test_function failed: it was not configured but must be according to the current high test strictness level.", + id="Discovered test is mandatory in high test strictness level, we're in high test strictness level, it was not configured: FAIL", + ), + pytest.param( + MyTestClass, + (HIGH_TEST_STRICTNESS_LEVEL), + LOW_TEST_STRICTNESS_LEVEL, + None, + SKIP_ACTION, + "Skipping MyTestClass.dumb_test_function: not found in the config.", + id="Discovered test is mandatory in high test strictness level, we are in low strictness level, it is not configured: SKIP", + ), + pytest.param( + MyTestClass, + set(), + HIGH_TEST_STRICTNESS_LEVEL, + None, + SKIP_ACTION, + "Skipping MyTestClass.dumb_test_function: not found in the config.", + id="Discovered test is not mandatory in any test strictness level, it was not configured: SKIP", + ), + pytest.param( + MyTestClass, + (HIGH_TEST_STRICTNESS_LEVEL), + HIGH_TEST_STRICTNESS_LEVEL, + config.GenericTestConfig(bypass_reason="A good reason."), + SKIP_ACTION, + "Skipping MyTestClass.dumb_test_function: A good reason.", + id="Discovered test is mandatory in high test strictness level, a bypass reason was provided: SKIP", + ), + pytest.param( + MyTestClass, + (HIGH_TEST_STRICTNESS_LEVEL), + LOW_TEST_STRICTNESS_LEVEL, + config.GenericTestConfig(bypass_reason="A good reason."), + SKIP_ACTION, + "Skipping MyTestClass.dumb_test_function: A good reason.", + id="Discovered test is mandatory in high test strictness level, we are in low test strictness level, a bypass reason was provided: SKIP (with bypass reason shown)", + ), + pytest.param( + MyTestClass, + (HIGH_TEST_STRICTNESS_LEVEL), + HIGH_TEST_STRICTNESS_LEVEL, + config.GenericTestConfig(tests=[config.SpecTestConfig()]), + PARAMETRIZE_ACTION, + "Parametrize MyTestClass.dumb_test_function: tests are configured.", + id="[High test strictness level] Discovered test is configured: PARAMETRIZE", + ), + pytest.param( + MyTestClass, + (HIGH_TEST_STRICTNESS_LEVEL), + LOW_TEST_STRICTNESS_LEVEL, + config.GenericTestConfig(tests=[config.SpecTestConfig()]), + PARAMETRIZE_ACTION, + "Parametrize MyTestClass.dumb_test_function: tests are configured.", + id="[Low test strictness level] Discovered test is configured: PARAMETRIZE", + ), + ], +) +def test_parametrize_skip_or_fail( + TestClass, test_class_MANDATORY_FOR_TEST_STRICTNESS_LEVELS, global_test_mode, test_configuration, expected_action, expected_reason +): + TestClass.MANDATORY_FOR_TEST_STRICTNESS_LEVELS = test_class_MANDATORY_FOR_TEST_STRICTNESS_LEVELS + test_action, reason = plugin.parametrize_skip_or_fail(TestClass, TestClass.dumb_test_function, global_test_mode, test_configuration) + assert (test_action, reason) == (expected_action, expected_reason) diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 28fc978af3bf..ab4c3417064d 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -14,6 +14,7 @@ | AppsFlyer | [![source-appsflyer-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-appsflyer-singer%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-appsflyer-singer) | | App Store | [![source-appstore-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-appstore-singer%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-appstore-singer) | | Asana | [![source-asana](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-asana%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-asana) | +| Ashby | [![source-ashby](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-ashby%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-ashby) | | AWS CloudTrail | [![source-aws-cloudtrail](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-aws-cloudtrail%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-aws-cloudtrail) | | Azure Table Storage | [![source-azure-table](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-azure-table%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-azure-table) | | BambooHR | [![source-bamboo-hr](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-bamboo-hr%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-bamboo-hr) | @@ -96,6 +97,7 @@ | Confluence | [![source-confluence](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-confluence%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-confluence) | | Qualaroo | [![source-qualaroo](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-qualaroo%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-qualaroo) | | QuickBooks | [![source-quickbooks-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-quickbooks-singer%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-quickbooks-singer) | +| RD Station Marketing | [![source-rd-station-marketing](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-rd-station-marketing%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-rd-station-marketing) | | Recharge | [![source-recharge](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-recharge%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-recharge) | | Recurly | [![source-recurly](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-recurly%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-recurly) | | Redshift | [![source-redshift](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-redshift%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-redshift) | @@ -104,6 +106,7 @@ | Salesloft | [![source-salesloft](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-salesloft%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-salesloft) | | Sendgrid | [![source-sendgrid](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-sendgrid%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-sendgrid) | | Sentry | [![source-sentry](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-sentry%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-sentry) | +| SFTP Bulk | [![source-sftp-bulk](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-sftp-bulk%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-sftp-bulk) | | Shopify | [![source-shopify](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-shopify%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-shopify) | | Slack | [![source-slack](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-slack%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-slack) | | Smartsheets | [![source-smartsheets](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-smartsheets%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-smartsheets) | @@ -118,6 +121,7 @@ | Twilio | [![source-twilio](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-twilio%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-twilio) | | Typeform | [![source-typeform](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-typeform%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-typeform) | | US Census | [![source-us-census](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-us-census%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-us-census) | +| Waiteraid | [![source-waiteraid]()](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-waiteraid%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-waiteraid) | | Whisky Hunter | [![source-whisky-hunter](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-whisky-hunter%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-whisky-hunter) | | Wrike | [![source-wrike](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-wrike%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-wrike) | | YouTube Analytics | [![source-youtube-analytics](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-youtube-analytics%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-youtube-analytics) | diff --git a/airbyte-integrations/connectors/destination-databricks/sample_secrets/staging_config.json b/airbyte-integrations/connectors/destination-databricks/sample_secrets/staging_config.json new file mode 100644 index 000000000000..82528e383eb3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-databricks/sample_secrets/staging_config.json @@ -0,0 +1,5 @@ +{ + "databricks_username": "", + "databricks_server_hostname": "abc-12345678-wxyz.cloud.databricks.com", + "databricks_personal_access_token": "dapi0123456789abcdefghij0123456789AB" +} diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksStagingLocationGetter.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksStagingLocationGetter.java new file mode 100644 index 000000000000..5eb2868588f2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksStagingLocationGetter.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.databricks; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Request personal staging locations from Databricks metastore. The equivalent curl command is: + * + * 
+ * curl --location --request POST \
+ *   'https:///api/2.1/unity-catalog/temporary-stage-credentials' \
+ *   --header 'Authorization: Bearer ' \
+ *   --form 'staging_url="stage://tmp//file.csv"' \
+ *   --form 'operation="PUT"' \
+ *   --form 'credential_type="PRESIGNED_URL"'
+ *
+ */ +public class DatabricksStagingLocationGetter { + + private static final Logger LOGGER = LoggerFactory.getLogger(DatabricksStagingLocationGetter.class); + private static final String PERSONAL_STAGING_REQUEST_URL = "https://%s/api/2.1/unity-catalog/temporary-stage-credentials"; + private static final String STAGING_URL = "stage://tmp/%s/%s"; + + private static final HttpClient httpClient = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_2) + .connectTimeout(Duration.ofSeconds(10)) + .build(); + + private static final Map staticRequestParams = Map.of( + "operation", "PUT", + "credential_type", "PRESIGNED_URL"); + + private static final String PARAM_STAGING_URL = "staging_url"; + private static final String RESPONSE_PRESIGNED_URL = "presigned_url"; + private static final String RESPONSE_URL = "url"; + private static final String RESPONSE_EXPIRATION = "expiration_time"; + + private final String username; + private final String serverHost; + private final String personalAccessToken; + + public DatabricksStagingLocationGetter(final String username, final String serverHost, final String personalAccessToken) { + this.username = username; + this.serverHost = serverHost; + this.personalAccessToken = personalAccessToken; + } + + /** + * @param filePath include path and filename: /. + * @return the pre-signed URL for the file in the personal staging location on the metastore. + */ + public PreSignedUrl getPreSignedUrl(final String filePath) throws IOException, InterruptedException { + final String stagingUrl = String.format(STAGING_URL, username, filePath); + LOGGER.info("Requesting Databricks personal staging location for {}", stagingUrl); + + final Map requestBody = new HashMap<>(staticRequestParams); + requestBody.put(PARAM_STAGING_URL, stagingUrl); + + final HttpRequest request = HttpRequest.newBuilder() + .POST(BodyPublishers.ofString(Jsons.serialize(requestBody))) + .uri(URI.create(String.format(PERSONAL_STAGING_REQUEST_URL, serverHost))) + .header("Authorization", "Bearer " + personalAccessToken) + .build(); + final HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + final JsonNode jsonResponse = Jsons.deserialize(response.body()); + if (jsonResponse.has(RESPONSE_PRESIGNED_URL) && jsonResponse.get(RESPONSE_PRESIGNED_URL).has(RESPONSE_URL) && jsonResponse.get( + RESPONSE_PRESIGNED_URL).has(RESPONSE_EXPIRATION)) { + return new PreSignedUrl( + jsonResponse.get(RESPONSE_PRESIGNED_URL).get(RESPONSE_URL).asText(), + jsonResponse.get(RESPONSE_PRESIGNED_URL).get(RESPONSE_EXPIRATION).asLong()); + } else { + final String message = String.format("Failed to get pre-signed URL for %s: %s", stagingUrl, jsonResponse); + LOGGER.error(message); + throw new RuntimeException(message); + } + + } + +} diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/PreSignedUrl.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/PreSignedUrl.java new file mode 100644 index 000000000000..f65e08301f83 --- /dev/null +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/PreSignedUrl.java @@ -0,0 +1,9 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.databricks; + +public record PreSignedUrl(String url, long expirationTimeMillis) { + +} diff --git a/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksStagingLocationGetterTest.java b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksStagingLocationGetterTest.java new file mode 100644 index 000000000000..7afd47493414 --- /dev/null +++ b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksStagingLocationGetterTest.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.databricks; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import java.io.IOException; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; + +public class DatabricksStagingLocationGetterTest { + + private static final String SECRETS_CONFIG_JSON = "secrets/staging_config.json"; + + @Test + public void testGetStagingLocation() throws IOException, InterruptedException { + final JsonNode config = Jsons.deserialize(IOs.readFile(Path.of(SECRETS_CONFIG_JSON))); + final DatabricksStagingLocationGetter stagingLocationGetter = new DatabricksStagingLocationGetter( + config.get("databricks_username").asText(), + config.get("databricks_server_hostname").asText(), + config.get("databricks_personal_access_token").asText()); + final PreSignedUrl preSignedUrl = stagingLocationGetter.getPreSignedUrl(System.currentTimeMillis() + "/test.csv"); + assertTrue(preSignedUrl.url().startsWith("https://")); + assertTrue(preSignedUrl.expirationTimeMillis() > 0); + } + +} diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/Dockerfile index 97bf1f52f693..a9e390d8feb9 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-elasticsearch-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/destination-elasticsearch-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle index ffc4c797814a..77da0535c523 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle @@ -33,6 +33,7 @@ dependencies { testImplementation libs.connectors.testcontainers.elasticsearch integrationTestJavaImplementation libs.connectors.testcontainers.elasticsearch + integrationTestJavaImplementation project(':airbyte-commons-worker') integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-elasticsearch') } diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestination.java b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestination.java index 44a7bb080e32..e2341d77f982 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestination.java +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestination.java @@ -4,12 +4,19 @@ package io.airbyte.integrations.destination.elasticsearch; +import static co.elastic.clients.elasticsearch.watcher.Input.HTTP; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.spec_modification.SpecModifyingDestination; +import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.ConnectorSpecification; +import java.net.URL; +import java.util.Objects; import java.util.stream.IntStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,6 +24,9 @@ public class ElasticsearchStrictEncryptDestination extends SpecModifyingDestination implements Destination { private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchStrictEncryptDestination.class); + private final ObjectMapper mapper = new ObjectMapper(); + private static final String NON_EMPTY_URL_ERR_MSG = "Server Endpoint is a required field"; + private static final String NON_SECURE_URL_ERR_MSG = "Server Endpoint requires HTTPS"; public ElasticsearchStrictEncryptDestination() { super(new ElasticsearchDestination()); @@ -38,4 +48,27 @@ public ConnectorSpecification modifySpec(ConnectorSpecification originalSpec) th return spec; } + @Override + public AirbyteConnectionStatus check(JsonNode config) throws Exception { + + final ConnectorConfiguration configObject = convertConfig(config); + if (Objects.isNull(configObject.getEndpoint())) { + return new AirbyteConnectionStatus() + .withStatus(AirbyteConnectionStatus.Status.FAILED) + .withMessage(NON_EMPTY_URL_ERR_MSG); + } + + if (new URL(configObject.getEndpoint()).getProtocol().equals(HTTP)) { + return new AirbyteConnectionStatus() + .withStatus(AirbyteConnectionStatus.Status.FAILED) + .withMessage(NON_SECURE_URL_ERR_MSG); + } + + return super.check(config); + } + + private ConnectorConfiguration convertConfig(JsonNode config) { + return mapper.convertValue(config, ConnectorConfiguration.class); + } + } diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java index c1e0ec6940bc..8bb93434fe4a 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java @@ -4,10 +4,13 @@ package io.airbyte.integrations.destination.elasticsearch; +import static org.junit.Assert.assertEquals; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.config.StandardCheckConnectionOutput.Status; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; @@ -18,6 +21,7 @@ import java.util.Map; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.testcontainers.elasticsearch.ElasticsearchContainer; public class ElasticsearchStrictEncryptDestinationAcceptanceTest extends DestinationAcceptanceTest { @@ -88,21 +92,29 @@ protected TestDataComparator getTestDataComparator() { @Override protected JsonNode getConfig() { - - final JsonNode authConfig = Jsons.jsonNode(Map.of( - "method", "basic", - "username", "elastic", - "password", "s3cret")); - return Jsons.jsonNode(ImmutableMap.builder() .put("endpoint", String.format("https://%s:%s", container.getHost(), container.getMappedPort(9200))) - .put("authenticationMethod", authConfig) + .put("authenticationMethod", getAuthConfig()) .put("ca_certificate", new String(container.copyFileFromContainer( "/usr/share/elasticsearch/config/certs/http_ca.crt", InputStream::readAllBytes), StandardCharsets.UTF_8)) .build()); } + protected JsonNode getUnsecureConfig() { + return Jsons.jsonNode(ImmutableMap.builder() + .put("endpoint", String.format("http://%s:%s", container.getHost(), container.getMappedPort(9200))) + .put("authenticationMethod", getAuthConfig()) + .build()); + } + + protected JsonNode getAuthConfig() { + return Jsons.jsonNode(Map.of( + "method", "basic", + "username", "elastic", + "password", "s3cret")); + } + @Override protected JsonNode getFailCheckConfig() { // should result in a failed connection check @@ -135,4 +147,9 @@ protected void tearDown(DestinationAcceptanceTest.TestDestinationEnv testEnv) { connection.allIndices().forEach(connection::deleteIndexIfPresent); } + @Test + public void testCheckConnectionInvalidHttpProtocol() throws Exception { + assertEquals(Status.FAILED, runCheck(getUnsecureConfig()).getStatus()); + } + } diff --git a/airbyte-integrations/connectors/destination-elasticsearch/Dockerfile b/airbyte-integrations/connectors/destination-elasticsearch/Dockerfile index f78e88860504..876c4adb2fdd 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/Dockerfile +++ b/airbyte-integrations/connectors/destination-elasticsearch/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-elasticsearch COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/destination-elasticsearch diff --git a/airbyte-integrations/connectors/destination-meilisearch/.dockerignore b/airbyte-integrations/connectors/destination-meilisearch/.dockerignore index 65c7d0ad3e73..6d35a84f68b8 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/.dockerignore +++ b/airbyte-integrations/connectors/destination-meilisearch/.dockerignore @@ -1,3 +1,5 @@ * !Dockerfile -!build +!main.py +!destination_meilisearch +!setup.py diff --git a/airbyte-integrations/connectors/destination-meilisearch/Dockerfile b/airbyte-integrations/connectors/destination-meilisearch/Dockerfile index b664503d7ba5..f573460c64a2 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/Dockerfile +++ b/airbyte-integrations/connectors/destination-meilisearch/Dockerfile @@ -1,20 +1,38 @@ -FROM airbyte/integration-base-java:dev AS build +FROM python:3.9.11-alpine3.15 as base -WORKDIR /airbyte +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code -ENV APPLICATION destination-meilisearch +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . -FROM airbyte/integration-base-java:dev +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code -WORKDIR /airbyte +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone -ENV APPLICATION destination-meilisearch +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash -COPY --from=build /airbyte /airbyte +# copy payload code only +COPY main.py ./ +COPY destination_meilisearch ./destination_meilisearch -LABEL io.airbyte.version=0.2.13 +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/destination-meilisearch diff --git a/airbyte-integrations/connectors/destination-meilisearch/README.md b/airbyte-integrations/connectors/destination-meilisearch/README.md index bbe121c3b0d3..560eabcc7a75 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/README.md +++ b/airbyte-integrations/connectors/destination-meilisearch/README.md @@ -1,32 +1,123 @@ -# MeiliSearch Destination +# Meilisearch Destination -This is the repository for the MeiliSearch destination connector, written in Java. +This is the repository for the Meilisearch destination connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/destinations/meilisearch). ## Local development ### Prerequisites **To iterate on this connector, make sure to complete this prerequisites section.** -#### Build & Activate Virtual Environment -First, build the module by running the following from the `airbyte` project root directory: +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +From the Airbyte repository root, run: ``` ./gradlew :airbyte-integrations:connectors:destination-meilisearch:build ``` #### Create credentials -If you are running MeiliSearch locally you may not need an api key at all. If there is an API key set for MeiliSearch, you can find instruction on how to find it in the [MeiliSearch docs](https://docs.meilisearch.com/reference/features/authentication.html#master-key). +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/meilisearch) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_meilisearch/spec.json` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, the integration tests do not require any external credentials. MeiliSearch is run from a test container. +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination meilisearch test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` ### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/destination-meilisearch:dev +``` + +You can also build the connector image via Gradle: ``` -# in airbyte root directory ./gradlew :airbyte-integrations:connectors:destination-meilisearch:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` docker run --rm airbyte/destination-meilisearch:dev spec -docker run --rm -v $(pwd)/airbyte-integrations/connectors/destination-meilisearch/secrets:/secrets airbyte/destination-meilisearch:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/airbyte-integrations/connectors/destination-meilisearch/secrets:/secrets airbyte/destination-meilisearch:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/airbyte-integrations/connectors/destination-meilisearch/secrets:/secrets -v $(pwd)/airbyte-integrations/connectors/destination-meilisearch/sample_files:/sample_files airbyte/destination-meilisearch:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-meilisearch:dev check --config /secrets/config.json +# messages.jsonl is a file containing line-separated JSON representing AirbyteMessages +cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-meilisearch:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests ``` ### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:destination-meilisearch:integrationTest` to run the standard integration test suite. +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Coming soon: + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:destination-meilisearch:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:destination-meilisearch:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/destination-meilisearch/build.gradle b/airbyte-integrations/connectors/destination-meilisearch/build.gradle index 9290baeddb9e..7849c8cdc050 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/build.gradle +++ b/airbyte-integrations/connectors/destination-meilisearch/build.gradle @@ -1,26 +1,8 @@ plugins { - id 'application' + id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-integration-test-java' } -application { - mainClass = 'io.airbyte.integrations.destination.meilisearch.MeiliSearchDestination' - // Needed for JDK17 - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0','--add-opens', 'java.base/java.lang=ALL-UNNAMED'] -} - -dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') - - implementation 'com.meilisearch.sdk:meilisearch-java:0.6.0' - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-meilisearch') - - integrationTestJavaImplementation libs.connectors.testcontainers - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) +airbytePython { + moduleDirectory 'destination_meilisearch' } diff --git a/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/__init__.py b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/__init__.py new file mode 100644 index 000000000000..bae2aaf851c0 --- /dev/null +++ b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .destination import DestinationMeilisearch + +__all__ = ["DestinationMeilisearch"] diff --git a/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/destination.py b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/destination.py new file mode 100644 index 000000000000..4b66b427f693 --- /dev/null +++ b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/destination.py @@ -0,0 +1,69 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from logging import Logger +from typing import Any, Iterable, Mapping + +from airbyte_cdk.destinations import Destination +from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type +from destination_meilisearch.writer import MeiliWriter +from meilisearch import Client + + +def get_client(config: Mapping[str, Any]) -> Client: + host = config.get("host") + api_key = config.get("api_key") + return Client(host, api_key) + + +class DestinationMeilisearch(Destination): + primary_key = "_ab_pk" + + def write( + self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] + ) -> Iterable[AirbyteMessage]: + client = get_client(config=config) + + for configured_stream in configured_catalog.streams: + steam_name = configured_stream.stream.name + if configured_stream.destination_sync_mode == DestinationSyncMode.overwrite: + client.delete_index(steam_name) + client.create_index(steam_name, {"primaryKey": self.primary_key}) + + writer = MeiliWriter(client, steam_name, self.primary_key) + for message in input_messages: + if message.type == Type.STATE: + writer.flush() + yield message + elif message.type == Type.RECORD: + writer.queue_write_operation(message.record.data) + else: + continue + writer.flush() + + def check(self, logger: Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: + try: + client = get_client(config=config) + + create_index_job = client.create_index("_airbyte", {"primaryKey": "id"}) + client.wait_for_task(create_index_job["taskUid"]) + + add_documents_job = client.index("_airbyte").add_documents( + [ + { + "id": 287947, + "title": "Shazam", + "overview": "A boy is given the ability", + } + ] + ) + client.wait_for_task(add_documents_job.task_uid) + + client.index("_airbyte").search("Shazam") + client.delete_index("_airbyte") + return AirbyteConnectionStatus(status=Status.SUCCEEDED) + except Exception as e: + logger.error(f"Check connection failed. Error: {e}") + return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {repr(e)}") diff --git a/airbyte-integrations/connectors/destination-meilisearch/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/spec.json similarity index 91% rename from airbyte-integrations/connectors/destination-meilisearch/src/main/resources/spec.json rename to airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/spec.json index d6afe8e712eb..19ab616ed1e3 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/spec.json @@ -1,15 +1,15 @@ { "documentationUrl": "https://docs.airbyte.com/integrations/destinations/meilisearch", + "supported_destination_sync_modes": ["overwrite", "append"], "supportsIncremental": true, - "supportsNormalization": false, "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"], + "supportsNormalization": false, "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MeiliSearch Destination Spec", + "title": "Destination Meilisearch", "type": "object", "required": ["host"], - "additionalProperties": true, + "additionalProperties": false, "properties": { "host": { "title": "Host", diff --git a/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/writer.py b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/writer.py new file mode 100644 index 000000000000..75e6b0b8bc37 --- /dev/null +++ b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/writer.py @@ -0,0 +1,36 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from collections.abc import Mapping +from logging import getLogger +from uuid import uuid4 + +from meilisearch import Client + +logger = getLogger("airbyte") + + +class MeiliWriter: + write_buffer = [] + flush_interval = 50000 + + def __init__(self, client: Client, steam_name: str, primary_key: str): + self.client = client + self.steam_name = steam_name + self.primary_key = primary_key + + def queue_write_operation(self, data: Mapping): + random_key = str(uuid4()) + self.write_buffer.append({**data, self.primary_key: random_key}) + if len(self.write_buffer) == self.flush_interval: + self.flush() + + def flush(self): + buffer_size = len(self.write_buffer) + if buffer_size == 0: + return + logger.info(f"flushing {buffer_size} records") + response = self.client.index(self.steam_name).add_documents(self.write_buffer) + self.client.wait_for_task(response.task_uid, 1800000, 1000) + self.write_buffer.clear() diff --git a/airbyte-integrations/connectors/destination-meilisearch/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-meilisearch/integration_tests/integration_test.py new file mode 100644 index 000000000000..86a8590a272c --- /dev/null +++ b/airbyte-integrations/connectors/destination-meilisearch/integration_tests/integration_test.py @@ -0,0 +1,102 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import json +import logging +import time +from typing import Any, Dict, Mapping + +import pytest +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + Status, + SyncMode, + Type, +) +from destination_meilisearch.destination import DestinationMeilisearch, get_client +from meilisearch import Client + + +@pytest.fixture(name="config") +def config_fixture() -> Mapping[str, Any]: + with open("secrets/config.json", "r") as f: + return json.loads(f.read()) + + +@pytest.fixture(name="configured_catalog") +def configured_catalog_fixture() -> ConfiguredAirbyteCatalog: + stream_schema = {"type": "object", "properties": {"string_col": {"type": "str"}, "int_col": {"type": "integer"}}} + + overwrite_stream = ConfiguredAirbyteStream( + stream=AirbyteStream( + name="_airbyte", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] + ), + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + + return ConfiguredAirbyteCatalog(streams=[overwrite_stream]) + + +@pytest.fixture(autouse=True) +def teardown(config: Mapping): + yield + client = get_client(config=config) + client.delete_index("_airbyte") + + +@pytest.fixture(name="client") +def client_fixture(config) -> Client: + client = get_client(config=config) + resp = client.create_index("_airbyte", {"primaryKey": "_ab_pk"}) + while True: + time.sleep(0.2) + task = client.get_task(resp["taskUid"]) + status = task["status"] + if status == "succeeded" or status == "failed": + break + return client + + +def test_check_valid_config(config: Mapping): + outcome = DestinationMeilisearch().check(logging.getLogger("airbyte"), config) + assert outcome.status == Status.SUCCEEDED + + +def test_check_invalid_config(): + outcome = DestinationMeilisearch().check( + logging.getLogger("airbyte"), {"api_key": "not_a_real_key", "host": "https://www.meilisearch.com"} + ) + assert outcome.status == Status.FAILED + + +def _state(data: Dict[str, Any]) -> AirbyteMessage: + return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=data)) + + +def _record(stream: str, str_value: str, int_value: int) -> AirbyteMessage: + return AirbyteMessage( + type=Type.RECORD, record=AirbyteRecordMessage(stream=stream, data={"str_col": str_value, "int_col": int_value}, emitted_at=0) + ) + + +def records_count(client: Client) -> int: + documents_results = client.index("_airbyte").get_documents() + return documents_results.total + + +def test_write(config: Mapping, configured_catalog: ConfiguredAirbyteCatalog, client: Client): + overwrite_stream = configured_catalog.streams[0].stream.name + first_state_message = _state({"state": "1"}) + first_record_chunk = [_record(overwrite_stream, str(i), i) for i in range(2)] + + destination = DestinationMeilisearch() + list(destination.write(config, configured_catalog, [*first_record_chunk, first_state_message])) + assert records_count(client) == 2 diff --git a/airbyte-integrations/connectors/destination-meilisearch/integration_tests/messages.jsonl b/airbyte-integrations/connectors/destination-meilisearch/integration_tests/messages.jsonl new file mode 100644 index 000000000000..e1d0682f9dad --- /dev/null +++ b/airbyte-integrations/connectors/destination-meilisearch/integration_tests/messages.jsonl @@ -0,0 +1,2 @@ +{"type": "RECORD", "record": {"stream": "ab-airbyte-testing", "data": {"_ab_pk": "my_value", "column2": 221, "column3": "2021-01-01T20:10:22", "column4": 1.214, "column5": [1,2,3]}, "emitted_at": 1626172757000}} +{"type": "RECORD", "record": {"stream": "ab-airbyte-testing", "data": {"_ab_pk": "my_value2", "column2": 222, "column3": "2021-01-02T22:10:22", "column5": [1,2,null]}, "emitted_at": 1626172757000}} diff --git a/airbyte-integrations/connectors/destination-meilisearch/main.py b/airbyte-integrations/connectors/destination-meilisearch/main.py new file mode 100644 index 000000000000..0a1d749e75b0 --- /dev/null +++ b/airbyte-integrations/connectors/destination-meilisearch/main.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from destination_meilisearch import DestinationMeilisearch + +if __name__ == "__main__": + DestinationMeilisearch().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-meilisearch/requirements.txt b/airbyte-integrations/connectors/destination-meilisearch/requirements.txt new file mode 100644 index 000000000000..d6e1198b1ab1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-meilisearch/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/airbyte-integrations/connectors/destination-meilisearch/sample_files/configured_catalog.json b/airbyte-integrations/connectors/destination-meilisearch/sample_files/configured_catalog.json new file mode 100644 index 000000000000..9ac002e358d3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-meilisearch/sample_files/configured_catalog.json @@ -0,0 +1,27 @@ +{ + "streams": [ + { + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "stream": { + "name": "ab-airbyte-testing", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "body": { + "type": "string" + }, + "attributes": { + "type": ["null", "object"] + } + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/destination-meilisearch/setup.py b/airbyte-integrations/connectors/destination-meilisearch/setup.py new file mode 100644 index 000000000000..f73276833626 --- /dev/null +++ b/airbyte-integrations/connectors/destination-meilisearch/setup.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = ["airbyte-cdk", "meilisearch>=0.22.0"] + +TEST_REQUIREMENTS = ["pytest~=6.1"] + +setup( + name="destination_meilisearch", + description="Destination implementation for Meilisearch.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/destination-meilisearch/src/main/java/io/airbyte/integrations/destination/meilisearch/MeiliSearchDestination.java b/airbyte-integrations/connectors/destination-meilisearch/src/main/java/io/airbyte/integrations/destination/meilisearch/MeiliSearchDestination.java deleted file mode 100644 index ba9caf250ac9..000000000000 --- a/airbyte-integrations/connectors/destination-meilisearch/src/main/java/io/airbyte/integrations/destination/meilisearch/MeiliSearchDestination.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.meilisearch; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.meilisearch.sdk.Client; -import com.meilisearch.sdk.Config; -import com.meilisearch.sdk.Index; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.text.Names; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer; -import io.airbyte.integrations.destination.buffered_stream_consumer.RecordWriter; -import io.airbyte.integrations.destination.record_buffer.InMemoryRecordBufferingStrategy; -import io.airbyte.protocol.models.AirbyteConnectionStatus; -import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; -import io.airbyte.protocol.models.AirbyteMessage; -import io.airbyte.protocol.models.AirbyteRecordMessage; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.DestinationSyncMode; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; -import java.util.function.Consumer; -import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - *

- * Since this is not a relational database, it therefore makes some slightly different choices. The - * main difference that we need to reckon with is that this destination does not work without a - * primary key for each stream. That primary key needs to be defined ahead of time. Only records for - * which that primary key is present can be uploaded. There are also some rules around the allowed - * formats of these primary keys. - *

- *

- * The strategy is to inject an extra airbyte primary key field in each record. The value of that - * field is a randomly generate UUID. This means that we have no ability to ever overwrite - * individual records that we put in MeiliSearch. - *

- *

- * Index names can only contain alphanumeric values, so we normalize stream names to meet these - * constraints. This is why streamName and indexName are treated separately in this connector. - *

- *

- * This destination can support full refresh and incremental. It does NOT support normalization. It - * breaks from the paradigm of having a "raw" and "normalized" table. There is no DBT for - * MeiliSearch so we write the data a single time in a way that makes it most likely to work well - * within MeiliSearch. - *

- */ -public class MeiliSearchDestination extends BaseConnector implements Destination { - - private static final Logger LOGGER = LoggerFactory.getLogger(MeiliSearchDestination.class); - - private static final int MAX_BATCH_SIZE_BYTES = 1024 * 1024 * 1024 / 4; // 256mib - private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSSSSSSSS"); - - public static final String AB_PK_COLUMN = "_ab_pk"; - public static final String AB_EMITTED_AT_COLUMN = "_ab_emitted_at"; - - @Override - public AirbyteConnectionStatus check(final JsonNode config) { - try { - LOGGER.info("config in check {}", config); - // create a fake index and add a record to it to make sure we can connect and have write access. - final Client client = getClient(config); - final Index index = client.index("_airbyte"); - index.addDocuments("[{\"id\": \"_airbyte\" }]"); - index.search("_airbyte"); - client.deleteIndex(index.getUid()); - return new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); - } catch (final Exception e) { - LOGGER.error("Check connection failed.", e); - return new AirbyteConnectionStatus().withStatus(Status.FAILED).withMessage("Check connection failed: " + e.getMessage()); - } - } - - @Override - public AirbyteMessageConsumer getConsumer(final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final Consumer outputRecordCollector) - throws Exception { - final Client client = getClient(config); - final Map indexNameToIndex = createIndices(catalog, client); - - return new BufferedStreamConsumer( - outputRecordCollector, - () -> LOGGER.info("Starting write to MeiliSearch."), - new InMemoryRecordBufferingStrategy(recordWriterFunction(indexNameToIndex), MAX_BATCH_SIZE_BYTES), - (hasFailed) -> LOGGER.info("Completed writing to MeiliSearch. Status: {}", hasFailed ? "FAILED" : "SUCCEEDED"), - catalog, - (data) -> true); - } - - private static Map createIndices(final ConfiguredAirbyteCatalog catalog, final Client client) throws Exception { - final Map map = new HashMap<>(); - for (final ConfiguredAirbyteStream stream : catalog.getStreams()) { - final String indexName = getIndexName(stream); - final DestinationSyncMode syncMode = stream.getDestinationSyncMode(); - if (syncMode == null) { - throw new IllegalStateException("Undefined destination sync mode"); - } - if (syncMode == DestinationSyncMode.OVERWRITE && indexExists(client, indexName)) { - client.deleteIndex(indexName); - } - - final Index index = client.getOrCreateIndex(indexName, AB_PK_COLUMN); - map.put(indexName, index); - } - return map; - } - - private static boolean indexExists(final Client client, final String indexName) throws Exception { - return Arrays.stream(client.getIndexList()) - .map(Index::getUid) - .anyMatch(actualIndexName -> actualIndexName.equals(indexName)); - } - - private static RecordWriter recordWriterFunction(final Map indexNameToWriteConfig) { - return (namePair, records) -> { - final String resolvedIndexName = getIndexName(namePair.getName()); - if (!indexNameToWriteConfig.containsKey(resolvedIndexName)) { - throw new IllegalArgumentException( - String.format("Message contained record from a stream that was not in the catalog. \nexpected streams: %s", - indexNameToWriteConfig.keySet())); - } - - final Index index = indexNameToWriteConfig.get(resolvedIndexName); - - // Only writes the data, not the full AirbyteRecordMessage. This is different from how database - // destinations work. There is not really a viable way to "transform" data after it is MeiliSearch. - // Tools like DBT do not apply. Therefore, we need to try to write data in the most usable format - // possible that does not require alteration. - final String json = Jsons.serialize(records - .stream() - .map(AirbyteRecordMessage::getData) - .peek(o -> ((ObjectNode) o).put(AB_PK_COLUMN, Names.toAlphanumericAndUnderscore(UUID.randomUUID().toString()))) - .peek(o -> ((ObjectNode) o).put(AB_EMITTED_AT_COLUMN, LocalDateTime.now().format(FORMATTER))) - .collect(Collectors.toList())); - final String s = index.addDocuments(json); - LOGGER.info("add docs response {}", s); - LOGGER.info("waiting for update to be applied started {}", Instant.now()); - try { - index.waitForPendingUpdate(Jsons.deserialize(s).get("updateId").asInt()); - } catch (final Exception e) { - LOGGER.error("waiting for update to be applied failed.", e); - LOGGER.error("printing MeiliSearch update statuses: {}", Arrays.asList(index.getUpdates())); - throw e; - } - LOGGER.info("waiting for update to be applied completed {}", Instant.now()); - }; - } - - private static String getIndexName(final String streamName) { - return Names.toAlphanumericAndUnderscore(streamName); - } - - private static String getIndexName(final ConfiguredAirbyteStream stream) { - return getIndexName(stream.getStream().getName()); - } - - static Client getClient(final JsonNode config) { - return new Client(new Config(config.get("host").asText(), config.has("api_key") ? config.get("api_key").asText() : null)); - } - - public static void main(final String[] args) throws Exception { - final Destination destination = new MeiliSearchDestination(); - LOGGER.info("starting destination: {}", MeiliSearchDestination.class); - new IntegrationRunner(destination).run(args); - LOGGER.info("completed destination: {}", MeiliSearchDestination.class); - } - -} diff --git a/airbyte-integrations/connectors/destination-meilisearch/src/test-integration/java/io/airbyte/integrations/destination/meilisearch/MeiliSearchDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-meilisearch/src/test-integration/java/io/airbyte/integrations/destination/meilisearch/MeiliSearchDestinationAcceptanceTest.java deleted file mode 100644 index b0226b6ccde8..000000000000 --- a/airbyte-integrations/connectors/destination-meilisearch/src/test-integration/java/io/airbyte/integrations/destination/meilisearch/MeiliSearchDestinationAcceptanceTest.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.meilisearch; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.meilisearch.sdk.Client; -import com.meilisearch.sdk.Index; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.stream.MoreStreams; -import io.airbyte.commons.text.Names; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Comparator; -import java.util.List; -import java.util.stream.Collectors; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.utility.DockerImageName; - -public class MeiliSearchDestinationAcceptanceTest extends DestinationAcceptanceTest { - - private static final Integer DEFAULT_MEILI_SEARCH_PORT = 7700; - private static final Integer EXPOSED_PORT = 7701; - - private static final String API_KEY = "masterKey"; - - private Client meiliSearchClient; - private GenericContainer genericContainer; - private JsonNode config; - - @Override - protected void setup(final TestDestinationEnv testEnv) throws IOException { - final Path meiliSearchDataDir = Files.createTempDirectory(Path.of("/tmp"), "meilisearch-integration-test"); - meiliSearchDataDir.toFile().deleteOnExit(); - - genericContainer = new GenericContainer<>(DockerImageName.parse("getmeili/meilisearch:v0.24.0")) - .withFileSystemBind(meiliSearchDataDir.toString(), "/data.ms"); - genericContainer.setPortBindings(ImmutableList.of(EXPOSED_PORT + ":" + DEFAULT_MEILI_SEARCH_PORT)); - genericContainer.start(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", String.format("http://%s:%s", genericContainer.getHost(), EXPOSED_PORT)) - .put("api_key", API_KEY) - .build()); - meiliSearchClient = MeiliSearchDestination.getClient(config); - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) { - genericContainer.stop(); - } - - @Override - protected String getImageName() { - return "airbyte/destination-meilisearch:dev"; - } - - @Override - protected JsonNode getConfig() { - return config; - } - - @Override - protected JsonNode getFailCheckConfig() { - final JsonNode invalidConfig = Jsons.clone(getConfig()); - ((ObjectNode) invalidConfig).put("host", "localhost:7702"); - return invalidConfig; - } - - @Override - protected TestDataComparator getTestDataComparator() { - return new AdvancedTestDataComparator(); - } - - @Override - protected boolean supportBasicDataTypeTest() { - return true; - } - - @Override - protected boolean supportArrayDataTypeTest() { - return true; - } - - @Override - protected boolean supportObjectDataTypeTest() { - return true; - } - - @Override - protected List retrieveRecords(final TestDestinationEnv env, - final String streamName, - final String namespace, - final JsonNode streamSchema) - throws Exception { - final Index index = meiliSearchClient.index(Names.toAlphanumericAndUnderscore(streamName)); - final String responseString = index.getDocuments(); - final JsonNode response = Jsons.deserialize(responseString); - return MoreStreams.toStream(response.iterator()) - // strip out the airbyte primary key because the test cases only expect the data, no the airbyte - // metadata column. - // We also sort the data by "emitted_at" and then remove that column, because the test cases only - // expect data, - // not the airbyte metadata column. - .peek(r -> ((ObjectNode) r).remove(MeiliSearchDestination.AB_PK_COLUMN)) - .sorted(Comparator.comparing(o -> o.get(MeiliSearchDestination.AB_EMITTED_AT_COLUMN).asText())) - .peek(r -> ((ObjectNode) r).remove(MeiliSearchDestination.AB_EMITTED_AT_COLUMN)) - .collect(Collectors.toList()); - } - -} diff --git a/airbyte-integrations/connectors/destination-meilisearch/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-meilisearch/unit_tests/unit_test.py new file mode 100644 index 000000000000..1d7e5e23c54c --- /dev/null +++ b/airbyte-integrations/connectors/destination-meilisearch/unit_tests/unit_test.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import patch + +from destination_meilisearch.writer import MeiliWriter + + +@patch("meilisearch.Client") +def test_queue_write_operation(client): + writer = MeiliWriter(client, "steam_name", "primary_key") + writer.queue_write_operation({"a": "a"}) + assert len(writer.write_buffer) == 1 + + +@patch("meilisearch.Client") +def test_flush(client): + writer = MeiliWriter(client, "steam_name", "primary_key") + writer.queue_write_operation({"a": "a"}) + writer.flush() + client.index.assert_called_once_with("steam_name") + client.wait_for_task.assert_called_once() diff --git a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile index 6bffbe8f7041..6b4ecd23bc7b 100644 --- a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mongodb-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/destination-mongodb-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.destination.mongodb/MongodbDestinationStrictEncrypt.java b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.destination.mongodb/MongodbDestinationStrictEncrypt.java index fdf208f48c57..2cebbbd508bd 100644 --- a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.destination.mongodb/MongodbDestinationStrictEncrypt.java +++ b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.destination.mongodb/MongodbDestinationStrictEncrypt.java @@ -18,11 +18,11 @@ public class MongodbDestinationStrictEncrypt extends SpecModifyingDestination im private static final Logger LOGGER = LoggerFactory.getLogger(MongodbDestinationStrictEncrypt.class); public MongodbDestinationStrictEncrypt() { - super(new MongodbDestination()); + super(MongodbDestination.sshWrappedDestination()); } @Override - public ConnectorSpecification modifySpec(ConnectorSpecification originalSpec) throws Exception { + public ConnectorSpecification modifySpec(final ConnectorSpecification originalSpec) throws Exception { final ConnectorSpecification spec = Jsons.clone(originalSpec); // removing tls property for a standalone instance to disable possibility to switch off a tls // connection @@ -30,7 +30,7 @@ public ConnectorSpecification modifySpec(ConnectorSpecification originalSpec) th return spec; } - public static void main(String[] args) throws Exception { + public static void main(final String[] args) throws Exception { final Destination destination = new MongodbDestinationStrictEncrypt(); LOGGER.info("starting destination: {}", MongodbDestinationStrictEncrypt.class); new IntegrationRunner(destination).run(args); diff --git a/airbyte-integrations/connectors/destination-mongodb/Dockerfile b/airbyte-integrations/connectors/destination-mongodb/Dockerfile index 00cb6114919d..cff1b88e848c 100644 --- a/airbyte-integrations/connectors/destination-mongodb/Dockerfile +++ b/airbyte-integrations/connectors/destination-mongodb/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mongodb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/destination-mongodb diff --git a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbDestination.java b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbDestination.java index 88ac858e0507..0a46d2700d90 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbDestination.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbDestination.java @@ -25,6 +25,7 @@ import io.airbyte.integrations.base.AirbyteTraceMessageUtility; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.integrations.base.ssh.SshWrappedDestination; import io.airbyte.integrations.destination.mongodb.exception.MongodbDatabaseException; import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteMessage; @@ -61,12 +62,16 @@ public class MongodbDestination extends BaseConnector implements Destination { private final MongodbNameTransformer namingResolver; + public static Destination sshWrappedDestination() { + return new SshWrappedDestination(new MongodbDestination(), JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY); + } + public MongodbDestination() { namingResolver = new MongodbNameTransformer(); } public static void main(final String[] args) throws Exception { - final Destination destination = new MongodbDestination(); + final Destination destination = sshWrappedDestination(); LOGGER.info("starting destination: {}", MongodbDestination.class); new IntegrationRunner(destination).run(args); LOGGER.info("completed destination: {}", MongodbDestination.class); diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java index e5642a090ed8..20e5d5f9fee5 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java @@ -19,6 +19,7 @@ import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.protocol.models.AirbyteConnectionStatus; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import org.bson.Document; @@ -27,14 +28,14 @@ public class MongodbDestinationAcceptanceTest extends DestinationAcceptanceTest { - private static final String DOCKER_IMAGE_NAME = "mongo:4.0.10"; - private static final String DATABASE_NAME = "admin"; + protected static final String DOCKER_IMAGE_NAME = "mongo:4.0.10"; + protected static final String DATABASE_NAME = "admin"; private static final String DATABASE_FAIL_NAME = "fail_db"; - private static final String AUTH_TYPE = "auth_type"; - private static final String AIRBYTE_DATA = "_airbyte_data"; + protected static final String AUTH_TYPE = "auth_type"; + protected static final String AIRBYTE_DATA = "_airbyte_data"; private MongoDBContainer container; - private final MongodbNameTransformer namingResolver = new MongodbNameTransformer(); + protected final MongodbNameTransformer namingResolver = new MongodbNameTransformer(); @Override protected String getImageName() { @@ -42,7 +43,7 @@ protected String getImageName() { } @Override - protected JsonNode getConfig() { + protected JsonNode getConfig() throws Exception { return Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, container.getHost()) .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) @@ -52,7 +53,7 @@ protected JsonNode getConfig() { } @Override - protected JsonNode getFailCheckConfig() { + protected JsonNode getFailCheckConfig() throws Exception { return Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, container.getHost()) .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) @@ -110,54 +111,72 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, */ @Test void testCheckIncorrectPasswordFailure() { - final JsonNode invalidConfig = getFailCheckConfig(); - ((ObjectNode) invalidConfig).put(JdbcUtils.DATABASE_KEY, DATABASE_NAME); - ((ObjectNode) invalidConfig.get(AUTH_TYPE)).put(JdbcUtils.PASSWORD_KEY, "fake"); - final MongodbDestination destination = new MongodbDestination(); - final AirbyteConnectionStatus status = destination.check(invalidConfig); - assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 18")); + try { + final JsonNode invalidConfig = getFailCheckConfig(); + ((ObjectNode) invalidConfig).put(JdbcUtils.DATABASE_KEY, DATABASE_NAME); + ((ObjectNode) invalidConfig.get(AUTH_TYPE)).put(JdbcUtils.PASSWORD_KEY, "fake"); + final MongodbDestination destination = new MongodbDestination(); + final AirbyteConnectionStatus status = destination.check(invalidConfig); + assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); + } catch (final Exception e) { + assertTrue(e instanceof IOException); + } } @Test public void testCheckIncorrectUsernameFailure() { - final JsonNode invalidConfig = getFailCheckConfig(); - ((ObjectNode) invalidConfig).put(JdbcUtils.DATABASE_KEY, DATABASE_NAME); - ((ObjectNode) invalidConfig.get(AUTH_TYPE)).put(JdbcUtils.USERNAME_KEY, "fakeusername"); - final MongodbDestination destination = new MongodbDestination(); - final AirbyteConnectionStatus status = destination.check(invalidConfig); - assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 18")); + try { + final JsonNode invalidConfig = getFailCheckConfig(); + ((ObjectNode) invalidConfig).put(JdbcUtils.DATABASE_KEY, DATABASE_NAME); + ((ObjectNode) invalidConfig.get(AUTH_TYPE)).put(JdbcUtils.USERNAME_KEY, "fakeusername"); + final MongodbDestination destination = new MongodbDestination(); + final AirbyteConnectionStatus status = destination.check(invalidConfig); + assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); + } catch (final Exception e) { + assertTrue(e instanceof IOException); + } + } @Test public void testCheckIncorrectDataBaseFailure() { - final JsonNode invalidConfig = getFailCheckConfig(); - ((ObjectNode) invalidConfig).put(JdbcUtils.DATABASE_KEY, DATABASE_FAIL_NAME); - final MongodbDestination destination = new MongodbDestination(); - final AirbyteConnectionStatus status = destination.check(invalidConfig); - assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 18")); + try { + final JsonNode invalidConfig = getFailCheckConfig(); + ((ObjectNode) invalidConfig).put(JdbcUtils.DATABASE_KEY, DATABASE_FAIL_NAME); + final MongodbDestination destination = new MongodbDestination(); + final AirbyteConnectionStatus status = destination.check(invalidConfig); + assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); + } catch (final Exception e) { + assertTrue(e instanceof IOException); + } + } @Test public void testCheckIncorrectHost() { - final JsonNode invalidConfig = getConfig(); - ((ObjectNode) invalidConfig).put(JdbcUtils.HOST_KEY, "localhost2"); - final MongodbDestination destination = new MongodbDestination(); - final AirbyteConnectionStatus status = destination.check(invalidConfig); - assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: -3")); + try { + final JsonNode invalidConfig = getConfig(); + ((ObjectNode) invalidConfig).put(JdbcUtils.HOST_KEY, "localhost2"); + final MongodbDestination destination = new MongodbDestination(); + final AirbyteConnectionStatus status = destination.check(invalidConfig); + assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); + } catch (final Exception e) { + assertTrue(e instanceof IOException); + } + } @Test public void testCheckIncorrectPort() { - final JsonNode invalidConfig = getConfig(); - ((ObjectNode) invalidConfig).put(JdbcUtils.PORT_KEY, 1234); - final MongodbDestination destination = new MongodbDestination(); - final AirbyteConnectionStatus status = destination.check(invalidConfig); - assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: -3")); + try { + final JsonNode invalidConfig = getConfig(); + ((ObjectNode) invalidConfig).put(JdbcUtils.PORT_KEY, 1234); + final MongodbDestination destination = new MongodbDestination(); + final AirbyteConnectionStatus status = destination.check(invalidConfig); + assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); + } catch (final Exception e) { + assertTrue(e instanceof IOException); + } } @Override @@ -174,13 +193,13 @@ protected void tearDown(final TestDestinationEnv testEnv) { /* Helpers */ - private JsonNode getAuthTypeConfig() { + protected JsonNode getAuthTypeConfig() { return Jsons.deserialize("{\n" + " \"authorization\": \"none\"\n" + "}"); } - private MongoDatabase getMongoDatabase(final String host, final int port, final String databaseName) { + protected MongoDatabase getMongoDatabase(final String host, final int port, final String databaseName) { try { final String connectionString = String.format("mongodb://%s:%s/", host, port); return new MongoDatabase(connectionString, databaseName); diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshKeyMongoDbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshKeyMongoDbDestinationAcceptanceTest.java new file mode 100644 index 000000000000..9a9e5d9889ce --- /dev/null +++ b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshKeyMongoDbDestinationAcceptanceTest.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.mongodb; + +import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.integrations.base.ssh.SshTunnel.TunnelMethod; + +public class SshKeyMongoDbDestinationAcceptanceTest extends SshMongoDbDestinationAcceptanceTest { + + @Override + public SshTunnel.TunnelMethod getTunnelMethod() { + return TunnelMethod.SSH_KEY_AUTH; + } +} diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshMongoDbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshMongoDbDestinationAcceptanceTest.java new file mode 100644 index 000000000000..3acae92257cc --- /dev/null +++ b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshMongoDbDestinationAcceptanceTest.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.mongodb; + +import static com.mongodb.client.model.Projections.excludeId; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import com.mongodb.client.MongoCursor; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.db.mongodb.MongoDatabase; +import io.airbyte.integrations.base.ssh.SshBastionContainer; +import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.integrations.util.HostPortResolver; +import java.util.ArrayList; +import java.util.List; +import org.bson.Document; +import org.testcontainers.containers.MongoDBContainer; +import org.testcontainers.containers.Network; + +public abstract class SshMongoDbDestinationAcceptanceTest extends MongodbDestinationAcceptanceTest { + + private static final Network network = Network.newNetwork(); + private static final SshBastionContainer bastion = new SshBastionContainer(); + private static MongoDBContainer container; + private static final int DEFAULT_PORT = 27017; + + public abstract SshTunnel.TunnelMethod getTunnelMethod(); + + @Override + protected void setup(final TestDestinationEnv testEnv) { + container = new MongoDBContainer(DOCKER_IMAGE_NAME) + .withNetwork(network) + .withExposedPorts(DEFAULT_PORT); + container.start(); + bastion.initAndStartBastion(network); + } + + @Override + protected JsonNode getConfig() throws Exception { + return bastion.getTunnelConfig(getTunnelMethod(), ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveIpAddress(container)) + .put(JdbcUtils.PORT_KEY, container.getExposedPorts().get(0)) + .put(JdbcUtils.DATABASE_KEY, DATABASE_NAME) + .put(AUTH_TYPE, getAuthTypeConfig())); + } + + @Override + protected JsonNode getFailCheckConfig() throws Exception { + // should result in a failed connection check + return bastion.getTunnelConfig(getTunnelMethod(), ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveIpAddress(container)) + .put(JdbcUtils.PORT_KEY, container.getExposedPorts().get(0)) + .put(JdbcUtils.DATABASE_KEY, DATABASE_NAME) + .put(AUTH_TYPE, Jsons.jsonNode(ImmutableMap.builder() + .put("authorization", "login/password") + .put(JdbcUtils.USERNAME_KEY, "user") + .put(JdbcUtils.PASSWORD_KEY, "invalid_pass") + .build()))); + } + + @Override + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) { + final MongoDatabase database = getMongoDatabase(HostPortResolver.resolveIpAddress(container), + container.getExposedPorts().get(0), DATABASE_NAME); + final var collection = database.getOrCreateNewCollection(namingResolver.getRawTableName(streamName)); + final List result = new ArrayList<>(); + try (final MongoCursor cursor = collection.find().projection(excludeId()).iterator()) { + while (cursor.hasNext()) { + result.add(Jsons.jsonNode(cursor.next().get(AIRBYTE_DATA))); + } + } + return result; + } + + + @Override + protected void tearDown(final TestDestinationEnv testEnv) { + container.stop(); + container.close(); + bastion.getContainer().stop(); + bastion.getContainer().close(); + } + +} diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshPasswordMongoDbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshPasswordMongoDbDestinationAcceptanceTest.java new file mode 100644 index 000000000000..379afd47200b --- /dev/null +++ b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshPasswordMongoDbDestinationAcceptanceTest.java @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.mongodb; + +import io.airbyte.integrations.base.ssh.SshTunnel; + +public class SshPasswordMongoDbDestinationAcceptanceTest extends SshMongoDbDestinationAcceptanceTest { + + @Override + public SshTunnel.TunnelMethod getTunnelMethod() { + return SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH; + } +} diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile index adbab7cd3ef6..cc49956af5b6 100644 --- a/airbyte-integrations/connectors/destination-redshift/Dockerfile +++ b/airbyte-integrations/connectors/destination-redshift/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.50 +LABEL io.airbyte.version=0.3.51 LABEL io.airbyte.name=airbyte/destination-redshift diff --git a/airbyte-integrations/connectors/destination-s3/Dockerfile b/airbyte-integrations/connectors/destination-s3/Dockerfile index 96b7b1227ee0..eb28aa69ba80 100644 --- a/airbyte-integrations/connectors/destination-s3/Dockerfile +++ b/airbyte-integrations/connectors/destination-s3/Dockerfile @@ -22,8 +22,8 @@ RUN /bin/bash -c 'set -e && \ yum install lzop lzo lzo-dev -y; \ elif [ "$ARCH" == "aarch64" ] || [ "$ARCH" = "arm64" ]; then \ echo "$ARCH" && \ - yum group install -y "Development Tools" \ - yum install lzop lzo lzo-dev wget curl unzip zip maven git -y; \ + yum group install -y "Development Tools"; \ + yum install lzop lzo lzo-dev wget curl unzip zip maven git which -y; \ wget http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz -P /tmp; \ cd /tmp && tar xvfz lzo-2.10.tar.gz; \ cd /tmp/lzo-2.10/ && ./configure --enable-shared --prefix /usr/local/lzo-2.10; \ @@ -40,5 +40,5 @@ RUN /bin/bash -c 'set -e && \ echo "unknown arch" ;\ fi' -LABEL io.airbyte.version=0.3.16 +LABEL io.airbyte.version=0.3.17 LABEL io.airbyte.name=airbyte/destination-s3 diff --git a/airbyte-integrations/connectors/destination-typesense/.dockerignore b/airbyte-integrations/connectors/destination-typesense/.dockerignore new file mode 100644 index 000000000000..d72007ad7e34 --- /dev/null +++ b/airbyte-integrations/connectors/destination-typesense/.dockerignore @@ -0,0 +1,5 @@ +* +!Dockerfile +!main.py +!destination_typesense +!setup.py diff --git a/airbyte-integrations/connectors/source-zoom-singer/Dockerfile b/airbyte-integrations/connectors/destination-typesense/Dockerfile similarity index 79% rename from airbyte-integrations/connectors/source-zoom-singer/Dockerfile rename to airbyte-integrations/connectors/destination-typesense/Dockerfile index 3b56601b682c..0f5659af70c9 100644 --- a/airbyte-integrations/connectors/source-zoom-singer/Dockerfile +++ b/airbyte-integrations/connectors/destination-typesense/Dockerfile @@ -7,9 +7,7 @@ WORKDIR /airbyte/integration_code # upgrade pip to the latest version RUN apk --no-cache upgrade \ && pip install --upgrade pip \ - && apk --no-cache add tzdata \ - && apk --no-cache add git \ - && apk --no-cache add build-base + && apk --no-cache add tzdata build-base COPY setup.py ./ @@ -31,10 +29,10 @@ RUN apk --no-cache add bash # copy payload code only COPY main.py ./ -COPY source_zoom_singer ./source_zoom_singer +COPY destination_typesense ./destination_typesense ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.4 -LABEL io.airbyte.name=airbyte/source-zoom-singer +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/destination-typesense diff --git a/airbyte-integrations/connectors/source-zoom-singer/README.md b/airbyte-integrations/connectors/destination-typesense/README.md similarity index 54% rename from airbyte-integrations/connectors/source-zoom-singer/README.md rename to airbyte-integrations/connectors/destination-typesense/README.md index 0177fc2c5f6c..01c677ffcf05 100644 --- a/airbyte-integrations/connectors/source-zoom-singer/README.md +++ b/airbyte-integrations/connectors/destination-typesense/README.md @@ -1,7 +1,7 @@ -# Source Zoom Singer +# Typesense Destination -This is the repository for the Zoom source connector, based on a Singer tap. -For information about how to use this connector within Airbyte, see [the User Documentation](https://docs.airbyte.io/integrations/sources/zoom). +This is the repository for the Typesense destination connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/destinations/typesense). ## Local development @@ -32,44 +32,24 @@ should work as you expect. #### Building via Gradle From the Airbyte repository root, run: ``` -./gradlew :airbyte-integrations:connectors:source-zoom:build +./gradlew :airbyte-integrations:connectors:destination-typesense:build ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/zoom) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zoom_singer/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/typesense) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_typesense/spec.json` file. Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. +See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source zoom test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination typesense test creds` and place them into `secrets/config.json`. ### Locally running the connector ``` -python main_dev.py spec -python main_dev.py check --config secrets/config.json -python main_dev.py discover --config secrets/config.json -python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector root run: -``` -pytest unit_tests -``` - -### Locally running the connector -``` -python main_dev.py spec -python main_dev.py check --config secrets/config.json -python main_dev.py discover --config secrets/config.json -python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` ### Locally running the connector docker image @@ -77,12 +57,12 @@ python -m pytest unit_tests #### Build First, make sure you build the latest Docker image: ``` -docker build . -t airbyte/source-zoom-singer:dev +docker build . -t airbyte/destination-typesense:dev ``` You can also build the connector image via Gradle: ``` -./gradlew :airbyte-integrations:connectors:source-zoom:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-typesense:airbyteDocker ``` When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in the Dockerfile. @@ -90,24 +70,54 @@ the Dockerfile. #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-zoom-singer:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zoom-singer:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zoom-singer:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-zoom-singer:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm airbyte/destination-typesense:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-typesense:dev check --config /secrets/config.json +# messages.jsonl is a file containing line-separated JSON representing AirbyteMessages +cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-typesense:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests ``` ### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-zoom-singer:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, create a directory `integration_tests` which contain your tests and run them with `pytest integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Coming soon: + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:destination-typesense:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:destination-typesense:integrationTest +``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/destination-typesense/build.gradle b/airbyte-integrations/connectors/destination-typesense/build.gradle new file mode 100644 index 000000000000..01ad66a130f7 --- /dev/null +++ b/airbyte-integrations/connectors/destination-typesense/build.gradle @@ -0,0 +1,8 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' +} + +airbytePython { + moduleDirectory 'destination_typesense' +} diff --git a/airbyte-integrations/connectors/destination-typesense/destination_typesense/__init__.py b/airbyte-integrations/connectors/destination-typesense/destination_typesense/__init__.py new file mode 100644 index 000000000000..cc3d48181b6a --- /dev/null +++ b/airbyte-integrations/connectors/destination-typesense/destination_typesense/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .destination import DestinationTypesense + +__all__ = ["DestinationTypesense"] diff --git a/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py b/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py new file mode 100644 index 000000000000..ac6eb889d319 --- /dev/null +++ b/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py @@ -0,0 +1,61 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from logging import Logger +from typing import Any, Iterable, Mapping + +from airbyte_cdk.destinations import Destination +from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type +from destination_typesense.writer import TypesenseWriter +from typesense import Client + + +def get_client(config: Mapping[str, Any]) -> Client: + api_key = config.get("api_key") + host = config.get("host") + port = config.get("port") or "8108" + protocol = config.get("protocol") or "https" + + client = Client({"api_key": api_key, "nodes": [{"host": host, "port": port, "protocol": protocol}], "connection_timeout_seconds": 2}) + + return client + + +class DestinationTypesense(Destination): + def write( + self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] + ) -> Iterable[AirbyteMessage]: + client = get_client(config=config) + + for configured_stream in configured_catalog.streams: + steam_name = configured_stream.stream.name + if configured_stream.destination_sync_mode == DestinationSyncMode.overwrite: + try: + client.collections[steam_name].delete() + except Exception: + pass + client.collections.create({"name": steam_name, "fields": [{"name": ".*", "type": "auto"}]}) + + writer = TypesenseWriter(client, steam_name, config.get("batch_size")) + for message in input_messages: + if message.type == Type.STATE: + writer.flush() + yield message + elif message.type == Type.RECORD: + writer.queue_write_operation(message.record.data) + else: + continue + writer.flush() + + def check(self, logger: Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: + try: + client = get_client(config=config) + client.collections.create({"name": "_airbyte", "fields": [{"name": "title", "type": "string"}]}) + client.collections["_airbyte"].documents.create({"id": "1", "title": "The Hunger Games"}) + client.collections["_airbyte"].documents["1"].retrieve() + client.collections["_airbyte"].delete() + return AirbyteConnectionStatus(status=Status.SUCCEEDED) + except Exception as e: + return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {repr(e)}") diff --git a/airbyte-integrations/connectors/destination-typesense/destination_typesense/spec.json b/airbyte-integrations/connectors/destination-typesense/destination_typesense/spec.json new file mode 100644 index 000000000000..ee2cb5c76fab --- /dev/null +++ b/airbyte-integrations/connectors/destination-typesense/destination_typesense/spec.json @@ -0,0 +1,46 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/destinations/typesense", + "supported_destination_sync_modes": ["overwrite", "append"], + "supportsIncremental": true, + "supportsDBT": false, + "supportsNormalization": false, + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Destination Typesense", + "type": "object", + "required": ["api_key", "host"], + "additionalProperties": false, + "properties": { + "api_key": { + "title": "API Key", + "type": "string", + "description": "Typesense API Key", + "order": 0 + }, + "host": { + "title": "Host", + "type": "string", + "description": "Hostname of the Typesense instance without protocol.", + "order": 1 + }, + "port": { + "title": "Port", + "type": "string", + "description": "Port of the Typesense instance. Ex: 8108, 80, 443. Default is 443", + "order": 2 + }, + "protocol": { + "title": "Protocol", + "type": "string", + "description": "Protocol of the Typesense instance. Ex: http or https. Default is https", + "order": 3 + }, + "batch_size": { + "title": "Batch size", + "type": "string", + "description": "How many documents should be imported together. Default 1000", + "order": 4 + } + } + } +} diff --git a/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py b/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py new file mode 100644 index 000000000000..904bfddca826 --- /dev/null +++ b/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py @@ -0,0 +1,35 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from collections.abc import Mapping +from logging import getLogger +from uuid import uuid4 + +from typesense import Client + +logger = getLogger("airbyte") + + +class TypesenseWriter: + write_buffer = [] + + def __init__(self, client: Client, steam_name: str, batch_size: int = 1000): + self.client = client + self.steam_name = steam_name + self.batch_size = batch_size + + def queue_write_operation(self, data: Mapping): + random_key = str(uuid4()) + data_with_id = data if "id" in data else {**data, "id": random_key} + self.write_buffer.append(data_with_id) + if len(self.write_buffer) == self.batch_size: + self.flush() + + def flush(self): + buffer_size = len(self.write_buffer) + if buffer_size == 0: + return + logger.info(f"flushing {buffer_size} records") + self.client.collections[self.steam_name].documents.import_(self.write_buffer) + self.write_buffer.clear() diff --git a/airbyte-integrations/connectors/destination-typesense/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-typesense/integration_tests/integration_test.py new file mode 100644 index 000000000000..fd603bb6991e --- /dev/null +++ b/airbyte-integrations/connectors/destination-typesense/integration_tests/integration_test.py @@ -0,0 +1,95 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import json +from logging import getLogger +from typing import Any, Dict, Mapping + +import pytest +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + Status, + SyncMode, + Type, +) +from destination_typesense.destination import DestinationTypesense, get_client +from typesense import Client + + +@pytest.fixture(name="config") +def config_fixture() -> Mapping[str, Any]: + with open("secrets/config.json", "r") as f: + return json.loads(f.read()) + + +@pytest.fixture(name="configured_catalog") +def configured_catalog_fixture() -> ConfiguredAirbyteCatalog: + stream_schema = {"type": "object", "properties": {"col1": {"type": "str"}, "col2": {"type": "integer"}}} + + overwrite_stream = ConfiguredAirbyteStream( + stream=AirbyteStream(name="_airbyte", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + + return ConfiguredAirbyteCatalog(streams=[overwrite_stream]) + + +@pytest.fixture(autouse=True) +def teardown(config: Mapping): + yield + client = get_client(config=config) + try: + client.collections["_airbyte"].delete() + except Exception: + pass + + +@pytest.fixture(name="client") +def client_fixture(config) -> Client: + client = get_client(config=config) + client.collections.create({"name": "_airbyte", "fields": [{"name": ".*", "type": "auto"}]}) + return client + + +def test_check_valid_config(config: Mapping): + outcome = DestinationTypesense().check(getLogger("airbyte"), config) + assert outcome.status == Status.SUCCEEDED + + +def test_check_invalid_config(): + outcome = DestinationTypesense().check(getLogger("airbyte"), {"api_key": "not_a_real_key", "host": "https://www.fake.com"}) + assert outcome.status == Status.FAILED + + +def _state(data: Dict[str, Any]) -> AirbyteMessage: + return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=data)) + + +def _record(stream: str, str_value: str, int_value: int) -> AirbyteMessage: + return AirbyteMessage( + type=Type.RECORD, record=AirbyteRecordMessage(stream=stream, data={"str_col": str_value, "int_col": int_value}, emitted_at=0) + ) + + +def records_count(client: Client) -> int: + documents_results = client.index("_airbyte").get_documents() + return documents_results.total + + +def test_write(config: Mapping, configured_catalog: ConfiguredAirbyteCatalog, client: Client): + overwrite_stream = configured_catalog.streams[0].stream.name + first_state_message = _state({"state": "1"}) + first_record_chunk = [_record(overwrite_stream, str(i), i) for i in range(2)] + + destination = DestinationTypesense() + list(destination.write(config, configured_catalog, [*first_record_chunk, first_state_message])) + collection = client.collections["_airbyte"].retrieve() + assert collection["num_documents"] == 2 diff --git a/airbyte-integrations/connectors/destination-typesense/main.py b/airbyte-integrations/connectors/destination-typesense/main.py new file mode 100644 index 000000000000..3d97913108e3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-typesense/main.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from destination_typesense import DestinationTypesense + +if __name__ == "__main__": + DestinationTypesense().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-typesense/requirements.txt b/airbyte-integrations/connectors/destination-typesense/requirements.txt new file mode 100644 index 000000000000..d6e1198b1ab1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-typesense/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/airbyte-integrations/connectors/destination-typesense/setup.py b/airbyte-integrations/connectors/destination-typesense/setup.py new file mode 100644 index 000000000000..5c7542d9a742 --- /dev/null +++ b/airbyte-integrations/connectors/destination-typesense/setup.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = ["airbyte-cdk", "typesense>=0.14.0"] + +TEST_REQUIREMENTS = ["pytest~=6.1", "typesense>=0.14.0"] + +setup( + name="destination_typesense", + description="Destination implementation for Typesense.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/destination-typesense/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-typesense/unit_tests/unit_test.py new file mode 100644 index 000000000000..991e3cf2d526 --- /dev/null +++ b/airbyte-integrations/connectors/destination-typesense/unit_tests/unit_test.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import patch + +from destination_typesense.writer import TypesenseWriter + + +@patch("typesense.Client") +def test_queue_write_operation(client): + writer = TypesenseWriter(client, "steam_name") + writer.queue_write_operation({"a": "a"}) + assert len(writer.write_buffer) == 1 + + +@patch("typesense.Client") +def test_flush(client): + writer = TypesenseWriter(client, "steam_name") + writer.queue_write_operation({"a": "a"}) + writer.flush() + client.collections.__getitem__.assert_called_once_with("steam_name") diff --git a/airbyte-integrations/connectors/destination-yugabytedb/.dockerignore b/airbyte-integrations/connectors/destination-yugabytedb/.dockerignore new file mode 100644 index 000000000000..65c7d0ad3e73 --- /dev/null +++ b/airbyte-integrations/connectors/destination-yugabytedb/.dockerignore @@ -0,0 +1,3 @@ +* +!Dockerfile +!build diff --git a/airbyte-integrations/connectors/destination-yugabytedb/Dockerfile b/airbyte-integrations/connectors/destination-yugabytedb/Dockerfile new file mode 100644 index 000000000000..fe75d31d32ba --- /dev/null +++ b/airbyte-integrations/connectors/destination-yugabytedb/Dockerfile @@ -0,0 +1,18 @@ +FROM airbyte/integration-base-java:dev AS build + +WORKDIR /airbyte +ENV APPLICATION destination-yugabytedb + +COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar + +RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar + +FROM airbyte/integration-base-java:dev + +WORKDIR /airbyte +ENV APPLICATION destination-yugabytedb + +COPY --from=build /airbyte /airbyte + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/destination-yugabytedb diff --git a/airbyte-integrations/connectors/destination-yugabytedb/README.md b/airbyte-integrations/connectors/destination-yugabytedb/README.md new file mode 100644 index 000000000000..7339896b055c --- /dev/null +++ b/airbyte-integrations/connectors/destination-yugabytedb/README.md @@ -0,0 +1,68 @@ +# Destination Yugabytedb + +This is the repository for the Yugabytedb destination connector in Java. +For information about how to use this connector within Airbyte, see [the User Documentation](https://docs.airbyte.io/integrations/destinations/yugabytedb). + +## Local development + +#### Building via Gradle +From the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:destination-yugabytedb:build +``` + +#### Create credentials +**If you are a community contributor**, generate the necessary credentials and place them in `secrets/config.json` conforming to the spec file in `src/main/resources/spec.json`. +Note that the `secrets` directory is git-ignored by default, so there is no danger of accidentally checking in sensitive information. + +**If you are an Airbyte core member**, follow the [instructions](https://docs.airbyte.io/connector-development#using-credentials-in-ci) to set up the credentials. + +### Locally running the connector docker image + +#### Build +Build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:destination-yugabytedb:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/destination-yugabytedb:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-yugabytedb:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-yugabytedb:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-yugabytedb:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Testing +We use `JUnit` for Java tests. + +### Unit and Integration Tests +Place unit tests under `src/test/io/airbyte/integrations/destinations/yugabytedb`. + +#### Acceptance Tests +Airbyte has a standard test suite that all destination connectors must pass. Implement the `TODO`s in +`src/test-integration/java/io/airbyte/integrations/destinations/yugabytedbDestinationAcceptanceTest.java`. + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:destination-yugabytedb:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:destination-yugabytedb:integrationTest +``` + +## Dependency Management + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/destination-yugabytedb/bootstrap.md b/airbyte-integrations/connectors/destination-yugabytedb/bootstrap.md new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/destination-yugabytedb/build.gradle b/airbyte-integrations/connectors/destination-yugabytedb/build.gradle new file mode 100644 index 000000000000..51ae7a928363 --- /dev/null +++ b/airbyte-integrations/connectors/destination-yugabytedb/build.gradle @@ -0,0 +1,32 @@ +plugins { + id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' +} + +application { + mainClass = 'io.airbyte.integrations.destination.yugabytedb.YugabytedbDestination' +} + +dependencies { + implementation project(':airbyte-config:config-models') + implementation project(':airbyte-protocol:protocol-models') + implementation project(':airbyte-integrations:bases:base-java') + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + implementation project(':airbyte-integrations:connectors:destination-jdbc') + implementation project(':airbyte-db:db-lib') + + implementation 'com.yugabyte:jdbc-yugabytedb:42.3.5-yb-1' + + testImplementation project(':airbyte-integrations:bases:standard-destination-test') + + testImplementation "org.assertj:assertj-core:3.21.0" + testImplementation "org.junit.jupiter:junit-jupiter:5.8.1" + testImplementation "org.testcontainers:junit-jupiter:1.17.5" + testImplementation "org.testcontainers:jdbc:1.17.5" + + + + integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') + integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-yugabytedb') +} diff --git a/airbyte-integrations/connectors/destination-yugabytedb/docker-compose.yml b/airbyte-integrations/connectors/destination-yugabytedb/docker-compose.yml new file mode 100644 index 000000000000..cbd967d1f4af --- /dev/null +++ b/airbyte-integrations/connectors/destination-yugabytedb/docker-compose.yml @@ -0,0 +1,31 @@ +version: '3' + + +# Note: add mount points at /mnt/master and /mnt/tserver for persistence + +services: + yb-master: + image: yugabytedb/yugabyte:latest + container_name: yb-master-n1 + command: [ "/home/yugabyte/bin/yb-master", + "--fs_data_dirs=/mnt/master", + "--master_addresses=yb-master-n1:7100", + "--rpc_bind_addresses=yb-master-n1:7100", + "--replication_factor=1"] + ports: + - "7000:7000" + + yb-tserver: + image: yugabytedb/yugabyte:latest + container_name: yb-tserver-n1 + command: [ "/home/yugabyte/bin/yb-tserver", + "--fs_data_dirs=/mnt/tserver", + "--start_pgsql_proxy", + "--rpc_bind_addresses=yb-tserver-n1:9100", + "--tserver_master_addrs=yb-master-n1:7100"] + ports: + - "9042:9042" + - "5433:5433" + - "9000:9000" + depends_on: + - yb-master diff --git a/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestination.java b/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestination.java new file mode 100644 index 000000000000..c7d6fca26337 --- /dev/null +++ b/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestination.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.yugabytedb; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; +import java.util.Collections; +import java.util.Map; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class YugabytedbDestination extends AbstractJdbcDestination { + + private static final Logger LOGGER = LoggerFactory.getLogger(YugabytedbDestination.class); + + public static final String DRIVER_CLASS = DatabaseDriver.YUGABYTEDB.getDriverClassName(); + + public YugabytedbDestination() { + super(DRIVER_CLASS, new YugabytedbNamingTransformer(), new YugabytedbSqlOperations()); + } + + public static void main(String[] args) throws Exception { + LOGGER.info("starting destination: {}", YugabytedbDestination.class); + new IntegrationRunner(new YugabytedbDestination()).run(args); + LOGGER.info("completed destination: {}", YugabytedbDestination.class); + } + + @Override + protected Map getDefaultConnectionProperties(JsonNode config) { + return Collections.emptyMap(); + } + + @Override + public JsonNode toJdbcConfig(JsonNode config) { + String schema = + Optional.ofNullable(config.get(JdbcUtils.SCHEMA_KEY)).map(JsonNode::asText).orElse("public"); + + String jdbcUrl = "jdbc:yugabytedb://" + config.get(JdbcUtils.HOST_KEY).asText() + ":" + + config.get(JdbcUtils.PORT_KEY).asText() + "/" + + config.get(JdbcUtils.DATABASE_KEY).asText(); + + ImmutableMap.Builder configBuilder = ImmutableMap.builder() + .put(JdbcUtils.USERNAME_KEY, config.get(JdbcUtils.USERNAME_KEY).asText()) + .put(JdbcUtils.JDBC_URL_KEY, jdbcUrl) + .put(JdbcUtils.SCHEMA_KEY, schema); + + if (config.has(JdbcUtils.PASSWORD_KEY)) { + configBuilder.put(JdbcUtils.PASSWORD_KEY, config.get(JdbcUtils.PASSWORD_KEY).asText()); + } + + if (config.has(JdbcUtils.JDBC_URL_PARAMS_KEY)) { + configBuilder.put(JdbcUtils.JDBC_URL_PARAMS_KEY, config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText()); + } + + return Jsons.jsonNode(configBuilder.build()); + } + +} diff --git a/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbNamingTransformer.java b/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbNamingTransformer.java new file mode 100644 index 000000000000..60b8ab1b44fb --- /dev/null +++ b/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbNamingTransformer.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.yugabytedb; + +import io.airbyte.integrations.destination.ExtendedNameTransformer; + +public class YugabytedbNamingTransformer extends ExtendedNameTransformer { + + @Override + public String applyDefaultCase(final String input) { + return input.toLowerCase(); + } + +} diff --git a/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbSqlOperations.java b/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbSqlOperations.java new file mode 100644 index 000000000000..cff16136ac2c --- /dev/null +++ b/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbSqlOperations.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.yugabytedb; + +import com.yugabyte.copy.CopyManager; +import com.yugabyte.core.BaseConnection; +import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.destination.jdbc.JdbcSqlOperations; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.List; + +public class YugabytedbSqlOperations extends JdbcSqlOperations { + + @Override + protected void insertRecordsInternal(JdbcDatabase database, + List records, + String schemaName, + String tableName) + throws Exception { + + if (records.isEmpty()) { + return; + } + + File tempFile = null; + try { + tempFile = Files.createTempFile(tableName + "-", ".tmp").toFile(); + writeBatchToFile(tempFile, records); + + File finalTempFile = tempFile; + database.execute(connection -> { + + var copyManager = new CopyManager(connection.unwrap(BaseConnection.class)); + var sql = String.format("COPY %s.%s FROM STDIN DELIMITER ',' CSV", schemaName, tableName); + + try (var bufferedReader = new BufferedReader(new FileReader(finalTempFile, StandardCharsets.UTF_8))) { + copyManager.copyIn(sql, bufferedReader); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } finally { + if (tempFile != null) { + Files.delete(tempFile.toPath()); + } + } + } + +} diff --git a/airbyte-integrations/connectors/destination-yugabytedb/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-yugabytedb/src/main/resources/spec.json new file mode 100644 index 000000000000..fe77cdd07639 --- /dev/null +++ b/airbyte-integrations/connectors/destination-yugabytedb/src/main/resources/spec.json @@ -0,0 +1,65 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/destinations/yugabytedb", + "supportsIncremental": true, + "supportsNormalization": false, + "supportsDBT": false, + "supported_destination_sync_modes": ["overwrite", "append"], + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Yugabytedb destination spec", + "type": "object", + "required": ["host", "port", "username", "database", "schema"], + "additionalProperties": true, + "properties": { + "host": { + "title": "Host", + "description": "The Hostname of the database.", + "type": "string", + "order": 0 + }, + "port": { + "title": "Port", + "description": "The Port of the database.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 3306, + "examples": ["3306"], + "order": 1 + }, + "database": { + "title": "Database", + "description": "Name of the database.", + "type": "string", + "order": 2 + }, + "username": { + "title": "Username", + "description": "The Username which is used to access the database.", + "type": "string", + "order": 3 + }, + "schema": { + "title": "Default Schema", + "description": "The default schema tables are written to if the source does not specify a namespace. The usual value for this field is \"public\".", + "type": "string", + "examples": ["public"], + "default": "public", + "order": 3 + }, + "password": { + "title": "Password", + "description": "The Password associated with the username.", + "type": "string", + "airbyte_secret": true, + "order": 4 + }, + "jdbc_url_params": { + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", + "title": "JDBC URL Params", + "type": "string", + "order": 5 + } + } + } +} diff --git a/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabyteDataSource.java b/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabyteDataSource.java new file mode 100644 index 000000000000..08f5c81f1ca2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabyteDataSource.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.yugabytedb; + +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import java.util.Collections; +import javax.sql.DataSource; + +public class YugabyteDataSource { + + private YugabyteDataSource() { + + } + + static DataSource getInstance(String host, int port, String database, String username, String password) { + String jdbcUrl = "jdbc:yugabytedb://" + host + ":" + port + "/" + database; + return DataSourceFactory.create( + username, + password, + DatabaseDriver.YUGABYTEDB.getDriverClassName(), + jdbcUrl, + Collections.emptyMap()); + } + +} diff --git a/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbContainerInitializr.java b/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbContainerInitializr.java new file mode 100644 index 000000000000..0f1022d43320 --- /dev/null +++ b/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbContainerInitializr.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.yugabytedb; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.JdbcDatabaseContainer; +import org.testcontainers.utility.DockerImageName; + +public class YugabytedbContainerInitializr { + + private static final Logger LOGGER = LoggerFactory.getLogger(YugabytedbContainerInitializr.class); + + private static YugabytedbContainer yugabytedbContainer; + + private YugabytedbContainerInitializr() { + + } + + public static YugabytedbContainer initContainer() { + if (yugabytedbContainer == null) { + yugabytedbContainer = new YugabytedbContainer(); + } + yugabytedbContainer.start(); + return yugabytedbContainer; + } + + static class YugabytedbContainer extends JdbcDatabaseContainer { + + private static final int YUGABYTE_PORT = 5433; + + public YugabytedbContainer() { + super(DockerImageName.parse("yugabytedb/yugabyte:2.15.2.0-b87")); + + this.setCommand("bin/yugabyted", "start", "--daemon=false"); + this.addExposedPort(YUGABYTE_PORT); + + } + + @Override + public String getDriverClassName() { + return "com.yugabyte.Driver"; + } + + @Override + public String getJdbcUrl() { + String params = constructUrlParameters("?", "&"); + return "jdbc:yugabytedb://" + getHost() + ":" + getMappedPort(YUGABYTE_PORT) + "/yugabyte" + params; + } + + @Override + public String getDatabaseName() { + return "yugabyte"; + } + + @Override + public String getUsername() { + return "yugabyte"; + } + + @Override + public String getPassword() { + return "yugabyte"; + } + + @Override + protected String getTestQueryString() { + return "SELECT 1"; + } + + } + +} diff --git a/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestinationAcceptanceTest.java new file mode 100644 index 000000000000..0c4d70581041 --- /dev/null +++ b/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestinationAcceptanceTest.java @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.yugabytedb; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.destination.ExtendedNameTransformer; +import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; +import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; +import java.sql.SQLException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestInstance; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class YugabytedbDestinationAcceptanceTest extends JdbcDestinationAcceptanceTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(YugabytedbDestinationAcceptanceTest.class); + + private YugabytedbContainerInitializr.YugabytedbContainer yugabytedbContainer; + + private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer(); + + private JsonNode jsonConfig; + + private JdbcDatabase database; + + private static final Set cleanupTables = new HashSet<>(); + + @BeforeAll + void initContainer() { + yugabytedbContainer = YugabytedbContainerInitializr.initContainer(); + } + + @Override + protected String getImageName() { + return "airbyte/destination-yugabytedb:dev"; + } + + @Override + protected void setup(TestDestinationEnv testEnv) throws Exception { + jsonConfig = Jsons.jsonNode(ImmutableMap.builder() + .put("host", yugabytedbContainer.getHost()) + .put("port", yugabytedbContainer.getMappedPort(5433)) + .put("database", yugabytedbContainer.getDatabaseName()) + .put("username", yugabytedbContainer.getUsername()) + .put("password", yugabytedbContainer.getPassword()) + .put("schema", "public") + .build()); + + database = new DefaultJdbcDatabase(YugabyteDataSource.getInstance( + yugabytedbContainer.getHost(), + yugabytedbContainer.getMappedPort(5433), + yugabytedbContainer.getDatabaseName(), + yugabytedbContainer.getUsername(), + yugabytedbContainer.getPassword())); + + } + + @Override + protected void tearDown(TestDestinationEnv testEnv) throws Exception { + database.execute(connection -> { + var statement = connection.createStatement(); + cleanupTables.forEach(tb -> { + try { + statement.execute("DROP TABLE " + tb + ";"); + } catch (SQLException e) { + throw new RuntimeException(e); + } + }); + }); + cleanupTables.clear(); + } + + @Override + protected JsonNode getConfig() { + return jsonConfig; + } + + @Override + protected JsonNode getFailCheckConfig() { + return Jsons.jsonNode(ImmutableMap.builder() + .put("host", yugabytedbContainer.getHost()) + .put("port", yugabytedbContainer.getMappedPort(5433)) + .put("database", yugabytedbContainer.getDatabaseName()) + .put("username", "usr") + .put("password", "pw") + .put("schema", "public") + .build()); + } + + @Override + protected boolean implementsNamespaces() { + return true; + } + + @Override + protected TestDataComparator getTestDataComparator() { + return new AdvancedTestDataComparator(); + } + + @Override + protected boolean supportBasicDataTypeTest() { + return true; + } + + @Override + protected boolean supportArrayDataTypeTest() { + return true; + } + + @Override + protected boolean supportObjectDataTypeTest() { + return true; + } + + @Override + protected List retrieveRecords(TestDestinationEnv testEnv, + String streamName, + String namespace, + JsonNode streamSchema) + throws SQLException { + + String tableName = namingResolver.getRawTableName(streamName); + String schemaName = namingResolver.getNamespace(namespace); + cleanupTables.add(schemaName + "." + tableName); + return retrieveRecordsFromTable(tableName, schemaName); + } + + private List retrieveRecordsFromTable(final String tableName, final String schemaName) + throws SQLException { + + return database.bufferedResultSetQuery( + connection -> { + var statement = connection.createStatement(); + return statement.executeQuery( + String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, + JavaBaseConstants.COLUMN_NAME_EMITTED_AT)); + }, + rs -> Jsons.deserialize(rs.getString(JavaBaseConstants.COLUMN_NAME_DATA))); + } + +} diff --git a/airbyte-integrations/connectors/destination-yugabytedb/src/test/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestinationTest.java b/airbyte-integrations/connectors/destination-yugabytedb/src/test/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestinationTest.java new file mode 100644 index 000000000000..2578ca428285 --- /dev/null +++ b/airbyte-integrations/connectors/destination-yugabytedb/src/test/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestinationTest.java @@ -0,0 +1,50 @@ +package io.airbyte.integrations.destination.yugabytedb; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import java.util.Collections; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class YugabytedbDestinationTest { + + private YugabytedbDestination yugabytedbDestination; + + @BeforeEach + void setup() { + yugabytedbDestination = new YugabytedbDestination(); + } + + @Test + void testToJdbcConfig() { + + var config = Jsons.jsonNode(ImmutableMap.builder() + .put("host", "localhost") + .put("port", 5433) + .put("database", "yugabyte") + .put("username", "yugabyte") + .put("password", "yugabyte") + .put("schema", "public") + .build()); + + var jdbcConfig = yugabytedbDestination.toJdbcConfig(config); + + assertThat(jdbcConfig.get("schema").asText()).isEqualTo("public"); + assertThat(jdbcConfig.get("username").asText()).isEqualTo("yugabyte"); + assertThat(jdbcConfig.get("password").asText()).isEqualTo("yugabyte"); + assertThat(jdbcConfig.get("jdbc_url").asText()).isEqualTo("jdbc:yugabytedb://localhost:5433/yugabyte"); + + } + + @Test + void testGetDefaultConnectionProperties() { + + var map = yugabytedbDestination.getDefaultConnectionProperties(Jsons.jsonNode(Collections.emptyMap())); + + assertThat(map).isEmpty(); + + } + +} diff --git a/airbyte-integrations/connectors/destination-yugabytedb/src/test/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbNamingTransformerTest.java b/airbyte-integrations/connectors/destination-yugabytedb/src/test/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbNamingTransformerTest.java new file mode 100644 index 000000000000..5565bc9d2ef9 --- /dev/null +++ b/airbyte-integrations/connectors/destination-yugabytedb/src/test/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbNamingTransformerTest.java @@ -0,0 +1,27 @@ +package io.airbyte.integrations.destination.yugabytedb; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class YugabytedbNamingTransformerTest { + + private YugabytedbNamingTransformer yugabytedbNamingTransformer; + + @BeforeEach + void setup() { + yugabytedbNamingTransformer = new YugabytedbNamingTransformer(); + } + + @Test + void testApplyDefaultCase() { + + var defaultCase = yugabytedbNamingTransformer.applyDefaultCase("DEFAULT_CASE"); + + assertThat(defaultCase).isEqualTo("default_case"); + + } + + +} diff --git a/airbyte-integrations/connectors/source-activecampaign/.dockerignore b/airbyte-integrations/connectors/source-activecampaign/.dockerignore new file mode 100644 index 000000000000..4c58fa10a6e6 --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_activecampaign +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-activecampaign/Dockerfile b/airbyte-integrations/connectors/source-activecampaign/Dockerfile new file mode 100644 index 000000000000..1b8ff074edaa --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_activecampaign ./source_activecampaign + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-activecampaign diff --git a/airbyte-integrations/connectors/source-activecampaign/README.md b/airbyte-integrations/connectors/source-activecampaign/README.md new file mode 100644 index 000000000000..da82771bf434 --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/README.md @@ -0,0 +1,79 @@ +# Activecampaign Source + +This is the repository for the Activecampaign configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/activecampaign). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-activecampaign:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/activecampaign) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_activecampaign/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source activecampaign test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-activecampaign:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-activecampaign:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-activecampaign:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-activecampaign:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-activecampaign:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-activecampaign:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-activecampaign:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-activecampaign:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-zoom-singer/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-activecampaign/__init__.py similarity index 57% rename from airbyte-integrations/connectors/source-zoom-singer/unit_tests/unit_test.py rename to airbyte-integrations/connectors/source-activecampaign/__init__.py index dddaea0060fa..1100c1c58cf5 100644 --- a/airbyte-integrations/connectors/source-zoom-singer/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-activecampaign/__init__.py @@ -1,7 +1,3 @@ # # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # - - -def test_example_method(): - assert True diff --git a/airbyte-integrations/connectors/source-activecampaign/acceptance-test-config.yml b/airbyte-integrations/connectors/source-activecampaign/acceptance-test-config.yml new file mode 100644 index 000000000000..7543dba988ef --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-activecampaign:dev +tests: + spec: + - spec_path: "source_activecampaign/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.txt" + # extra_fields: no + # exact_order: no + # extra_records: yes + # incremental: # TODO if your connector does not implement incremental sync, remove this block + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state_path: "integration_tests/abnormal_state.json" + # full_refresh: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-activecampaign/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-activecampaign/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-activecampaign/bootstrap.md b/airbyte-integrations/connectors/source-activecampaign/bootstrap.md new file mode 100644 index 000000000000..a941ac39c9f8 --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/bootstrap.md @@ -0,0 +1,13 @@ +# ActiveCampaign + +The ActiveCampaign source connector interacts with the ActiveCampaign API, which +provides metrics for yor transactional email, email marketing, marketing automation, +sales automation, and CRM software. + +[Authentication](https://developers.activecampaign.com/reference/authentication) +is handled via a regular `Authorization` HTTP header. + +Account username is used as part of API [base URL](https://developers.activecampaign.com/reference/url). + +See the [official documentation](https://developers.activecampaign.com/reference/overview) +for details on how the API works. diff --git a/airbyte-integrations/connectors/source-activecampaign/build.gradle b/airbyte-integrations/connectors/source-activecampaign/build.gradle new file mode 100644 index 000000000000..ca76b9550daa --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_activecampaign' +} diff --git a/airbyte-integrations/connectors/source-activecampaign/integration_tests/__init__.py b/airbyte-integrations/connectors/source-activecampaign/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-activecampaign/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-activecampaign/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-activecampaign/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-activecampaign/integration_tests/acceptance.py new file mode 100644 index 000000000000..1302b2f57e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-activecampaign/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-activecampaign/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..a74ca3665f1a --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/integration_tests/configured_catalog.json @@ -0,0 +1,58 @@ +{ + "streams": [ + { + "stream": { + "name": "campaigns", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "contacts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "deals", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "segments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "lists", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "forms", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-activecampaign/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-activecampaign/integration_tests/invalid_config.json new file mode 100644 index 000000000000..70b3d19fe9c6 --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "", + "account_username": "invalid_account" +} diff --git a/airbyte-integrations/connectors/source-activecampaign/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-activecampaign/integration_tests/sample_config.json new file mode 100644 index 000000000000..f6c93954a7a2 --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/integration_tests/sample_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "", + "account_username": "dainiuxazz" +} diff --git a/airbyte-integrations/connectors/source-activecampaign/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-activecampaign/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-activecampaign/main.py b/airbyte-integrations/connectors/source-activecampaign/main.py new file mode 100644 index 000000000000..d7e7b299c1c0 --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_activecampaign import SourceActivecampaign + +if __name__ == "__main__": + source = SourceActivecampaign() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-activecampaign/requirements.txt b/airbyte-integrations/connectors/source-activecampaign/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-activecampaign/setup.py b/airbyte-integrations/connectors/source-activecampaign/setup.py new file mode 100644 index 000000000000..b7d4d4645db4 --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_activecampaign", + description="Source implementation for Activecampaign.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/__init__.py b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/__init__.py new file mode 100644 index 000000000000..c350061c7a62 --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceActivecampaign + +__all__ = ["SourceActivecampaign"] diff --git a/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/activecampaign.yaml b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/activecampaign.yaml new file mode 100644 index 000000000000..71ab72c835ea --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/activecampaign.yaml @@ -0,0 +1,100 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + # API Docs: https://developers.activecampaign.com/reference/schema + field_pointer: ["{{ options['name'] }}"] + + requester: + # API Docs: https://developers.activecampaign.com/reference/url + url_base: "https://{{ config['account_username'] }}.api-us1.com/api/3" + http_method: "GET" + authenticator: + type: ApiKeyAuthenticator + header: "Api-Token" + api_token: "{{ config['api_key'] }}" + + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + + # API Docs: https://developers.activecampaign.com/reference/pagination + paginator: + type: DefaultPaginator + url_base: "*ref(definitions.requester.url_base)" + page_size_option: + inject_into: "request_parameter" + field_name: "limit" + pagination_strategy: + type: "OffsetIncrement" + page_size: 20 + page_token_option: + inject_into: "request_parameter" + field_name: "offset" + + requester: + $ref: "*ref(definitions.requester)" + + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + + # API Docs: https://developers.activecampaign.com/reference/list-all-campaigns + campaigns_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "campaigns" + primary_key: "id" + path: "/campaigns" + + # API Docs: https://developers.activecampaign.com/reference/list-all-contacts + contacts_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "contacts" + primary_key: "id" + path: "/contacts" + + # API Docs: https://developers.activecampaign.com/reference/list-all-deals + deals_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "deals" + primary_key: "id" + path: "/deals" + + # API Docs: https://developers.activecampaign.com/reference/retrieve-all-lists + lists_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "lists" + primary_key: "id" + path: "/lists" + + # API Docs: https://developers.activecampaign.com/reference/list-all-segments + segments_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "segments" + primary_key: "id" + path: "/segments" + + # API Docs: https://developers.activecampaign.com/reference/forms-1 + forms_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "forms" + primary_key: "id" + path: "/forms" + +streams: + - "*ref(definitions.campaigns_stream)" + - "*ref(definitions.contacts_stream)" + - "*ref(definitions.lists_stream)" + - "*ref(definitions.deals_stream)" + - "*ref(definitions.segments_stream)" + - "*ref(definitions.forms_stream)" + +check: + stream_names: ["campaigns"] diff --git a/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/schemas/campaigns.json b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/schemas/campaigns.json new file mode 100644 index 000000000000..569f22f47dbb --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/schemas/campaigns.json @@ -0,0 +1,383 @@ +{ + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "userid": { + "type": "string" + }, + "segmentid": { + "type": "string" + }, + "bounceid": { + "type": "string" + }, + "realcid": { + "type": "string" + }, + "sendid": { + "type": "string" + }, + "threadid": { + "type": "string" + }, + "seriesid": { + "type": "string" + }, + "formid": { + "type": "string" + }, + "basetemplateid": { + "type": "string" + }, + "basemessageid": { + "type": "string" + }, + "addressid": { + "type": "string" + }, + "source": { + "type": "string" + }, + "name": { + "type": "string" + }, + "cdate": { + "type": "string" + }, + "mdate": { + "type": "string" + }, + "sdate": { + "type": ["string", "null"] + }, + "ldate": { + "type": ["string", "null"] + }, + "send_amt": { + "type": "string" + }, + "total_amt": { + "type": "string" + }, + "opens": { + "type": "string" + }, + "uniqueopens": { + "type": "string" + }, + "linkclicks": { + "type": "string" + }, + "uniquelinkclicks": { + "type": "string" + }, + "subscriberclicks": { + "type": "string" + }, + "forwards": { + "type": "string" + }, + "uniqueforwards": { + "type": "string" + }, + "hardbounces": { + "type": "string" + }, + "softbounces": { + "type": "string" + }, + "unsubscribes": { + "type": "string" + }, + "unsubreasons": { + "type": "string" + }, + "updates": { + "type": "string" + }, + "socialshares": { + "type": "string" + }, + "replies": { + "type": "string" + }, + "uniquereplies": { + "type": "string" + }, + "status": { + "type": "string" + }, + "public": { + "type": "string" + }, + "mail_transfer": { + "type": "string" + }, + "mail_send": { + "type": "string" + }, + "mail_cleanup": { + "type": "string" + }, + "mailer_log_file": { + "type": "string" + }, + "tracklinks": { + "type": "string" + }, + "tracklinksanalytics": { + "type": "string" + }, + "trackreads": { + "type": "string" + }, + "trackreadsanalytics": { + "type": "string" + }, + "analytics_campaign_name": { + "type": "string" + }, + "tweet": { + "type": "string" + }, + "facebook": { + "type": "string" + }, + "survey": { + "type": "string" + }, + "embed_images": { + "type": "string" + }, + "htmlunsub": { + "type": "string" + }, + "textunsub": { + "type": "string" + }, + "htmlunsubdata": { + "type": ["string", "null"] + }, + "textunsubdata": { + "type": ["string", "null"] + }, + "recurring": { + "type": "string" + }, + "willrecur": { + "type": "string" + }, + "split_type": { + "type": "string" + }, + "split_content": { + "type": "string" + }, + "split_offset": { + "type": "string" + }, + "split_offset_type": { + "type": "string" + }, + "split_winner_messageid": { + "type": "string" + }, + "split_winner_awaiting": { + "type": "string" + }, + "responder_offset": { + "type": "string" + }, + "responder_type": { + "type": "string" + }, + "responder_existing": { + "type": "string" + }, + "reminder_field": { + "type": "string" + }, + "reminder_format": { + "type": ["string", "null"] + }, + "reminder_type": { + "type": "string" + }, + "reminder_offset": { + "type": "string" + }, + "reminder_offset_type": { + "type": "string" + }, + "reminder_offset_sign": { + "type": "string" + }, + "reminder_last_cron_run": { + "type": ["string", "null"] + }, + "activerss_interval": { + "type": "string" + }, + "activerss_url": { + "type": ["string", "null"] + }, + "activerss_items": { + "type": "string" + }, + "ip4": { + "type": "string" + }, + "laststep": { + "type": "string" + }, + "managetext": { + "type": "string" + }, + "schedule": { + "type": "string" + }, + "scheduleddate": { + "type": ["string", "null"] + }, + "waitpreview": { + "type": ["string", "null"] + }, + "deletestamp": { + "type": ["string", "null"] + }, + "replysys": { + "type": "string" + }, + "links": { + "type": "object", + "properties": { + "user": { + "type": "string" + }, + "automation": { + "type": "string" + }, + "campaignMessage": { + "type": "string" + }, + "campaignMessages": { + "type": "string" + }, + "links": { + "type": "string" + }, + "campaignLists": { + "type": "string" + } + }, + "required": [ + "user", + "automation", + "campaignMessage", + "campaignMessages", + "links", + "campaignLists" + ] + }, + "id": { + "type": "string" + }, + "user": { + "type": "string" + }, + "automation": { + "type": ["string", "null"] + } + }, + "required": [ + "type", + "userid", + "segmentid", + "bounceid", + "realcid", + "sendid", + "threadid", + "seriesid", + "formid", + "basetemplateid", + "basemessageid", + "addressid", + "source", + "name", + "cdate", + "mdate", + "sdate", + "ldate", + "send_amt", + "total_amt", + "opens", + "uniqueopens", + "linkclicks", + "uniquelinkclicks", + "subscriberclicks", + "forwards", + "uniqueforwards", + "hardbounces", + "softbounces", + "unsubscribes", + "unsubreasons", + "updates", + "socialshares", + "replies", + "uniquereplies", + "status", + "public", + "mail_transfer", + "mail_send", + "mail_cleanup", + "mailer_log_file", + "tracklinks", + "tracklinksanalytics", + "trackreads", + "trackreadsanalytics", + "analytics_campaign_name", + "tweet", + "facebook", + "survey", + "embed_images", + "htmlunsub", + "textunsub", + "htmlunsubdata", + "textunsubdata", + "recurring", + "willrecur", + "split_type", + "split_content", + "split_offset", + "split_offset_type", + "split_winner_messageid", + "split_winner_awaiting", + "responder_offset", + "responder_type", + "responder_existing", + "reminder_field", + "reminder_format", + "reminder_type", + "reminder_offset", + "reminder_offset_type", + "reminder_offset_sign", + "reminder_last_cron_run", + "activerss_interval", + "activerss_url", + "activerss_items", + "ip4", + "laststep", + "managetext", + "schedule", + "scheduleddate", + "waitpreview", + "deletestamp", + "replysys", + "links", + "id", + "user", + "automation" + ] +} diff --git a/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/schemas/contacts.json b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/schemas/contacts.json new file mode 100644 index 000000000000..346253e3a5da --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/schemas/contacts.json @@ -0,0 +1,245 @@ +{ + "type": "object", + "properties": { + "cdate": { + "type": "string" + }, + "email": { + "type": "string" + }, + "phone": { + "type": "string" + }, + "firstName": { + "type": "string" + }, + "lastName": { + "type": "string" + }, + "orgid": { + "type": "string" + }, + "orgname": { + "type": "string" + }, + "segmentio_id": { + "type": "string" + }, + "bounced_hard": { + "type": "string" + }, + "bounced_soft": { + "type": "string" + }, + "bounced_date": { + "type": ["string", "null"] + }, + "ip": { + "type": "string" + }, + "ua": { + "type": ["string", "null"] + }, + "hash": { + "type": "string" + }, + "socialdata_lastcheck": { + "type": ["string", "null"] + }, + "email_local": { + "type": "string" + }, + "email_domain": { + "type": "string" + }, + "sentcnt": { + "type": "string" + }, + "rating_tstamp": { + "type": ["string", "null"] + }, + "gravatar": { + "type": "string" + }, + "deleted": { + "type": "string" + }, + "anonymized": { + "type": "string" + }, + "adate": { + "type": ["string", "null"] + }, + "udate": { + "type": "string" + }, + "edate": { + "type": ["string", "null"] + }, + "deleted_at": { + "type": ["string", "null"] + }, + "created_utc_timestamp": { + "type": "string" + }, + "updated_utc_timestamp": { + "type": "string" + }, + "created_timestamp": { + "type": "string" + }, + "updated_timestamp": { + "type": "string" + }, + "created_by": { + "type": ["string", "null"] + }, + "updated_by": { + "type": ["string", "null"] + }, + "email_empty": { + "type": "boolean" + }, + "mpp_tracking": { + "type": "string" + }, + "scoreValues": { + "type": "array", + "items": { + "type": "string" + } + }, + "accountContacts": { + "type": "array", + "items": { + "type": "string" + } + }, + "links": { + "type": "object", + "properties": { + "bounceLogs": { + "type": "string" + }, + "contactAutomations": { + "type": "string" + }, + "contactData": { + "type": "string" + }, + "contactGoals": { + "type": "string" + }, + "contactLists": { + "type": "string" + }, + "contactLogs": { + "type": "string" + }, + "contactTags": { + "type": "string" + }, + "contactDeals": { + "type": "string" + }, + "deals": { + "type": "string" + }, + "fieldValues": { + "type": "string" + }, + "geoIps": { + "type": "string" + }, + "notes": { + "type": "string" + }, + "organization": { + "type": "string" + }, + "plusAppend": { + "type": "string" + }, + "trackingLogs": { + "type": "string" + }, + "scoreValues": { + "type": "string" + }, + "accountContacts": { + "type": "string" + }, + "automationEntryCounts": { + "type": "string" + } + }, + "required": [ + "bounceLogs", + "contactAutomations", + "contactData", + "contactGoals", + "contactLists", + "contactLogs", + "contactTags", + "contactDeals", + "deals", + "fieldValues", + "geoIps", + "notes", + "organization", + "plusAppend", + "trackingLogs", + "scoreValues", + "accountContacts", + "automationEntryCounts" + ] + }, + "id": { + "type": "string" + }, + "organization": { + "type": "null" + } + }, + "required": [ + "cdate", + "email", + "phone", + "firstName", + "lastName", + "orgid", + "orgname", + "segmentio_id", + "bounced_hard", + "bounced_soft", + "bounced_date", + "ip", + "ua", + "hash", + "socialdata_lastcheck", + "email_local", + "email_domain", + "sentcnt", + "rating_tstamp", + "gravatar", + "deleted", + "anonymized", + "adate", + "udate", + "edate", + "deleted_at", + "created_utc_timestamp", + "updated_utc_timestamp", + "created_timestamp", + "updated_timestamp", + "created_by", + "updated_by", + "email_empty", + "mpp_tracking", + "scoreValues", + "accountContacts", + "links", + "id", + "organization" + ] +} diff --git a/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/schemas/deals.json b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/schemas/deals.json new file mode 100644 index 000000000000..0b180501de0e --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/schemas/deals.json @@ -0,0 +1,131 @@ +{ + "type": "object", + "properties": { + "owner": { + "type": ["string", "null"] + }, + "contact": { + "type": ["string", "null"] + }, + "organization": { + "type": ["string", "null"] + }, + "group": { + "type": ["string", "null"] + }, + "stage": { + "type": ["string", "null"] + }, + "title": { + "type": ["string", "null"] + }, + "description": { + "type": ["string", "null"] + }, + "percent": { + "type": ["string", "null"] + }, + "cdate": { + "type": ["string", "null"] + }, + "mdate": { + "type": ["string", "null"] + }, + "nextdate": { + "type": ["string", "null"] + }, + "nexttaskid": { + "type": ["string", "null"] + }, + "nextTask": { + "type": ["string", "null"] + }, + "value": { + "type": ["string", "null"] + }, + "currency": { + "type": ["string", "null"] + }, + "winProbability": { + "type": ["integer", "null"] + }, + "winProbabilityMdate": { + "type": ["string", "null"] + }, + "status": { + "type": ["string", "null"] + }, + "activitycount": { + "type": ["string", "null"] + }, + "nextdealid": { + "type": ["string", "null"] + }, + "edate": { + "type": ["string", "null"] + }, + "links": { + "type": "object", + "properties": { + "dealActivities": { + "type": ["string", "null"] + }, + "contact": { + "type": ["string", "null"] + }, + "contactDeals": { + "type": ["string", "null"] + }, + "group": { + "type": ["string", "null"] + }, + "nextTask": { + "type": ["string", "null"] + }, + "notes": { + "type": ["string", "null"] + }, + "account": { + "type": ["string", "null"] + }, + "customerAccount": { + "type": ["string", "null"] + }, + "organization": { + "type": ["string", "null"] + }, + "owner": { + "type": ["string", "null"] + }, + "scoreValues": { + "type": ["string", "null"] + }, + "stage": { + "type": ["string", "null"] + }, + "tasks": { + "type": ["string", "null"] + }, + "dealCustomFieldData": { + "type": ["string", "null"] + } + } + }, + "id": { + "type": "string" + }, + "hash": { + "type": ["string", "null"] + }, + "isDisabled": { + "type": ["boolean", "null"] + }, + "account": { + "type": ["string", "null"] + }, + "customerAccount": { + "type": ["string", "null"] + } + }, + "required": ["id"] +} diff --git a/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/schemas/forms.json b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/schemas/forms.json new file mode 100644 index 000000000000..195e03258de9 --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/schemas/forms.json @@ -0,0 +1,277 @@ +{ + "$id": "root", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "name": { + "type": ["string", "null"], + "examples": ["Testing"] + }, + "action": { + "type": ["string", "null"] + }, + "actiondata": { + "type": "object", + "properties": { + "actions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": ["string", "null"], + "examples": ["subscribe-to-list"] + }, + "email": { + "type": ["string", "null"], + "examples": ["johndoe@example.com"] + }, + "list": { + "type": ["string", "null"], + "examples": ["1"] + } + } + } + } + } + }, + "submit": { + "type": ["string", "null"], + "examples": ["show-thank-you"] + }, + "submitdata": { + "type": "object", + "properties": { + "url": { + "type": ["string", "null"] + } + } + }, + "url": { + "type": ["string", "null"] + }, + "layout": { + "type": ["string", "null"], + "examples": ["inline-form"] + }, + "title": { + "type": ["string", "null"] + }, + "body": { + "type": ["string", "null"] + }, + "button": { + "type": ["string", "null"], + "examples": ["Submit"] + }, + "thanks": { + "type": ["string", "null"], + "examples": ["Thanks for signing up!"] + }, + "style": { + "type": "object", + "properties": { + "background": { + "type": ["string", "null"], + "examples": ["FFFFFF"] + }, + "dark": { + "type": ["boolean", "null"], + "examples": [true, false] + }, + "fontcolor": { + "type": ["string", "null"], + "examples": ["000000"] + }, + "layout": { + "type": ["string", "null"], + "examples": ["normal"] + }, + "border": { + "type": "object", + "properties": { + "width": { + "type": ["integer", "null"], + "examples": [0] + }, + "style": { + "type": ["string", "null"], + "examples": ["solid"] + }, + "color": { + "type": ["string", "null"], + "examples": ["B0B0B0"] + }, + "radius": { + "type": ["integer", "null"], + "default": 0, + "examples": [0] + } + } + }, + "width": { + "type": ["integer", "null"], + "examples": [500] + }, + "ac_branding": { + "type": ["boolean", "null"], + "examples": [true, false] + }, + "button": { + "type": "object", + "properties": { + "padding": { + "type": ["integer", "null"], + "examples": [10] + }, + "background": { + "type": ["string", "null"], + "examples": ["333333"] + }, + "fontcolor": { + "type": ["string", "null"], + "examples": ["FFFFFF"] + }, + "border": { + "type": "object", + "properties": { + "radius": { + "type": ["integer", "null"], + "examples": [4] + }, + "color": { + "type": ["string", "null"], + "examples": ["333333"] + }, + "style": { + "type": ["string", "null"], + "examples": ["solid"] + }, + "width": { + "type": ["integer", "null"], + "examples": [0] + } + } + } + } + } + } + }, + "options": { + "type": "object", + "properties": { + "blanks_overwrite": { + "type": ["boolean", "null"], + "examples": [true, false] + }, + "confaction": { + "type": ["string", "null"], + "examples": ["show-message"] + }, + "sendoptin": { + "type": ["boolean", "null"], + "examples": [true, false] + }, + "optin_id": { + "type": ["integer", "null"], + "examples": [1] + }, + "optin_created": { + "type": "boolean", + "default": true, + "examples": [true, false] + }, + "confform": { + "type": ["string", "null"], + "examples": ["2"] + } + } + }, + "cfields": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": ["string", "null"], + "examples": ["header"] + }, + "header": { + "type": ["string", "null"], + "examples": ["Subscribe for Email Updates"] + }, + "class": { + "type": ["string", "null"], + "examples": ["_x41699710"] + } + } + } + }, + "parentformid": { + "type": ["string", "null"], + "examples": ["0"] + }, + "userid": { + "type": ["string", "null"], + "examples": ["1"] + }, + "addressid": { + "type": ["string", "null"], + "examples": ["0"] + }, + "cdate": { + "type": ["string", "null"], + "examples": ["2018-08-17T13:47:31-05:00"] + }, + "udate": { + "type": ["string", "null"], + "examples": ["2018-08-17T13:47:38-05:00"] + }, + "entries": { + "type": ["string", "null"], + "examples": ["0"] + }, + "aid": { + "type": ["string", "null"], + "default": "0", + "examples": ["0"] + }, + "defaultscreenshot": { + "type": ["string", "null"], + "default": "http://img-us1.com/default-form.gif", + "examples": ["http://img-us1.com/default-form.gif"] + }, + "recent": { + "type": "array" + }, + "contacts": { + "type": "integer", + "default": 0, + "examples": [0] + }, + "deals": { + "type": "integer", + "default": 0, + "examples": [0] + }, + "links": { + "type": "object", + "properties": { + "address": { + "type": ["string", "null"], + "default": "https://:account.api-us1.com/api/3/forms/1/address", + "examples": ["https://:account.api-us1.com/api/3/forms/1/address"] + } + } + }, + "id": { + "type": "string", + "default": "1", + "examples": ["1"] + }, + "address": { + "type": ["string", "null"] + } + }, + "required": ["id"] +} diff --git a/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/schemas/lists.json b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/schemas/lists.json new file mode 100644 index 000000000000..9baba34c9f62 --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/schemas/lists.json @@ -0,0 +1,149 @@ +{ + "type": "object", + "properties": { + "stringid": { + "type": ["string", "null"] + }, + "userid": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "cdate": { + "type": ["string", "null"] + }, + "p_use_tracking": { + "type": ["string", "null"] + }, + "p_use_analytics_read": { + "type": ["string", "null"] + }, + "p_use_analytics_link": { + "type": ["string", "null"] + }, + "p_use_twitter": { + "type": ["string", "null"] + }, + "p_use_facebook": { + "type": ["string", "null"] + }, + "p_embed_image": { + "type": ["string", "null"] + }, + "p_use_captcha": { + "type": ["string", "null"] + }, + "send_last_broadcast": { + "type": ["string", "null"] + }, + "private": { + "type": ["string", "null"] + }, + "analytics_domains": { + "type": ["string", "null"] + }, + "analytics_source": { + "type": ["string", "null"] + }, + "analytics_ua": { + "type": ["string", "null"] + }, + "twitter_token": { + "type": ["string", "null"] + }, + "twitter_token_secret": { + "type": ["string", "null"] + }, + "facebook_session": { + "type": ["string", "null"] + }, + "carboncopy": { + "type": ["string", "null"] + }, + "subscription_notify": { + "type": ["string", "null"] + }, + "unsubscription_notify": { + "type": ["string", "null"] + }, + "require_name": { + "type": ["string", "null"] + }, + "get_unsubscribe_reason": { + "type": ["string", "null"] + }, + "to_name": { + "type": ["string", "null"] + }, + "optinoptout": { + "type": ["string", "null"] + }, + "sender_name": { + "type": ["string", "null"] + }, + "sender_addr1": { + "type": ["string", "null"] + }, + "sender_addr2": { + "type": ["string", "null"] + }, + "sender_city": { + "type": ["string", "null"] + }, + "sender_state": { + "type": ["string", "null"] + }, + "sender_zip": { + "type": ["string", "null"] + }, + "sender_country": { + "type": ["string", "null"] + }, + "sender_phone": { + "type": ["string", "null"] + }, + "sender_url": { + "type": ["string", "null"] + }, + "sender_reminder": { + "type": ["string", "null"] + }, + "fulladdress": { + "type": ["string", "null"] + }, + "optinmessageid": { + "type": ["string", "null"] + }, + "optoutconf": { + "type": ["string", "null"] + }, + "deletestamp": { + "type": ["string", "null"] + }, + "udate": { + "type": ["string", "null"] + }, + "links": { + "type": "object", + "properties": { + "contactGoalLists": { + "type": ["string", "null"] + }, + "user": { + "type": ["string", "null"] + }, + "addressLists": { + "type": ["string", "null"] + } + } + }, + "id": { + "type": "string" + }, + "user": { + "type": ["string", "null"] + } + }, + "required": ["id"] +} diff --git a/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/schemas/segments.json b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/schemas/segments.json new file mode 100644 index 000000000000..3ac3ece26b10 --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/schemas/segments.json @@ -0,0 +1,29 @@ +{ + "type": "object", + "properties": { + "name": { + "type": ["string", "null"], + "examples": ["Segment of Automation 1"] + }, + "logic": { + "type": ["string", "null"], + "examples": ["and"] + }, + "hidden": { + "type": ["string", "null"], + "examples": ["0"] + }, + "seriesid": { + "type": ["string", "null"], + "examples": ["1"] + }, + "links": { + "type": ["array", "null"] + }, + "id": { + "type": "string", + "examples": ["1"] + } + }, + "required": ["id"] +} diff --git a/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/source.py b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/source.py new file mode 100644 index 000000000000..5ef118e94175 --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceActivecampaign(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "activecampaign.yaml"}) diff --git a/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/spec.yaml b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/spec.yaml new file mode 100644 index 000000000000..8ad5244c2cc7 --- /dev/null +++ b/airbyte-integrations/connectors/source-activecampaign/source_activecampaign/spec.yaml @@ -0,0 +1,20 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/activecampaign +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Activecampaign Spec + type: object + required: + - api_key + - account_username + + additionalProperties: true + + properties: + api_key: + type: string + description: API Key + airbyte_secret: true + + account_username: + type: string + description: Account Username diff --git a/airbyte-integrations/connectors/source-airtable/Dockerfile b/airbyte-integrations/connectors/source-airtable/Dockerfile index fc67fe16c272..3de8ef2803c0 100644 --- a/airbyte-integrations/connectors/source-airtable/Dockerfile +++ b/airbyte-integrations/connectors/source-airtable/Dockerfile @@ -34,5 +34,5 @@ COPY source_airtable ./source_airtable ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-airtable diff --git a/airbyte-integrations/connectors/source-airtable/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-airtable/acceptance-test-docker.sh old mode 100644 new mode 100755 diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/helpers.py b/airbyte-integrations/connectors/source-airtable/source_airtable/helpers.py index b571a42b101d..52dee3407c36 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/helpers.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/helpers.py @@ -13,8 +13,8 @@ class Helpers(object): @staticmethod - def get_first_row(auth: TokenAuthenticator, base_id: str, table: str) -> Dict[str, Any]: - url = f"https://api.airtable.com/v0/{base_id}/{table}?pageSize=1" + def get_most_complete_row(auth: TokenAuthenticator, base_id: str, table: str, sample_size: int = 100) -> Dict[str, Any]: + url = f"https://api.airtable.com/v0/{base_id}/{table}?pageSize={sample_size}" try: response = requests.get(url, headers=auth.get_auth_header()) response.raise_for_status() @@ -26,8 +26,12 @@ def get_first_row(auth: TokenAuthenticator, base_id: str, table: str) -> Dict[st else: raise Exception(f"Error getting first row from table {table}: {e}") json_response = response.json() - record = json_response.get("records", [])[0] - return record + records = json_response.get("records", []) + most_complete_row = records[0] + for record in records: + if len(record.keys()) > len(most_complete_row.keys()): + most_complete_row = record + return most_complete_row @staticmethod def get_json_schema(record: Dict[str, Any]) -> Dict[str, str]: diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/source.py b/airbyte-integrations/connectors/source-airtable/source_airtable/source.py index 32f17b1590a4..8d9617dc0fed 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/source.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/source.py @@ -75,7 +75,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: auth = TokenAuthenticator(token=config["api_key"]) for table in config["tables"]: try: - Helpers.get_first_row(auth, config["base_id"], table) + Helpers.get_most_complete_row(auth, config["base_id"], table) except Exception as e: return False, str(e) return True, None @@ -84,7 +84,7 @@ def discover(self, logger: AirbyteLogger, config) -> AirbyteCatalog: streams = [] auth = TokenAuthenticator(token=config["api_key"]) for table in config["tables"]: - record = Helpers.get_first_row(auth, config["base_id"], table) + record = Helpers.get_most_complete_row(auth, config["base_id"], table) json_schema = Helpers.get_json_schema(record) airbyte_stream = Helpers.get_airbyte_stream(table, json_schema) streams.append(airbyte_stream) @@ -94,7 +94,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = TokenAuthenticator(token=config["api_key"]) streams = [] for table in config["tables"]: - record = Helpers.get_first_row(auth, config["base_id"], table) + record = Helpers.get_most_complete_row(auth, config["base_id"], table) json_schema = Helpers.get_json_schema(record) stream = AirtableStream(base_id=config["base_id"], table_name=table, authenticator=auth, schema=json_schema) streams.append(stream) diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/spec.json b/airbyte-integrations/connectors/source-airtable/source_airtable/spec.json index b032dd767d47..3f5bfddbd407 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/spec.json +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/spec.json @@ -5,7 +5,6 @@ "title": "Airtable Source Spec", "type": "object", "required": ["api_key", "base_id", "tables"], - "additionalProperties": false, "properties": { "api_key": { "type": "string", diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/test_helpers.py b/airbyte-integrations/connectors/source-airtable/unit_tests/test_helpers.py index 589c0dcfe99c..8641d2d9339d 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/test_helpers.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/test_helpers.py @@ -48,24 +48,24 @@ def expected_json_schema(): } -def test_get_first_row(auth, base_id, table, json_response): +def test_get_most_complete_row(auth, base_id, table, json_response): with patch("requests.get") as mock_get: mock_get.return_value.status_code = HTTPStatus.OK mock_get.return_value.json.return_value = json_response - assert Helpers.get_first_row(auth, base_id, table) == {"id": "abc", "fields": {"name": "test"}} + assert Helpers.get_most_complete_row(auth, base_id, table) == {"id": "abc", "fields": {"name": "test"}} -def test_get_first_row_invalid_api_key(base_id, table): +def test_get_most_complete_row_invalid_api_key(base_id, table): with pytest.raises(Exception): auth = TokenAuthenticator("invalid_api_key") - Helpers.get_first_row(auth, base_id, table) + Helpers.get_most_complete_row(auth, base_id, table) -def test_get_first_row_table_not_found(auth, base_id, table): +def test_get_most_complete_row_table_not_found(auth, base_id, table): with patch("requests.exceptions.HTTPError") as mock_get: mock_get.return_value.status_code = HTTPStatus.NOT_FOUND with pytest.raises(Exception): - Helpers.get_first_row(auth, base_id, table) + Helpers.get_most_complete_row(auth, base_id, table) def test_get_json_schema(json_response, expected_json_schema): diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/test_source.py b/airbyte-integrations/connectors/source-airtable/unit_tests/test_source.py index 33adb61820f6..14f30ce4cf58 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/test_source.py @@ -18,11 +18,11 @@ def test_spec(config): def test_discover(config, mocker): source = SourceAirtable() - logger_mock, Helpers.get_first_row = MagicMock(), MagicMock() + logger_mock, Helpers.get_most_complete_row = MagicMock(), MagicMock() airbyte_catalog = source.discover(logger_mock, config) assert [stream.name for stream in airbyte_catalog.streams] == config["tables"] assert isinstance(airbyte_catalog, AirbyteCatalog) - assert Helpers.get_first_row.call_count == 2 + assert Helpers.get_most_complete_row.call_count == 2 @patch("requests.get") @@ -34,7 +34,7 @@ def test_check_connection(config): def test_streams(config): source = SourceAirtable() - Helpers.get_first_row = MagicMock() + Helpers.get_most_complete_row = MagicMock() streams = source.streams(config) assert len(streams) == 2 assert [stream.name for stream in streams] == config["tables"] diff --git a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile index 8410e032a8cd..4eae2bdf2c0a 100644 --- a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-alloydb-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.15 +LABEL io.airbyte.version=1.0.16 LABEL io.airbyte.name=airbyte/source-alloydb-strict-encrypt diff --git a/airbyte-integrations/connectors/source-alloydb/Dockerfile b/airbyte-integrations/connectors/source-alloydb/Dockerfile index 284a1d21d21f..7bffadf916b1 100644 --- a/airbyte-integrations/connectors/source-alloydb/Dockerfile +++ b/airbyte-integrations/connectors/source-alloydb/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-alloydb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.15 +LABEL io.airbyte.version=1.0.16 LABEL io.airbyte.name=airbyte/source-alloydb diff --git a/airbyte-integrations/connectors/source-asana/setup.py b/airbyte-integrations/connectors/source-asana/setup.py index 41cee5e6fc12..348d94cc679c 100644 --- a/airbyte-integrations/connectors/source-asana/setup.py +++ b/airbyte-integrations/connectors/source-asana/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk~=0.2", ] TEST_REQUIREMENTS = ["pytest~=6.1", "requests-mock~=1.9.3", "source-acceptance-test"] diff --git a/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py b/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py index 7871fb5f4110..666c5aa26d9b 100644 --- a/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py @@ -2,6 +2,8 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from unittest.mock import PropertyMock, patch + from airbyte_cdk.logger import AirbyteLogger from source_asana.source import SourceAsana @@ -26,10 +28,11 @@ def test_check_connection_empty_config(config): def test_check_connection_exception(config): - ok, error_msg = SourceAsana().check_connection(logger, config=config) + with patch("source_asana.streams.Workspaces.use_cache", new_callable=PropertyMock, return_value=False): + ok, error_msg = SourceAsana().check_connection(logger, config=config) - assert not ok - assert error_msg + assert not ok + assert error_msg def test_streams(config): diff --git a/airbyte-integrations/connectors/source-ashby/.dockerignore b/airbyte-integrations/connectors/source-ashby/.dockerignore new file mode 100644 index 000000000000..0068973abcf8 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_ashby +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-ashby/Dockerfile b/airbyte-integrations/connectors/source-ashby/Dockerfile new file mode 100644 index 000000000000..7f1808573b73 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/Dockerfile @@ -0,0 +1,38 @@ +FROM --platform=linux/amd64 python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_ashby ./source_ashby + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-ashby diff --git a/airbyte-integrations/connectors/source-ashby/README.md b/airbyte-integrations/connectors/source-ashby/README.md new file mode 100644 index 000000000000..83803e0b944c --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/README.md @@ -0,0 +1,79 @@ +# Ashby Source + +This is the repository for the Ashby configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/ashby). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-ashby:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/ashby) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_ashby/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source ashby test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-ashby:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-ashby:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-ashby:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-ashby:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-ashby:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-ashby:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-ashby:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-ashby:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-ashby/__init__.py b/airbyte-integrations/connectors/source-ashby/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-ashby/acceptance-test-config.yml b/airbyte-integrations/connectors/source-ashby/acceptance-test-config.yml new file mode 100644 index 000000000000..f6e3ec2bb01c --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/acceptance-test-config.yml @@ -0,0 +1,20 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-ashby:dev +tests: + spec: + - spec_path: "source_ashby/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: ["users"] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-ashby/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-ashby/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-ashby/bootstrap.md b/airbyte-integrations/connectors/source-ashby/bootstrap.md new file mode 100644 index 000000000000..da30541a2583 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/bootstrap.md @@ -0,0 +1,13 @@ +# Ashby + +## Overview + +TO DO + +## Endpoints + +Ashby API consists of X endpoints which can be extracted data from: + +## API Reference + +The API reference documents: [https://developers.ashbyhq.com/reference/introduction](https://developers.ashbyhq.com/reference/introduction) diff --git a/airbyte-integrations/connectors/source-ashby/build.gradle b/airbyte-integrations/connectors/source-ashby/build.gradle new file mode 100644 index 000000000000..17e5df44150e --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_ashby' +} diff --git a/airbyte-integrations/connectors/source-ashby/integration_tests/__init__.py b/airbyte-integrations/connectors/source-ashby/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-ashby/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-ashby/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-ashby/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-ashby/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..3e9633bb7780 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/integration_tests/configured_catalog.json @@ -0,0 +1,130 @@ +{ + "streams": [ + { + "stream": { + "name": "applications", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "archive_reasons", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "candidate_tags", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "candidates", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "custom_fields", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "departments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "feedback_form_definitions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "interview_schedules", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "job_postings", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "jobs", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "locations", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "offers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "sources", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-ashby/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-ashby/integration_tests/invalid_config.json new file mode 100644 index 000000000000..fe3ad83b3f75 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "", + "start_date": "2022-10-25" +} diff --git a/airbyte-integrations/connectors/source-ashby/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-ashby/integration_tests/sample_config.json new file mode 100644 index 000000000000..e8e548e57cdd --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/integration_tests/sample_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "", + "start_date": "2022-10-25T00:00:00Z" +} diff --git a/airbyte-integrations/connectors/source-ashby/integration_tests/simple_catalog.json b/airbyte-integrations/connectors/source-ashby/integration_tests/simple_catalog.json new file mode 100644 index 000000000000..ed4bcd511a15 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/integration_tests/simple_catalog.json @@ -0,0 +1,13 @@ +{ + "streams": [ + { + "stream": { + "name": "applications", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-zoom-singer/main.py b/airbyte-integrations/connectors/source-ashby/main.py similarity index 68% rename from airbyte-integrations/connectors/source-zoom-singer/main.py rename to airbyte-integrations/connectors/source-ashby/main.py index 135a9790d88f..dd9c301cc14d 100644 --- a/airbyte-integrations/connectors/source-zoom-singer/main.py +++ b/airbyte-integrations/connectors/source-ashby/main.py @@ -6,8 +6,8 @@ import sys from airbyte_cdk.entrypoint import launch -from source_zoom_singer import SourceZoomSinger +from source_ashby import SourceAshby if __name__ == "__main__": - source = SourceZoomSinger() + source = SourceAshby() launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-ashby/requirements.txt b/airbyte-integrations/connectors/source-ashby/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-ashby/setup.py b/airbyte-integrations/connectors/source-ashby/setup.py new file mode 100644 index 000000000000..46db533a3953 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_ashby", + description="Source implementation for Ashby.", + author="Elliot Trabac", + author_email="elliot.trabac1@gmail.com", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/__init__.py b/airbyte-integrations/connectors/source-ashby/source_ashby/__init__.py new file mode 100644 index 000000000000..f88575f93250 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceAshby + +__all__ = ["SourceAshby"] diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/ashby.yaml b/airbyte-integrations/connectors/source-ashby/source_ashby/ashby.yaml new file mode 100644 index 000000000000..29090d6ed8c5 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/ashby.yaml @@ -0,0 +1,156 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: ["results"] + requester: + url_base: "https://api.ashbyhq.com" + http_method: "POST" + authenticator: + type: BasicHttpAuthenticator + username: "{{ config['api_key'] }}" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: DefaultPaginator + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response.nextCursor }}" + page_size: 100 + page_size_option: + field_name: "per_page" + inject_into: "body_json" + page_token_option: + inject_into: "body_json" + field_name: "cursor" + url_base: + $ref: "*ref(definitions.requester.url_base)" + requester: + $ref: "*ref(definitions.requester)" + + # base stream + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + + # stream definitions + applications_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "applications" + primary_key: "id" + path: "/application.list" + retriever: + $ref: "*ref(definitions.base_stream.retriever)" + requester: + $ref: "*ref(definitions.requester)" + request_options_provider: + request_body_json: + createdAfter: "{{ timestamp(config['start_date']) * 1000 }}" + archive_reasons_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "archive_reasons" + primary_key: "id" + path: "/archiveReason.list" + candidate_tags_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "candidate_tags" + primary_key: "id" + path: "/candidateTag.list" + candidates_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "candidates" + primary_key: "id" + path: "/candidate.list" + custom_fields_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "custom_fields" + primary_key: "id" + path: "/customField.list" + departments_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "departments" + primary_key: "id" + path: "/department.list" + feedback_form_definitions_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "feedback_form_definitions" + primary_key: "id" + path: "/feedbackFormDefinition.list" + interview_schedules_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "interview_schedules" + primary_key: "id" + path: "/interviewSchedule.list" + retriever: + $ref: "*ref(definitions.base_stream.retriever)" + requester: + $ref: "*ref(definitions.requester)" + request_options_provider: + request_body_json: + createdAfter: "{{ timestamp(config['start_date']) * 1000 }}" + job_postings_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "job_postings" + primary_key: "id" + path: "/jobPosting.list" + jobs_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "jobs" + primary_key: "id" + path: "/job.list" + locations_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "locations" + primary_key: "id" + path: "/location.list" + offers_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "offers" + primary_key: "id" + path: "/offer.list" + sources_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "sources" + primary_key: "id" + path: "/source.list" + users_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "users" + primary_key: "id" + path: "/user.list" + +streams: + - "*ref(definitions.applications_stream)" + - "*ref(definitions.archive_reasons_stream)" + - "*ref(definitions.candidate_tags_stream)" + - "*ref(definitions.candidates_stream)" + - "*ref(definitions.custom_fields_stream)" + - "*ref(definitions.departments_stream)" + - "*ref(definitions.feedback_form_definitions_stream)" + - "*ref(definitions.interview_schedules_stream)" + - "*ref(definitions.job_postings_stream)" + - "*ref(definitions.jobs_stream)" + - "*ref(definitions.locations_stream)" + - "*ref(definitions.offers_stream)" + - "*ref(definitions.sources_stream)" + - "*ref(definitions.users_stream)" + +check: + stream_names: + - "users" diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/applications.json b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/applications.json new file mode 100644 index 000000000000..bc8fac3339fc --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/applications.json @@ -0,0 +1,41 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "candidate": { + "type": ["null", "object"] + }, + "status": { + "type": ["null", "string"] + }, + "creditedToUser": { + "type": ["null", "object"] + }, + "customFields": { + "type": ["null", "array"] + }, + "currentInterviewStage": { + "type": ["null", "object"] + }, + "archiveReason": { + "type": ["null", "string"] + }, + "job": { + "type": ["null", "object"] + }, + "hiringTeam": { + "type": ["null", "array"] + } + } +} diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/archive_reasons.json b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/archive_reasons.json new file mode 100644 index 000000000000..e67645b5aecc --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/archive_reasons.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "text": { + "type": ["null", "string"] + }, + "reasonType": { + "type": ["null", "string"] + }, + "isArchived": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/candidate_tags.json b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/candidate_tags.json new file mode 100644 index 000000000000..5d201e06e57e --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/candidate_tags.json @@ -0,0 +1,15 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "isArchived": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/candidates.json b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/candidates.json new file mode 100644 index 000000000000..03c5c4a3e0f7 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/candidates.json @@ -0,0 +1,48 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "primaryEmailAddress": { + "type": ["null", "object"] + }, + "emailAddresses": { + "type": ["null", "array"] + }, + "phoneNumbers": { + "type": ["null", "array"] + }, + "socialLinks": { + "type": ["null", "array"] + }, + "tags": { + "type": ["null", "array"] + }, + "position": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "school": { + "type": ["null", "string"] + }, + "applicationIds": { + "type": ["null", "array"] + }, + "resumeFileHandle": { + "type": ["null", "object"] + }, + "fileHandles": { + "type": ["null", "array"] + }, + "customFields": { + "type": ["null", "array"] + } + } +} diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/custom_fields.json b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/custom_fields.json new file mode 100644 index 000000000000..2c240e698f2f --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/custom_fields.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "objectType": { + "type": ["null", "string"] + }, + "isArchived": { + "type": ["null", "boolean"] + }, + "fieldType": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/departments.json b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/departments.json new file mode 100644 index 000000000000..7a453d7b2911 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/departments.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "parentId": { + "type": ["null", "string"] + }, + "isArchived": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/feedback_form_definitions.json b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/feedback_form_definitions.json new file mode 100644 index 000000000000..2c7d16b395f0 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/feedback_form_definitions.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "organizationId": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "isArchived": { + "type": ["null", "boolean"] + }, + "isDefaultForm": { + "type": ["null", "boolean"] + }, + "formDefinition": { + "type": ["null", "object"] + }, + "interviewId": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/interview_schedules.json b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/interview_schedules.json new file mode 100644 index 000000000000..2deb1e63f40c --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/interview_schedules.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "applicationId": { + "type": ["null", "string"] + }, + "interviewStageId": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/job_postings.json b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/job_postings.json new file mode 100644 index 000000000000..3f0fbbb984bb --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/job_postings.json @@ -0,0 +1,40 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "jobId": { + "type": ["null", "string"] + }, + "departmentName": { + "type": ["null", "string"] + }, + "teamName": { + "type": ["null", "string"] + }, + "locationName": { + "type": ["null", "string"] + }, + "employmentType": { + "type": ["null", "string"] + }, + "isListed": { + "type": ["null", "boolean"] + }, + "publishedDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "externalLink": { + "type": ["null", "string"] + }, + "locationIds": { + "type": ["null", "object"] + } + } +} diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/jobs.json b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/jobs.json new file mode 100644 index 000000000000..11761c082b27 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/jobs.json @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "confidential": { + "type": ["null", "boolean"] + }, + "status": { + "type": ["null", "string"] + }, + "employmentType": { + "type": ["null", "string"] + }, + "locationId": { + "type": ["null", "string"] + }, + "departmentId": { + "type": ["null", "string"] + }, + "defaultInterviewPlanId": { + "type": ["null", "string"] + }, + "interviewPlanIds": { + "type": ["null", "array"] + }, + "jobPostingIds": { + "type": ["null", "array"] + }, + "hiringTeam": { + "type": ["null", "array"] + }, + "customFields": { + "type": ["null", "array"] + }, + "customRequisitionId": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/locations.json b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/locations.json new file mode 100644 index 000000000000..87afaa151049 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/locations.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "isArchived": { + "type": ["null", "boolean"] + }, + "address": { + "type": ["null", "object"] + }, + "isRemote": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/offers.json b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/offers.json new file mode 100644 index 000000000000..7dbaac2537f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/offers.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "decidedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "applicationId": { + "type": ["null", "string"] + }, + "acceptanceStatus": { + "type": ["null", "string"] + }, + "latestVersion": { + "type": ["null", "object"] + } + } +} diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/sources.json b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/sources.json new file mode 100644 index 000000000000..5d201e06e57e --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/sources.json @@ -0,0 +1,15 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "isArchived": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/users.json b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/users.json new file mode 100644 index 000000000000..bd9e96ef5e67 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/schemas/users.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "firstName": { + "type": ["null", "string"] + }, + "lastName": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/source.py b/airbyte-integrations/connectors/source-ashby/source_ashby/source.py new file mode 100644 index 000000000000..80b47fd139ea --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceAshby(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "ashby.yaml"}) diff --git a/airbyte-integrations/connectors/source-ashby/source_ashby/spec.yaml b/airbyte-integrations/connectors/source-ashby/source_ashby/spec.yaml new file mode 100644 index 000000000000..2f8f423ce856 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/source_ashby/spec.yaml @@ -0,0 +1,24 @@ +documentationUrl: https://developers.ashbyhq.com/reference/introduction +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Ashby Spec + type: object + required: + - api_key + - start_date + additionalProperties: true + properties: + api_key: + type: string + title: Ashby API key + description: The Ashby API Key, see doc here. + airbyte_secret: true + start_date: + type: string + title: Start date + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + description: >- + UTC date and time in the format 2017-01-25T00:00:00Z. Any data before + this date will not be replicated. + examples: + - "2017-01-25T00:00:00Z" diff --git a/airbyte-integrations/connectors/source-ashby/unit_tests/test_dummy.py b/airbyte-integrations/connectors/source-ashby/unit_tests/test_dummy.py new file mode 100644 index 000000000000..f1f977513d63 --- /dev/null +++ b/airbyte-integrations/connectors/source-ashby/unit_tests/test_dummy.py @@ -0,0 +1,10 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +def test_dummy(): + """ + Dummy test to prevent gradle from failing test for this connector + """ + assert True diff --git a/airbyte-integrations/connectors/source-auth0/.dockerignore b/airbyte-integrations/connectors/source-auth0/.dockerignore new file mode 100644 index 000000000000..ad33b28741d3 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_auth0 +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-auth0/Dockerfile b/airbyte-integrations/connectors/source-auth0/Dockerfile new file mode 100644 index 000000000000..d3e82093abba --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.13-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_auth0 ./source_auth0 + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-auth0 diff --git a/airbyte-integrations/connectors/source-auth0/README.md b/airbyte-integrations/connectors/source-auth0/README.md new file mode 100644 index 000000000000..684853a8716e --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/README.md @@ -0,0 +1,132 @@ +# Auth0 Source + +This is the repository for the Auth0 source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/auth0). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-auth0:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/auth0) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_auth0/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config_access-token.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source auth0 test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-auth0:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-auth0:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-auth0:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-auth0:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-auth0:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-auth0:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-auth0:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-auth0:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-auth0/acceptance-test-config.yml b/airbyte-integrations/connectors/source-auth0/acceptance-test-config.yml new file mode 100644 index 000000000000..e9b68ceeea84 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/acceptance-test-config.yml @@ -0,0 +1,22 @@ +connector_image: airbyte/source-auth0:dev +tests: + spec: + - spec_path: "source_auth0/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-auth0/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-auth0/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-auth0/build.gradle b/airbyte-integrations/connectors/source-auth0/build.gradle new file mode 100644 index 000000000000..f493b96498ec --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_auth0' +} diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/__init__.py b/airbyte-integrations/connectors/source-auth0/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-auth0/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..3237ca3c641f --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/abnormal_state.json @@ -0,0 +1,3 @@ +{ + "users": { "updated_at": "3021-09-08T07:04:28.000Z" } +} diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-auth0/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/catalog.json b/airbyte-integrations/connectors/source-auth0/integration_tests/catalog.json new file mode 100644 index 000000000000..c9582a1dce62 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/catalog.json @@ -0,0 +1,14 @@ +{ + "streams": [ + { + "stream": { + "name": "users", + "json_schema": {} + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["updated_at"], + "primary_key": [["user_id"]] + } + ] +} diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-auth0/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..cac1ca9af2e3 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/configured_catalog.json @@ -0,0 +1,15 @@ +{ + "streams": [ + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["updated_at"], + "primary_key": [["user_id"]] + } + ] +} diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-auth0/integration_tests/invalid_config.json new file mode 100644 index 000000000000..828d82b44254 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/invalid_config.json @@ -0,0 +1,7 @@ +{ + "base_url": "https://dev-kxzbr15d.us.auth0.com/", + "credentials": { + "auth_type": "oauth2_access_token", + "access_token": "Invalid-token" + } +} diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/sample_config_access-token.json b/airbyte-integrations/connectors/source-auth0/integration_tests/sample_config_access-token.json new file mode 100644 index 000000000000..73af3bf1822c --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/sample_config_access-token.json @@ -0,0 +1,7 @@ +{ + "base_url": "https://dev-yourOrg.us.auth0.com", + "credentials": { + "auth_type": "oauth2_access_token", + "access_token": "api-key-just-for-testing" + } +} diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/sample_config_oauth2-confidential.json b/airbyte-integrations/connectors/source-auth0/integration_tests/sample_config_oauth2-confidential.json new file mode 100644 index 000000000000..397458e958d4 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/sample_config_oauth2-confidential.json @@ -0,0 +1,9 @@ +{ + "base_url": "https://dev-your-org.us.auth0.com", + "credentials": { + "auth_type": "oauth2_confidential_application", + "client_secret": "top-secret", + "client_id": "click-copy-icon-from-applications-page", + "audience": "https://dev-your-org.us.auth0.com/api/v2/" + } +} diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-auth0/integration_tests/sample_state.json new file mode 100644 index 000000000000..b9b68f1828f3 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/sample_state.json @@ -0,0 +1,3 @@ +{ + "users": { "updated_at": "2021-09-08T07:04:28.000Z" } +} diff --git a/airbyte-integrations/connectors/source-auth0/main.py b/airbyte-integrations/connectors/source-auth0/main.py new file mode 100644 index 000000000000..967d18925949 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_auth0 import SourceAuth0 + +if __name__ == "__main__": + source = SourceAuth0() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-auth0/requirements.txt b/airbyte-integrations/connectors/source-auth0/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-auth0/setup.py b/airbyte-integrations/connectors/source-auth0/setup.py new file mode 100644 index 000000000000..4c1aea85c909 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.2", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_auth0", + description="Source implementation for Auth0.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/__init__.py b/airbyte-integrations/connectors/source-auth0/source_auth0/__init__.py new file mode 100644 index 000000000000..2bf07f06358d --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/source_auth0/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceAuth0 + +__all__ = ["SourceAuth0"] diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/authenticator.py b/airbyte-integrations/connectors/source-auth0/source_auth0/authenticator.py new file mode 100644 index 000000000000..07da20173c22 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/source_auth0/authenticator.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import logging +from typing import Any, Mapping +from urllib import parse + +from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator + +logger = logging.getLogger("airbyte") + + +class Auth0Oauth2Authenticator(Oauth2Authenticator): + def __init__(self, base_url: str, audience: str, client_id: str, client_secret: str): + super().__init__(parse.urljoin(base_url, "/oauth/token"), client_id, client_secret, "") + self.audience = audience.rstrip("/") + "/" + + def build_refresh_request_body(self) -> Mapping[str, Any]: + if not self.get_refresh_token(): + return { + "grant_type": "client_credentials", + "client_id": self.get_client_id(), + "client_secret": self.get_client_secret(), + "audience": self.audience, + } + else: + return super().build_refresh_request_body() diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/users.json b/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/users.json new file mode 100644 index 000000000000..5f1a13dbd812 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/users.json @@ -0,0 +1,81 @@ +{ + "type": ["object", "null"], + "additionalProperties": true, + "properties": { + "_links": { + "type": ["object", "null"] + }, + "user_id": { + "type": [ "string", "null"] + }, + "username": { + "type": [ "string", "null"] + }, + "email": { + "type": [ "string", "null"] + }, + "email_verified": { + "type": ["boolean", "null"] + }, + "phone_number": { + "type": ["string", "null"] + }, + "phone_verified": { + "type": ["boolean", "null"] + }, + "created_at": { + "format": "date-time", + "type": ["string", "null"] + }, + "updated_at": { + "format": "date-time", + "type": ["string", "null"] + }, + "identities": { + "type": "array", + "items": { + "type": ["object", "null"], + "additionalProperties": true + } + }, + "app_metadata": { + "type": ["object", "null"], + "additionalProperties": true + }, + "user_metadata": { + "type": ["object", "null"], + "additionalProperties": true + }, + "picture": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "nickname": { + "type": ["string", "null"] + }, + "multifactor": { + "type": ["object", "null"], + "additionalProperties": true + }, + "last_ip": { + "type": ["string", "null"] + }, + "logins_count": { + "type": ["integer", "null"] + }, + "blocked": { + "type": ["boolean", "null"] + }, + "last_login": { + "type": ["string", "null"] + }, + "given_name": { + "type": ["string", "null"] + }, + "family_name": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/source.py b/airbyte-integrations/connectors/source-auth0/source_auth0/source.py new file mode 100644 index 000000000000..0df4c6752549 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/source_auth0/source.py @@ -0,0 +1,151 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import logging +from abc import ABC, abstractmethod +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from urllib import parse + +import pendulum +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import IncrementalMixin, Stream +from airbyte_cdk.sources.streams.http import HttpStream +from source_auth0.utils import get_api_endpoint, initialize_authenticator + + +# Basic full refresh stream +class Auth0Stream(HttpStream, ABC): + api_version = "v2" + page_size = 50 + resource_name = "entities" + + def __init__(self, url_base: str, *args, **kwargs): + super().__init__(*args, **kwargs) + self.api_endpoint = get_api_endpoint(url_base, self.api_version) + + def path(self, **kwargs) -> str: + return self.resource_name + + @property + def url_base(self) -> str: + return self.api_endpoint + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + body = response.json() + if "total" in body and "start" in body and "limit" in body and "length" in body: + try: + start = int(body["start"]) + limit = int(body["limit"]) + length = int(body["length"]) + total = int(body["total"]) + current = start // limit + if length < limit or (start + length) == total: + return None + else: + token = { + "page": current + 1, + "per_page": limit, + } + return token + except Exception: + return None + else: + if not body or len(body) < self.page_size: + return None + else: + return { + "page": 0, + "per_page": self.page_size, + } + + def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + return { + "page": 0, + "per_page": self.page_size, + "include_totals": "true", + **(next_page_token or {}), + } + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + yield from response.json().get(self.resource_name) + + def backoff_time(self, response: requests.Response) -> Optional[float]: + # The rate limit resets on the timestamp indicated + # https://auth0.com/docs/troubleshoot/customer-support/operational-policies/rate-limit-policy/management-api-endpoint-rate-limits + if response.status_code == requests.codes.TOO_MANY_REQUESTS: + next_reset_epoch = int(response.headers["x-ratelimit-reset"]) + next_reset = pendulum.from_timestamp(next_reset_epoch) + next_reset_duration = pendulum.now("UTC").diff(next_reset) + return next_reset_duration.seconds + + +class IncrementalAuth0Stream(Auth0Stream, IncrementalMixin): + min_id = "" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._cursor_value = self.min_id + + @property + @abstractmethod + def cursor_field(self) -> str: + pass + + @property + def state(self) -> MutableMapping[str, Any]: + return {self.cursor_field: self._cursor_value} + + @state.setter + def state(self, value: MutableMapping[str, Any]): + self._cursor_value = value.get(self.cursor_field) + + def request_params( + self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=self.state, next_page_token=next_page_token, **kwargs) + latest_entry = self.state.get(self.cursor_field) + filter_param = {"include_totals": "false", "sort": f"{self.cursor_field}:1", "q": f"{self.cursor_field}:{{{latest_entry} TO *]"} + params.update(filter_param) + return params + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + entities = response.json() + if entities: + last_item = entities[-1] + self.state = last_item + yield from entities + + +class Users(IncrementalAuth0Stream): + min_id = "1900-01-01T00:00:00.000Z" + primary_key = "user_id" + resource_name = "users" + cursor_field = "updated_at" + + +# Source +class SourceAuth0(AbstractSource): + def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, any]: + try: + auth = initialize_authenticator(config) + api_endpoint = get_api_endpoint(config.get("base_url"), "v2") + url = parse.urljoin(api_endpoint, "users") + response = requests.get( + url, + params={"per_page": 1}, + headers=auth.get_auth_header(), + ) + + if response.status_code == requests.codes.ok: + return True, None + + return False, response.json() + except Exception: + return False, "Failed to authenticate with the provided credentials" + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + initialization_params = {"authenticator": initialize_authenticator(config), "url_base": config.get("base_url")} + return [Users(**initialization_params)] diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/spec.yaml b/airbyte-integrations/connectors/source-auth0/source_auth0/spec.yaml new file mode 100644 index 000000000000..c697881823b5 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/source_auth0/spec.yaml @@ -0,0 +1,79 @@ +documentationUrl: https://auth0.com/docs/api/management/v2/ +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Auth0 Management API Spec + type: object + required: + - base_url + - credentials + additionalProperties: true + properties: + base_url: + type: string + title: Base URL + examples: + - "https://dev-yourOrg.us.auth0.com/" + description: The Authentication API is served over HTTPS. All URLs referenced in the documentation have the following base `https://YOUR_DOMAIN` + credentials: + title: Authentication Method + type: object + oneOf: + - type: object + title: OAuth2 Confidential Application + required: + - auth_type + - client_id + - client_secret + - audience + properties: + auth_type: + type: string + title: Authentication Method + const: oauth2_confidential_application + order: 0 + client_id: + title: Client ID + description: >- + Your application's Client ID. You can find this value on the application's settings tab + after you login the admin portal. + type: string + examples: + - "Client_ID" + client_secret: + title: Client Secret + description: >- + Your application's Client Secret. You can find this value on the application's settings tab + after you login the admin portal. + type: string + examples: + - "Client_Secret" + airbyte_secret: true + audience: + title: Audience + description: >- + The audience for the token, which is your API. You can find this in the Identifier field on your API's settings tab + type: string + examples: + - https://dev-yourOrg.us.auth0.com/api/v2/ + - type: object + title: OAuth2 Access Token + required: + - access_token + - auth_type + properties: + auth_type: + type: string + title: Authentication Method + const: oauth2_access_token + order: 0 + access_token: + title: OAuth2 Access Token + description: >- + Also called API Access Token + The access token used to call the Auth0 Management API Token. It's a JWT that contains specific grant permissions knowns as scopes. + type: string + airbyte_secret: true \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/utils.py b/airbyte-integrations/connectors/source-auth0/source_auth0/utils.py new file mode 100644 index 000000000000..620a37d8dbcf --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/source_auth0/utils.py @@ -0,0 +1,44 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import datetime +import logging +from typing import Dict +from urllib import parse + +from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator +from requests.auth import AuthBase + +from .authenticator import Auth0Oauth2Authenticator + +logger = logging.getLogger("airbyte") + + +def get_api_endpoint(url_base: str, version: str) -> str: + return parse.urljoin(url_base, f"/api/{version}/") + + +def initialize_authenticator(config: Dict) -> AuthBase: + credentials = config.get("credentials") + if not credentials: + raise Exception("Config validation error. `credentials` not specified.") + + auth_type = credentials.get("auth_type") + if not auth_type: + raise Exception("Config validation error. `auth_type` not specified.") + + if auth_type == "oauth2_access_token": + return TokenAuthenticator(credentials.get("access_token")) + + if auth_type == "oauth2_confidential_application": + return Auth0Oauth2Authenticator( + base_url=config.get("base_url"), + audience=credentials.get("audience"), + client_secret=credentials.get("client_secret"), + client_id=credentials.get("client_id"), + ) + + +def datetime_to_string(date: datetime.datetime) -> str: + return date.strftime("%Y-%m-%dT%H:%M:%S.000Z") diff --git a/airbyte-integrations/connectors/source-auth0/unit_tests/conftest.py b/airbyte-integrations/connectors/source-auth0/unit_tests/conftest.py new file mode 100644 index 000000000000..63a7a01ff9f2 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/unit_tests/conftest.py @@ -0,0 +1,142 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import pendulum +import pytest + + +@pytest.fixture() +def url_base(): + """ + URL base for test + """ + return "https://dev-yourOrg.us.auth0.com" + + +@pytest.fixture() +def api_url(url_base): + """ + Just return API url based on url_base + """ + return f"{url_base}/api/v2" + + +@pytest.fixture() +def oauth_config(url_base): + """ + Credentials for oauth2.0 authorization + """ + return { + "credentials": { + "auth_type": "oauth2_confidential_application", + "client_secret": "test_client_secret", + "client_id": "test_client_id", + "audience": f"{url_base}/api/v2", + }, + "base_url": url_base, + } + + +@pytest.fixture() +def wrong_oauth_config_bad_credentials_record(url_base): + """ + Malformed Credentials for oauth2.0 authorization + credentials -> credential + """ + return { + "credential": { + "auth_type": "oauth2.0", + "client_secret": "test_client_secret", + "client_id": "test_client_id", + }, + "base_url": url_base, + } + + +@pytest.fixture() +def wrong_oauth_config_bad_auth_type(url_base): + """ + Wrong Credentials format for oauth2.0 authorization + absent "auth_type" field + """ + return { + "credentials": { + "client_secret": "test_client_secret", + "client_id": "test_client_id", + "refresh_token": "test_refresh_token", + }, + "base_url": url_base, + } + + +@pytest.fixture() +def token_config(url_base): + """ + Just test 'token' + """ + return { + "credentials": {"auth_type": "oauth2_access_token", "access_token": "test-token"}, + "base_url": url_base, + } + + +@pytest.fixture() +def user_status_filter(): + statuses = ["ACTIVE", "DEPROVISIONED", "LOCKED_OUT", "PASSWORD_EXPIRED", "PROVISIONED", "RECOVERY", "STAGED", "SUSPENDED"] + return " or ".join([f'status eq "{status}"' for status in statuses]) + + +@pytest.fixture() +def users_instance(): + """ + Users instance object response + """ + return { + "blocked": False, + "created_at": "2022-10-21T04:10:34.240Z", + "email": "rodrick_waelchi73@yahoo.com", + "email_verified": False, + "family_name": "Kerluke", + "given_name": "Nick", + "identities": [ + { + "user_id": "15164a44-8064-4ef9-ac31-fb08814da3f9", + "connection": "Username-Password-Authentication", + "provider": "auth0", + "isSocial": False, + } + ], + "name": "Linda Sporer IV", + "nickname": "Marty", + "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", + "updated_at": "2022-10-21T04:10:34.240Z", + "user_id": "auth0|15164a44-8064-4ef9-ac31-fb08814da3f9", + "user_metadata": {}, + "app_metadata": {}, + } + + +@pytest.fixture() +def latest_record_instance(): + """ + Last Record instance object response + """ + return { + "id": "test_user_group_id", + "created": "2022-07-18T07:58:11.000Z", + "lastUpdated": "2022-07-18T07:58:11.000Z", + } + + +@pytest.fixture() +def error_failed_to_authorize_with_provided_credentials(): + """ + Error raised when using incorrect oauth2.0 credentials + """ + return "Failed to authenticate with the provided credentials" + + +@pytest.fixture() +def start_date(): + return pendulum.parse("2021-03-21T20:49:13Z") diff --git a/airbyte-integrations/connectors/source-auth0/unit_tests/test_source.py b/airbyte-integrations/connectors/source-auth0/unit_tests/test_source.py new file mode 100644 index 000000000000..54b46c95fd58 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/unit_tests/test_source.py @@ -0,0 +1,76 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator +from source_auth0.authenticator import Auth0Oauth2Authenticator +from source_auth0.source import SourceAuth0, Users, initialize_authenticator + + +class TestAuthentication: + def test_init_token_authentication_init(self, token_config): + token_auth = initialize_authenticator(config=token_config) + assert isinstance(token_auth, TokenAuthenticator) + + def test_init_oauth2_authentication_init(self, oauth_config): + oauth_auth = initialize_authenticator(config=oauth_config) + assert isinstance(oauth_auth, Auth0Oauth2Authenticator) + + def test_init_oauth2_authentication_wrong_credentials_record(self, wrong_oauth_config_bad_credentials_record): + try: + initialize_authenticator(config=wrong_oauth_config_bad_credentials_record) + except Exception as e: + assert e.args[0] == "Config validation error. `credentials` not specified." + + def test_init_oauth2_authentication_wrong_oauth_config_bad_auth_type(self, wrong_oauth_config_bad_auth_type): + try: + initialize_authenticator(config=wrong_oauth_config_bad_auth_type) + except Exception as e: + assert e.args[0] == "Config validation error. `auth_type` not specified." + + def test_check_connection_ok(self, requests_mock, oauth_config, url_base): + oauth_auth = initialize_authenticator(config=oauth_config) + assert isinstance(oauth_auth, Auth0Oauth2Authenticator) + + source_auth0 = SourceAuth0() + requests_mock.get(f"{url_base}/api/v2/users?per_page=1", json={"connect": "ok"}) + requests_mock.post(f"{url_base}/oauth/token", json={"access_token": "test_token", "expires_in": 948}) + assert source_auth0.check_connection(logger=MagicMock(), config=oauth_config) == (True, None) + + def test_check_connection_error_status_code(self, requests_mock, oauth_config, url_base): + oauth_auth = initialize_authenticator(config=oauth_config) + assert isinstance(oauth_auth, Auth0Oauth2Authenticator) + + source_auth0 = SourceAuth0() + requests_mock.get(f"{url_base}/api/v2/users?per_page=1", status_code=400, json={}) + requests_mock.post(f"{url_base}/oauth/token", json={"access_token": "test_token", "expires_in": 948}) + + assert source_auth0.check_connection(logger=MagicMock(), config=oauth_config) == (False, {}) + + def test_check_connection_error_with_exception( + self, requests_mock, oauth_config, url_base, error_failed_to_authorize_with_provided_credentials + ): + oauth_auth = initialize_authenticator(config=oauth_config) + assert isinstance(oauth_auth, Auth0Oauth2Authenticator) + + source_auth0 = SourceAuth0() + requests_mock.get(f"{url_base}/api/v2/users?per_page=1", status_code=400, json="ss") + requests_mock.post(f"{url_base}/oauth/token", json={"access_token": "test_token", "expires_in": 948}) + + assert source_auth0.check_connection(logger=MagicMock(), config="wrong_config") == ( + False, + error_failed_to_authorize_with_provided_credentials, + ) + + def test_check_streams(self, requests_mock, oauth_config, api_url): + oauth_auth = initialize_authenticator(config=oauth_config) + assert isinstance(oauth_auth, Auth0Oauth2Authenticator) + + source_auth0 = SourceAuth0() + requests_mock.get(f"{api_url}/api/v2/users?per_page=1", json={"connect": "ok"}) + requests_mock.post(f"{api_url}/oauth/token", json={"access_token": "test_token", "expires_in": 948}) + streams = source_auth0.streams(config=oauth_config) + for i, _ in enumerate([Users]): + assert isinstance(streams[i], _) diff --git a/airbyte-integrations/connectors/source-auth0/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-auth0/unit_tests/test_streams.py new file mode 100644 index 000000000000..fbd153a4d6c3 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/unit_tests/test_streams.py @@ -0,0 +1,241 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import time +from abc import ABC +from unittest.mock import MagicMock + +import pytest +import requests +from airbyte_cdk.models import SyncMode +from source_auth0.source import Auth0Stream, IncrementalAuth0Stream, Users + + +@pytest.fixture +def patch_base_class(mocker): + """ + Base patcher for used streams + """ + mocker.patch.object(Auth0Stream, "primary_key", "test_primary_key") + mocker.patch.object(Auth0Stream, "__abstractmethods__", set()) + mocker.patch.object(IncrementalAuth0Stream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalAuth0Stream, "__abstractmethods__", set()) + + +class TestAuth0Stream: + def test_auth0_stream_request_params(self, patch_base_class, url_base): + stream = Auth0Stream(url_base=url_base) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = { + "page": 0, + "per_page": 50, + "include_totals": "true", + } + assert stream.request_params(**inputs) == expected_params + + def test_auth0_stream_parse_response(self, patch_base_class, requests_mock, url_base, api_url): + stream = Auth0Stream(url_base=url_base) + requests_mock.get(f"{api_url}", json={"entities": [{"a": 123}, {"b": "xx"}]}) + resp = requests.get(f"{api_url}") + inputs = {"response": resp, "stream_state": MagicMock()} + expected_parsed_object = [{"a": 123}, {"b": "xx"}] + assert list(stream.parse_response(**inputs)) == expected_parsed_object + + def test_auth0_stream_backoff_time(self, patch_base_class, url_base): + response_mock = requests.Response() + stream = Auth0Stream(url_base=url_base) + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time + + def test_auth0_stream_incremental_request_params(self, patch_base_class, url_base): + stream = IncrementalAuth0Stream(url_base=url_base) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = { + "page": 0, + "per_page": 50, + "include_totals": "false", + "sort": "None:1", + "q": "None:{ TO *]", + } + assert stream.request_params(**inputs) == expected_params + + def test_incremental_auth0_stream_parse_response(self, patch_base_class, requests_mock, url_base, api_url): + stream = IncrementalAuth0Stream(url_base=url_base) + requests_mock.get(f"{api_url}", json=[{"a": 123}, {"b": "xx"}]) + resp = requests.get(f"{api_url}") + inputs = {"response": resp, "stream_state": MagicMock()} + expected_parsed_object = [{"a": 123}, {"b": "xx"}] + assert list(stream.parse_response(**inputs)) == expected_parsed_object + + def test_incremental_auth0_stream_backoff_time(self, patch_base_class, url_base): + response_mock = MagicMock() + stream = IncrementalAuth0Stream(url_base=url_base) + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time + + def test_auth0_stream_incremental_backoff_time_empty(self, patch_base_class, url_base): + stream = IncrementalAuth0Stream(url_base=url_base) + response = MagicMock(requests.Response) + response.status_code = 200 + expected_params = None + inputs = {"response": response} + assert stream.backoff_time(**inputs) == expected_params + + def test_auth0_stream_incremental_back_off_now(self, patch_base_class, url_base): + stream = IncrementalAuth0Stream(url_base=url_base) + response = MagicMock(requests.Response) + response.status_code = requests.codes.TOO_MANY_REQUESTS + response.headers = {"x-ratelimit-reset": int(time.time())} + expected_params = (0, 2) + inputs = {"response": response} + get_backoff_time = stream.backoff_time(**inputs) + assert expected_params[0] <= get_backoff_time <= expected_params[1] + + def test_auth0_stream_incremental_get_updated_state(self, patch_base_class, latest_record_instance, url_base): + class TestIncrementalAuth0Stream(IncrementalAuth0Stream, ABC): + cursor_field = "lastUpdated" + + stream = TestIncrementalAuth0Stream(url_base=url_base) + stream._cursor_field = "lastUpdated" + assert stream._cursor_value == "" + stream.state = {"lastUpdated": "123"} + assert stream._cursor_value == "123" + + def test_auth0_stream_http_method(self, patch_base_class, url_base): + stream = Auth0Stream(url_base=url_base) + expected_method = "GET" + assert stream.http_method == expected_method + + +class TestNextPageToken: + def test_next_page_token(self, patch_base_class, url_base): + stream = Auth0Stream(url_base=url_base) + json = { + "start": "0", + "limit": 50, + "length": "50", + "total": 51, + } + response = MagicMock(requests.Response) + response.json = MagicMock(return_value=json) + inputs = {"response": response} + expected_token = {"page": 1, "per_page": 50} + result = stream.next_page_token(**inputs) + assert result == expected_token + + def test_next_page_token_invalid_cursor(self, patch_base_class, url_base): + stream = Auth0Stream(url_base=url_base) + json = { + "start": "0", + "limit": 50, + "length": "abc", + "total": 51, + } + response = MagicMock(requests.Response) + response.json = MagicMock(return_value=json) + inputs = {"response": response} + expected_token = None + result = stream.next_page_token(**inputs) + assert result == expected_token + + def test_next_page_token_missing_cursor(self, patch_base_class, url_base): + stream = Auth0Stream(url_base=url_base) + json = { + "limit": 50, + "total": 51, + } + response = MagicMock(requests.Response) + response.json = MagicMock(return_value=json) + inputs = {"response": response} + expected_token = None + result = stream.next_page_token(**inputs) + assert result == expected_token + + def test_next_page_token_one_page_only(self, patch_base_class, url_base): + stream = Auth0Stream(url_base=url_base) + json = { + "start": 0, + "limit": 50, + "length": 1, + "total": 1, + } + response = MagicMock(requests.Response) + response.json = MagicMock(return_value=json) + inputs = {"response": response} + expected_token = None + result = stream.next_page_token(**inputs) + assert result == expected_token + + def test_next_page_token_last_page_incomplete(self, patch_base_class, url_base): + stream = Auth0Stream(url_base=url_base) + json = { + "start": "50", + "limit": 50, + "length": "1", + "total": 51, + } + response = MagicMock(requests.Response) + response.json = MagicMock(return_value=json) + inputs = {"response": response} + expected_token = None + result = stream.next_page_token(**inputs) + assert result == expected_token + + def test_next_page_token_last_page_complete(self, patch_base_class, url_base): + stream = Auth0Stream(url_base=url_base) + json = { + "start": "50", + "limit": 50, + "length": "50", + "total": 100, + } + response = MagicMock(requests.Response) + response.json = MagicMock(return_value=json) + inputs = {"response": response} + expected_token = None + result = stream.next_page_token(**inputs) + assert result == expected_token + + +class TestStreamUsers: + def test_stream_users(self, patch_base_class, users_instance, url_base, api_url, requests_mock): + stream = Users(url_base=url_base) + requests_mock.get( + f"{api_url}/users", + json=[users_instance], + ) + inputs = {"sync_mode": SyncMode.incremental} + assert list(stream.read_records(**inputs)) == [users_instance] + + def test_users_request_params_out_of_next_page_token(self, patch_base_class, url_base): + stream = Users(url_base=url_base) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = { + "include_totals": "false", + "page": 0, + "per_page": 50, + "q": "updated_at:{1900-01-01T00:00:00.000Z TO *]", + "sort": "updated_at:1", + } + assert stream.request_params(**inputs) == expected_params + + def test_users_source_request_params_have_next_cursor(self, patch_base_class, url_base): + stream = Users(url_base=url_base) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"page": 1, "per_page": 50}} + expected_params = { + "include_totals": "false", + "page": 1, + "per_page": 50, + "q": "updated_at:{1900-01-01T00:00:00.000Z TO *]", + "sort": "updated_at:1", + } + assert stream.request_params(**inputs) == expected_params + + def test_users_source_parse_response(self, requests_mock, patch_base_class, users_instance, url_base, api_url): + stream = Users(url_base=url_base) + requests_mock.get( + f"{api_url}/users", + json=[users_instance], + ) + assert list(stream.parse_response(response=requests.get(f"{api_url}/users"))) == [users_instance] diff --git a/airbyte-integrations/connectors/source-clockify/.dockerignore b/airbyte-integrations/connectors/source-clockify/.dockerignore new file mode 100644 index 000000000000..a5f918cda764 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_clockify +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-clockify/.gitignore b/airbyte-integrations/connectors/source-clockify/.gitignore new file mode 100644 index 000000000000..945168c8f81b --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/.gitignore @@ -0,0 +1,3 @@ +users.yml +projects.yml +schemas diff --git a/airbyte-integrations/connectors/source-clockify/Dockerfile b/airbyte-integrations/connectors/source-clockify/Dockerfile new file mode 100644 index 000000000000..a19cd8507ded --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.13-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_clockify ./source_clockify + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-clockify diff --git a/airbyte-integrations/connectors/source-clockify/README.md b/airbyte-integrations/connectors/source-clockify/README.md new file mode 100644 index 000000000000..0f4544e0896e --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/README.md @@ -0,0 +1,132 @@ +# Clockify Source + +This is the repository for the Clockify source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/clockify). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-clockify:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/clockify) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_clockify/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source clockify test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-clockify:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-clockify:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-clockify:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-clockify:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-clockify:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-clockify:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-clockify:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-clockify:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-clockify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-clockify/acceptance-test-config.yml new file mode 100644 index 000000000000..451c4f772c54 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/acceptance-test-config.yml @@ -0,0 +1,19 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-clockify:dev +tests: + spec: + - spec_path: "source_clockify/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-clockify/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-clockify/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-clockify/build.gradle b/airbyte-integrations/connectors/source-clockify/build.gradle new file mode 100644 index 000000000000..9fbbaa9b16f7 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_clockify' +} diff --git a/airbyte-integrations/connectors/source-clockify/integration_tests/__init__.py b/airbyte-integrations/connectors/source-clockify/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-clockify/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-clockify/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-clockify/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-clockify/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..a939aad354a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/integration_tests/configured_catalog.json @@ -0,0 +1,109 @@ +{ + "streams": [ + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "users", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": null, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "projects", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": null, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "clients", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": null, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "tags", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": null, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "user_groups", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": null, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "time_entries", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": null, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "tasks", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": null, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + } + ] +} diff --git a/airbyte-integrations/connectors/source-clockify/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-clockify/integration_tests/invalid_config.json new file mode 100644 index 000000000000..306220396277 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "", + "workspace_id": "" +} diff --git a/airbyte-integrations/connectors/source-clockify/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-clockify/integration_tests/sample_config.json new file mode 100644 index 000000000000..b5ef13de8006 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/integration_tests/sample_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "", + "workspace_id": "" +} diff --git a/airbyte-integrations/connectors/source-clockify/main.py b/airbyte-integrations/connectors/source-clockify/main.py new file mode 100644 index 000000000000..72002deb5c04 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_clockify import SourceClockify + +if __name__ == "__main__": + source = SourceClockify() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-clockify/requirements.txt b/airbyte-integrations/connectors/source-clockify/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-clockify/setup.py b/airbyte-integrations/connectors/source-clockify/setup.py new file mode 100644 index 000000000000..4e83a518e06e --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/setup.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.2.0", +] + +TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "source-acceptance-test", "responses"] + +setup( + name="source_clockify", + description="Source implementation for Clockify.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/__init__.py b/airbyte-integrations/connectors/source-clockify/source_clockify/__init__.py new file mode 100644 index 000000000000..952d1a5a6623 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceClockify + +__all__ = ["SourceClockify"] diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/clients.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/clients.json new file mode 100644 index 000000000000..00a10979e40c --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/clients.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/schema#", + "properties": { + "address": { + "type": ["null", "string"] + }, + "archived": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "note": { + "type": ["null", "string"] + }, + "workspaceId": { + "type": "string" + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/projects.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/projects.json new file mode 100644 index 000000000000..faa01b58a0fd --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/projects.json @@ -0,0 +1,173 @@ +{ + "$schema": "http://json-schema.org/schema#", + "properties": { + "archived": { + "type": "boolean" + }, + "billable": { + "type": "boolean" + }, + "budgetEstimate": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "integer" + }, + { + "properties": { + "estimate": { + "type": ["null", "string"] + }, + "type": { + "type": "string" + }, + "resetOption": { + "type": ["null", "string"] + }, + "active": { + "type": "boolean" + } + }, + "type": "object" + } + ] + }, + "clientId": { + "type": "string" + }, + "clientName": { + "type": "string" + }, + "color": { + "type": "string" + }, + "costRate": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "properties": { + "amount": { + "type": ["null", "string", "integer"] + }, + "currency": { + "type": ["null", "string"] + } + }, + "type": "object" + } + ] + }, + "duration": { + "type": "string" + }, + "estimate": { + "properties": { + "estimate": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "hourlyRate": { + "properties": { + "amount": { + "type": "integer" + }, + "currency": { + "type": "string" + } + }, + "type": "object" + }, + "id": { + "type": "string" + }, + "memberships": { + "items": { + "properties": { + "costRate": { + "type": "null" + }, + "hourlyRate": { + "anyOf": [ + { + "type": "null" + }, + { + "properties": { + "amount": { + "type": "integer" + }, + "currency": { + "type": "string" + } + }, + "type": "object" + } + ] + }, + "membershipStatus": { + "type": "string" + }, + "membershipType": { + "type": "string" + }, + "targetId": { + "type": "string" + }, + "userId": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "name": { + "type": "string" + }, + "note": { + "type": "string" + }, + "public": { + "type": "boolean" + }, + "template": { + "type": "boolean" + }, + "timeEstimate": { + "properties": { + "active": { + "type": "boolean" + }, + "estimate": { + "type": "string" + }, + "includeNonBillable": { + "type": "boolean" + }, + "resetOption": { + "type": ["null", "string"] + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "workspaceId": { + "type": "string" + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tags.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tags.json new file mode 100644 index 000000000000..75b53dd8cfea --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tags.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/schema#", + "properties": { + "archived": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "workspaceId": { + "type": "string" + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tasks.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tasks.json new file mode 100644 index 000000000000..441785586cce --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tasks.json @@ -0,0 +1,78 @@ +{ + "$schema": "http://json-schema.org/schema#", + "properties": { + "assigneeId": { + "type": ["null", "string"] + }, + "assigneeIds": { + "items": { + "type": "string" + }, + "type": "array" + }, + "billable": { + "type": "boolean" + }, + "costRate": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "properties": { + "amount": { + "type": ["null", "string", "integer"] + }, + "currency": { + "type": ["null", "string"] + } + }, + "type": "object" + } + ] + }, + "duration": { + "type": ["null", "string"] + }, + "estimate": { + "type": "string" + }, + "hourlyRate": { + "anyOf": [ + { + "type": "null" + }, + { + "properties": { + "amount": { + "type": "integer" + }, + "currency": { + "type": "string" + } + }, + "type": "object" + } + ] + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "projectId": { + "type": "string" + }, + "status": { + "type": "string" + }, + "userGroupIds": { + "type": "array" + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/time_entries.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/time_entries.json new file mode 100644 index 000000000000..8fb163736583 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/time_entries.json @@ -0,0 +1,66 @@ +{ + "$schema": "http://json-schema.org/schema#", + "properties": { + "billable": { + "type": "boolean" + }, + "customFieldValues": { + "type": "array" + }, + "description": { + "type": "string" + }, + "id": { + "type": "string" + }, + "isLocked": { + "type": "boolean" + }, + "kioskId": { + "type": ["null", "string"] + }, + "projectId": { + "type": ["null", "string"] + }, + "tagIds": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "taskId": { + "type": ["null", "string"] + }, + "timeInterval": { + "properties": { + "duration": { + "type": "string" + }, + "end": { + "type": "string" + }, + "start": { + "type": "string" + } + }, + "type": "object" + }, + "type": { + "type": "string" + }, + "userId": { + "type": "string" + }, + "workspaceId": { + "type": "string" + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/user_groups.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/user_groups.json new file mode 100644 index 000000000000..f7ffaae51e6b --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/user_groups.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/schema#", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "userIds": { + "items": { + "type": "string" + }, + "type": "array" + }, + "workspaceId": { + "type": "string" + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/users.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/users.json new file mode 100644 index 000000000000..7d3d6d27cbd7 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/users.json @@ -0,0 +1,133 @@ +{ + "$schema": "http://json-schema.org/schema#", + "properties": { + "activeWorkspace": { + "type": "string" + }, + "customFields": { + "type": "array" + }, + "defaultWorkspace": { + "type": "string" + }, + "email": { + "type": "string" + }, + "id": { + "type": "string" + }, + "memberships": { + "type": "array" + }, + "name": { + "type": "string" + }, + "profilePicture": { + "type": "string" + }, + "settings": { + "properties": { + "alerts": { + "type": "boolean" + }, + "approval": { + "type": "boolean" + }, + "collapseAllProjectLists": { + "type": "boolean" + }, + "dashboardPinToTop": { + "type": "boolean" + }, + "dashboardSelection": { + "type": "string" + }, + "dashboardViewType": { + "type": "string" + }, + "dateFormat": { + "type": "string" + }, + "groupSimilarEntriesDisabled": { + "type": "boolean" + }, + "isCompactViewOn": { + "type": "boolean" + }, + "lang": { + "type": "string" + }, + "longRunning": { + "type": "boolean" + }, + "multiFactorEnabled": { + "type": "boolean" + }, + "myStartOfDay": { + "type": "string" + }, + "onboarding": { + "type": "boolean" + }, + "projectListCollapse": { + "type": "integer" + }, + "projectPickerTaskFilter": { + "type": "boolean" + }, + "pto": { + "type": "boolean" + }, + "reminders": { + "type": "boolean" + }, + "scheduledReports": { + "type": "boolean" + }, + "scheduling": { + "type": "boolean" + }, + "sendNewsletter": { + "type": "boolean" + }, + "showOnlyWorkingDays": { + "type": "boolean" + }, + "summaryReportSettings": { + "properties": { + "group": { + "type": "string" + }, + "subgroup": { + "type": "string" + } + }, + "type": "object" + }, + "theme": { + "type": "string" + }, + "timeFormat": { + "type": "string" + }, + "timeTrackingManual": { + "type": "boolean" + }, + "timeZone": { + "type": "string" + }, + "weekStart": { + "type": "string" + }, + "weeklyUpdates": { + "type": "boolean" + } + }, + "type": "object" + }, + "status": { + "type": "string" + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/source.py b/airbyte-integrations/connectors/source-clockify/source_clockify/source.py new file mode 100644 index 000000000000..b4bcd649b241 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/source.py @@ -0,0 +1,33 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from typing import Any, List, Mapping, Tuple + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator + +from .streams import Clients, Projects, Tags, Tasks, TimeEntries, UserGroups, Users + + +# Source +class SourceClockify(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + try: + workspace_stream = Users( + authenticator=TokenAuthenticator(token=config["api_key"], auth_header="X-Api-Key", auth_method=""), + workspace_id=config["workspace_id"], + ) + next(workspace_stream.read_records(sync_mode=SyncMode.full_refresh)) + return True, None + except Exception as e: + return False, f"Please check that your API key and workspace id are entered correctly: {repr(e)}" + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + authenticator = TokenAuthenticator(token=config["api_key"], auth_header="X-Api-Key", auth_method="") + + args = {"authenticator": authenticator, "workspace_id": config["workspace_id"]} + + return [Users(**args), Projects(**args), Clients(**args), Tags(**args), UserGroups(**args), TimeEntries(**args), Tasks(**args)] diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/spec.json b/airbyte-integrations/connectors/source-clockify/source_clockify/spec.json new file mode 100644 index 000000000000..ecd182c8e160 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/spec.json @@ -0,0 +1,23 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/clockify", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Clockify Spec", + "type": "object", + "required": ["workspace_id", "api_key"], + "additionalProperties": true, + "properties": { + "workspace_id": { + "title": "Workspace Id", + "description": "WorkSpace Id", + "type": "string" + }, + "api_key": { + "title": "API Key", + "description": "You can get your api access_key here This API is Case Sensitive.", + "type": "string", + "airbyte_secret": true + } + } + } +} diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/streams.py b/airbyte-integrations/connectors/source-clockify/source_clockify/streams.py new file mode 100644 index 000000000000..4fb36a3977fa --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/streams.py @@ -0,0 +1,123 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, Mapping, MutableMapping, Optional + +import requests +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream +from requests.auth import AuthBase + + +class ClockifyStream(HttpStream, ABC): + url_base = "https://api.clockify.me/api/v1/" + page_size = 50 + page = 1 + primary_key = None + + def __init__(self, workspace_id: str, **kwargs): + super().__init__(**kwargs) + self.workspace_id = workspace_id + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + next_page = response.json() + self.page = self.page + 1 + if next_page: + return {"page": self.page} + + def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = { + "page-size": self.page_size, + } + + if next_page_token: + params.update(next_page_token) + + return params + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + yield from response.json() + + +class Users(ClockifyStream): + @property + def use_cache(self) -> bool: + return True + + def path(self, **kwargs) -> str: + return f"workspaces/{self.workspace_id}/users" + + +class Projects(ClockifyStream): + @property + def use_cache(self) -> bool: + return True + + def path(self, **kwargs) -> str: + return f"workspaces/{self.workspace_id}/projects" + + +class Clients(ClockifyStream): + def path(self, **kwargs) -> str: + return f"workspaces/{self.workspace_id}/clients" + + +class Tags(ClockifyStream): + def path(self, **kwargs) -> str: + return f"workspaces/{self.workspace_id}/tags" + + +class UserGroups(ClockifyStream): + def path(self, **kwargs) -> str: + return f"workspaces/{self.workspace_id}/user-groups" + + +class TimeEntries(HttpSubStream, ClockifyStream): + def __init__(self, authenticator: AuthBase, workspace_id: Mapping[str, Any], **kwargs): + super().__init__( + authenticator=authenticator, + workspace_id=workspace_id, + parent=Users(authenticator=authenticator, workspace_id=workspace_id, **kwargs), + ) + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + """ + self.authenticator (which should be used as the + authenticator for Users) is object of NoAuth() + + so self._session.auth is used instead + """ + users_stream = Users(authenticator=self._session.auth, workspace_id=self.workspace_id) + for user in users_stream.read_records(sync_mode=SyncMode.full_refresh): + yield {"user_id": user["id"]} + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + user_id = stream_slice["user_id"] + return f"workspaces/{self.workspace_id}/user/{user_id}/time-entries" + + +class Tasks(HttpSubStream, ClockifyStream): + def __init__(self, authenticator: AuthBase, workspace_id: Mapping[str, Any], **kwargs): + super().__init__( + authenticator=authenticator, + workspace_id=workspace_id, + parent=Projects(authenticator=authenticator, workspace_id=workspace_id, **kwargs), + ) + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + """ + self.authenticator (which should be used as the + authenticator for Projects) is object of NoAuth() + + so self._session.auth is used instead + """ + projects_stream = Projects(authenticator=self._session.auth, workspace_id=self.workspace_id) + for project in projects_stream.read_records(sync_mode=SyncMode.full_refresh): + yield {"project_id": project["id"]} + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + project_id = stream_slice["project_id"] + return f"workspaces/{self.workspace_id}/projects/{project_id}/tasks" diff --git a/airbyte-integrations/connectors/source-clockify/unit_tests/__init__.py b/airbyte-integrations/connectors/source-clockify/unit_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-clockify/unit_tests/conftest.py b/airbyte-integrations/connectors/source-clockify/unit_tests/conftest.py new file mode 100644 index 000000000000..ae1ad3fd9d3d --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/unit_tests/conftest.py @@ -0,0 +1,10 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import pytest + + +@pytest.fixture(scope="session", name="config") +def config_fixture(): + return {"api_key": "test_api_key", "workspace_id": "workspace_id"} diff --git a/airbyte-integrations/connectors/source-clockify/unit_tests/test_source.py b/airbyte-integrations/connectors/source-clockify/unit_tests/test_source.py new file mode 100644 index 000000000000..3f6200a9ef68 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/unit_tests/test_source.py @@ -0,0 +1,33 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +import responses +from source_clockify.source import SourceClockify + + +def setup_responses(): + responses.add( + responses.GET, + "https://api.clockify.me/api/v1/workspaces/workspace_id/users", + json={"access_token": "test_api_key", "expires_in": 3600}, + ) + + +@responses.activate +def test_check_connection(config): + setup_responses() + source = SourceClockify() + logger_mock = MagicMock() + assert source.check_connection(logger_mock, config) == (True, None) + + +def test_streams(mocker): + source = SourceClockify() + config_mock = MagicMock() + streams = source.streams(config_mock) + + expected_streams_number = 7 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-clockify/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-clockify/unit_tests/test_streams.py new file mode 100644 index 000000000000..e64e95119dc0 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/unit_tests/test_streams.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +import pytest +from airbyte_cdk.models import SyncMode +from source_clockify.streams import ClockifyStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(ClockifyStream, "path", "v0/example_endpoint") + mocker.patch.object(ClockifyStream, "primary_key", "test_primary_key") + mocker.patch.object(ClockifyStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = ClockifyStream(workspace_id=MagicMock()) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {"page-size": 50} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = ClockifyStream(workspace_id=MagicMock()) + inputs = {"response": MagicMock()} + expected_token = {"page": 2} + assert stream.next_page_token(**inputs) == expected_token + + +def test_read_records(patch_base_class): + stream = ClockifyStream(workspace_id=MagicMock()) + assert stream.read_records(sync_mode=SyncMode.full_refresh) + + +def test_request_headers(patch_base_class): + stream = ClockifyStream(workspace_id=MagicMock()) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = ClockifyStream(workspace_id=MagicMock()) + expected_method = "GET" + assert stream.http_method == expected_method diff --git a/airbyte-integrations/connectors/source-coin-api/.dockerignore b/airbyte-integrations/connectors/source-coin-api/.dockerignore new file mode 100644 index 000000000000..3920b85c3975 --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_coin_api +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-coin-api/Dockerfile b/airbyte-integrations/connectors/source-coin-api/Dockerfile new file mode 100644 index 000000000000..9d1cfc2f242a --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_coin_api ./source_coin_api + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-coin-api diff --git a/airbyte-integrations/connectors/source-coin-api/README.md b/airbyte-integrations/connectors/source-coin-api/README.md new file mode 100644 index 000000000000..71ce0cd08342 --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/README.md @@ -0,0 +1,79 @@ +# Coin Api Source + +This is the repository for the Coin Api configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/coin-api). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-coin-api:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/coin-api) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_coin_api/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source coin-api test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-coin-api:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-coin-api:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-coin-api:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coin-api:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coin-api:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-coin-api:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-coin-api:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-coin-api:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-coin-api/__init__.py b/airbyte-integrations/connectors/source-coin-api/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-coin-api/acceptance-test-config.yml b/airbyte-integrations/connectors/source-coin-api/acceptance-test-config.yml new file mode 100644 index 000000000000..3309e585bb35 --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/acceptance-test-config.yml @@ -0,0 +1,20 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-coin-api:dev +tests: + spec: + - spec_path: "source_coin_api/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-coin-api/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-coin-api/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-coin-api/build.gradle b/airbyte-integrations/connectors/source-coin-api/build.gradle new file mode 100644 index 000000000000..69d7e64f059a --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_coin_api' +} diff --git a/airbyte-integrations/connectors/source-coin-api/integration_tests/__init__.py b/airbyte-integrations/connectors/source-coin-api/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-coin-api/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-coin-api/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..6814db5677a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/integration_tests/abnormal_state.json @@ -0,0 +1,10 @@ +{ + "ohlcv_historical_data": { + "time_start": "2023-01-01T00:00:00", + "time_end": "2024-01-01T00:00:00" + }, + "trades_historical_data": { + "time_start": "2023-01-01T00:00:00", + "time_end": "2024-01-01T00:00:00" + } +} diff --git a/airbyte-integrations/connectors/source-coin-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-coin-api/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-coin-api/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-coin-api/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..b9a1a6787615 --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "ohlcv_historical_data", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "trades_historical_data", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-coin-api/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-coin-api/integration_tests/invalid_config.json new file mode 100644 index 000000000000..0e8273ceb640 --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/integration_tests/invalid_config.json @@ -0,0 +1,8 @@ +{ + "api_key": "YOUR_API_KEY", + "environment": "production", + "symbol_id": "KRAKENFTS_PERP_BTC_USD", + "period": "SOME_PERIOD", + "start_date": "2025-01-01T00:00:00", + "limit": 100000 +} diff --git a/airbyte-integrations/connectors/source-coin-api/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-coin-api/integration_tests/sample_config.json new file mode 100644 index 000000000000..e0c0fa7cc132 --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/integration_tests/sample_config.json @@ -0,0 +1,8 @@ +{ + "api_key": "YOUR_API_KEY", + "environment": "production", + "symbol_id": "KRAKENFTS_PERP_BTC_USD", + "period": "1DAY", + "start_date": "2021-01-01T00:00:00", + "limit": 1000 +} diff --git a/airbyte-integrations/connectors/source-coin-api/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-coin-api/integration_tests/sample_state.json new file mode 100644 index 000000000000..160d19e1b14e --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/integration_tests/sample_state.json @@ -0,0 +1,10 @@ +{ + "ohlcv_historical_data": { + "time_start": "2021-01-01T00:00:00", + "time_end": "2023-01-01T00:00:00" + }, + "trades_historical_data": { + "time_start": "2021-01-01T00:00:00", + "time_end": "2023-01-01T00:00:00" + } +} diff --git a/airbyte-integrations/connectors/source-coin-api/main.py b/airbyte-integrations/connectors/source-coin-api/main.py new file mode 100644 index 000000000000..682ac97ff7fc --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_coin_api import SourceCoinApi + +if __name__ == "__main__": + source = SourceCoinApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-coin-api/requirements.txt b/airbyte-integrations/connectors/source-coin-api/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-coin-api/setup.py b/airbyte-integrations/connectors/source-coin-api/setup.py new file mode 100644 index 000000000000..2a0261aafb9b --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_coin_api", + description="Source implementation for Coin Api.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-coin-api/source_coin_api/__init__.py b/airbyte-integrations/connectors/source-coin-api/source_coin_api/__init__.py new file mode 100644 index 000000000000..485db71240cd --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/source_coin_api/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceCoinApi + +__all__ = ["SourceCoinApi"] diff --git a/airbyte-integrations/connectors/source-coin-api/source_coin_api/coin_api.yaml b/airbyte-integrations/connectors/source-coin-api/source_coin_api/coin_api.yaml new file mode 100644 index 000000000000..6031b0ced17c --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/source_coin_api/coin_api.yaml @@ -0,0 +1,68 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: [] + requester: + url_base: "{{ 'https://rest.coinapi.io/v1' if config['environment'] == 'production' else 'https://rest-sandbox.coinapi.io/v1' }}" + http_method: "GET" + authenticator: + type: ApiKeyAuthenticator + header: "X-CoinAPI-Key" + api_token: "{{ config['api_key'] }}" + request_options_provider: + request_parameters: + period_id: "{{ config['period'] }}" + time_start: "{{ config['start_date'] }}" + time_end: "{{ config['end_date'] }}" + limit: "{{ config['limit'] }}" + stream_slicer: + type: DatetimeStreamSlicer + start_datetime: + datetime: "{{ config['start_time'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%S" + end_datetime: + datetime: "{{ config['end_time'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%S" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: 1d + start_time_option: + field_name: time_start + inject_into: request_parameter + end_time_option: + field_name: time_end + inject_into: request_parameter + + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + requester: + $ref: "*ref(definitions.requester)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + ohlcv_historical_data_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "ohlcv_historical_data" + primary_key: "time_period_start" + path: "/ohlcv/{{ config['symbol_id'] }}/history" + stream_cursor_field: "time_period_start" + trades_historical_data_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "trades_historical_data" + primary_key: "uuid" + path: "/trades/{{ config['symbol_id'] }}/history" + stream_cursor_field: "time_exchange" + +streams: + - "*ref(definitions.ohlcv_historical_data_stream)" + - "*ref(definitions.trades_historical_data_stream)" + +check: + stream_names: + - "ohlcv_historical_data" diff --git a/airbyte-integrations/connectors/source-coin-api/source_coin_api/schemas/ohlcv_historical_data.json b/airbyte-integrations/connectors/source-coin-api/source_coin_api/schemas/ohlcv_historical_data.json new file mode 100644 index 000000000000..33536d642338 --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/source_coin_api/schemas/ohlcv_historical_data.json @@ -0,0 +1,40 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "time_period_start": { + "type": ["null", "string"], + "format": "date-time" + }, + "time_period_end": { + "type": ["null", "string"], + "format": "date-time" + }, + "time_open": { + "type": ["null", "string"], + "format": "date-time" + }, + "time_close": { + "type": ["null", "string"], + "format": "date-time" + }, + "price_open": { + "type": ["null", "number"] + }, + "price_high": { + "type": ["null", "number"] + }, + "price_low": { + "type": ["null", "number"] + }, + "price_close": { + "type": ["null", "number"] + }, + "volume_traded": { + "type": ["null", "number"] + }, + "trades_count": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-coin-api/source_coin_api/schemas/trades_historical_data.json b/airbyte-integrations/connectors/source-coin-api/source_coin_api/schemas/trades_historical_data.json new file mode 100644 index 000000000000..f667e424fa23 --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/source_coin_api/schemas/trades_historical_data.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "symbol_id": { + "type": ["null", "string"] + }, + "time_period_end": { + "type": ["null", "string"], + "format": "date-time" + }, + "time_exchange": { + "type": ["null", "string"], + "format": "date-time" + }, + "time_coinapi": { + "type": ["null", "string"], + "format": "date-time" + }, + "uuid": { + "type": ["null", "string"] + }, + "price": { + "type": ["null", "number"] + }, + "size": { + "type": ["null", "number"] + }, + "taker_side": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-coin-api/source_coin_api/source.py b/airbyte-integrations/connectors/source-coin-api/source_coin_api/source.py new file mode 100644 index 000000000000..43fc49e48bc4 --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/source_coin_api/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceCoinApi(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "coin_api.yaml"}) diff --git a/airbyte-integrations/connectors/source-coin-api/source_coin_api/spec.yaml b/airbyte-integrations/connectors/source-coin-api/source_coin_api/spec.yaml new file mode 100644 index 000000000000..638b6f452174 --- /dev/null +++ b/airbyte-integrations/connectors/source-coin-api/source_coin_api/spec.yaml @@ -0,0 +1,64 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/coin-api +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Coin API Spec + type: object + required: + - api_key + - environment + - symbol_id + - period + - start_date + properties: + api_key: + type: string + description: API Key + airbyte_secret: true + order: 0 + environment: + type: string + description: | + The environment to use. Either sandbox or production. + enum: + - sandbox + - production + default: sandbox + order: 1 + symbol_id: + type: string + description: | + The symbol ID to use. See the documentation for a list. + https://docs.coinapi.io/#list-all-symbols-get + order: 2 + period: + type: string + description: + The period to use. See the documentation for a list. + https://docs.coinapi.io/#list-all-periods-get + examples: + - 5SEC + - 2MTH + start_date: + type: string + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$" + description: The start date in ISO 8601 format. + examples: + - "2019-01-01T00:00:00" + end_date: + type: string + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$" + description: | + The end date in ISO 8601 format. If not supplied, data will be returned + from the start date to the current time, or when the count of result + elements reaches its limit. + examples: + - "2019-01-01T00:00:00" + limit: + type: integer + description: | + The maximum number of elements to return. If not supplied, the default + is 100. For numbers larger than 100, each 100 items is counted as one + request for pricing purposes. Maximum value is 100000. + minimum: 1 + maximum: 100000 + default: 100 diff --git a/airbyte-integrations/connectors/source-convertkit/.dockerignore b/airbyte-integrations/connectors/source-convertkit/.dockerignore new file mode 100644 index 000000000000..7239fffd0a29 --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_convertkit +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-convertkit/Dockerfile b/airbyte-integrations/connectors/source-convertkit/Dockerfile new file mode 100644 index 000000000000..39466118d645 --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_convertkit ./source_convertkit + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-convertkit diff --git a/airbyte-integrations/connectors/source-convertkit/README.md b/airbyte-integrations/connectors/source-convertkit/README.md new file mode 100644 index 000000000000..e616271b56ca --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/README.md @@ -0,0 +1,79 @@ +# Convertkit Source + +This is the repository for the Convertkit configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/convertkit). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-convertkit:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/convertkit) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_convertkit/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source convertkit test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-convertkit:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-convertkit:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-convertkit:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-convertkit:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-convertkit:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-convertkit:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-convertkit:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-convertkit:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-convertkit/__init__.py b/airbyte-integrations/connectors/source-convertkit/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-convertkit/acceptance-test-config.yml b/airbyte-integrations/connectors/source-convertkit/acceptance-test-config.yml new file mode 100644 index 000000000000..fa0f4391dccd --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-convertkit:dev +tests: + spec: + - spec_path: "source_convertkit/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.txt" + # extra_fields: no + # exact_order: no + # extra_records: yes + # incremental: # TODO if your connector does not implement incremental sync, remove this block + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-convertkit/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-convertkit/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-convertkit/build.gradle b/airbyte-integrations/connectors/source-convertkit/build.gradle new file mode 100644 index 000000000000..7e7db2e9e9c9 --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_convertkit' +} diff --git a/airbyte-integrations/connectors/source-convertkit/integration_tests/__init__.py b/airbyte-integrations/connectors/source-convertkit/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-convertkit/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-convertkit/integration_tests/acceptance.py new file mode 100644 index 000000000000..1302b2f57e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-convertkit/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-convertkit/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..8bb19362a6f8 --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/integration_tests/configured_catalog.json @@ -0,0 +1,49 @@ +{ + "streams": [ + { + "stream": { + "name": "forms", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "sequences", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "tags", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "subscribers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "broadcasts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-convertkit/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-convertkit/integration_tests/invalid_config.json new file mode 100644 index 000000000000..96e67fbca8a2 --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "api_secret": "" +} diff --git a/airbyte-integrations/connectors/source-convertkit/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-convertkit/integration_tests/sample_config.json new file mode 100644 index 000000000000..434c2977d8ba --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "api_secret": "" +} diff --git a/airbyte-integrations/connectors/source-convertkit/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-convertkit/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-convertkit/main.py b/airbyte-integrations/connectors/source-convertkit/main.py new file mode 100644 index 000000000000..0bf30250344d --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_convertkit import SourceConvertkit + +if __name__ == "__main__": + source = SourceConvertkit() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-convertkit/requirements.txt b/airbyte-integrations/connectors/source-convertkit/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-convertkit/setup.py b/airbyte-integrations/connectors/source-convertkit/setup.py new file mode 100644 index 000000000000..46a7c93718ad --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_convertkit", + description="Source implementation for Convertkit.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-convertkit/source_convertkit/__init__.py b/airbyte-integrations/connectors/source-convertkit/source_convertkit/__init__.py new file mode 100644 index 000000000000..668ce26356fd --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/source_convertkit/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceConvertkit + +__all__ = ["SourceConvertkit"] diff --git a/airbyte-integrations/connectors/source-convertkit/source_convertkit/convertkit.yaml b/airbyte-integrations/connectors/source-convertkit/source_convertkit/convertkit.yaml new file mode 100644 index 000000000000..e3c90fbb0e80 --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/source_convertkit/convertkit.yaml @@ -0,0 +1,87 @@ +version: "0.1.0" +definitions: + selector: + extractor: + field_pointer: [ "{{ options['name'] }}" ] + requester: + # API Docs: https://developers.convertkit.com/#overview + url_base: "https://api.convertkit.com/v3" + http_method: "GET" + # API Docs: https://developers.convertkit.com/#api-basics + request_options_provider: + request_parameters: + api_secret: "{{ config['api_secret'] }}" + increment_paginator: + type: DefaultPaginator + url_base: "*ref(definitions.requester.url_base)" + page_size_option: + inject_into: "request_parameter" + field_name: "limit" + pagination_strategy: + type: PageIncrement + page_size: 50 + page_token_option: + inject_into: "request_parameter" + field_name: "page" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + requester: + $ref: "*ref(definitions.requester)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + # API Docs: https://developers.convertkit.com/#forms + forms_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "forms" + primary_key: "id" + path: "/forms" + # API Docs: https://developers.convertkit.com/#sequences + sequences_stream: + $ref: "*ref(definitions.base_stream)" + retriever: + $ref: "*ref(definitions.retriever)" + record_selector: + extractor: + field_pointer: [ "courses" ] + $options: + name: "sequences" + primary_key: "id" + path: "/sequences" + # API Docs: https://developers.convertkit.com/#tags + tags_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "tags" + primary_key: "id" + path: "/tags" + retriever: + $ref: "*ref(definitions.retriever)" + paginator: + $ref: "*ref(definitions.increment_paginator)" + # API Docs: https://developers.convertkit.com/#subscribers + subscribers_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "subscribers" + primary_key: "id" + path: "/subscribers" + # API Docs: https://developers.convertkit.com/#broadcasts + broadcasts_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "broadcasts" + primary_key: "id" + path: "/broadcasts" +streams: + - "*ref(definitions.forms_stream)" + - "*ref(definitions.sequences_stream)" + - "*ref(definitions.tags_stream)" + - "*ref(definitions.subscribers_stream)" + - "*ref(definitions.broadcasts_stream)" +check: + stream_names: ["forms"] diff --git a/airbyte-integrations/connectors/source-convertkit/source_convertkit/schemas/broadcasts.json b/airbyte-integrations/connectors/source-convertkit/source_convertkit/schemas/broadcasts.json new file mode 100644 index 000000000000..2d9c6ef73ebd --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/source_convertkit/schemas/broadcasts.json @@ -0,0 +1,23 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "created_at": { + "type": [ + "string", + "null" + ] + }, + "subject": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "id" + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-convertkit/source_convertkit/schemas/forms.json b/airbyte-integrations/connectors/source-convertkit/source_convertkit/schemas/forms.json new file mode 100644 index 000000000000..95519975d677 --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/source_convertkit/schemas/forms.json @@ -0,0 +1,83 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "uid": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "created_at": { + "type": [ + "string", + "null" + ] + }, + "archived": { + "type": [ + "boolean", + "null" + ] + }, + "type": { + "type": [ + "string", + "null" + ] + }, + "url": { + "type": [ + "string", + "null" + ] + }, + "embed_js": { + "type": [ + "string", + "null" + ] + }, + "embed_url": { + "type": [ + "string", + "null" + ] + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "description": { + "type": [ + "string", + "null" + ] + }, + "sign_up_button_text": { + "type": [ + "string", + "null" + ] + }, + "success_message": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "id" + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-convertkit/source_convertkit/schemas/sequences.json b/airbyte-integrations/connectors/source-convertkit/source_convertkit/schemas/sequences.json new file mode 100644 index 000000000000..3767adac7c42 --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/source_convertkit/schemas/sequences.json @@ -0,0 +1,35 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "hold": { + "type": [ + "boolean", + "null" + ] + }, + "repeat": { + "type": [ + "boolean", + "null" + ] + }, + "created_at": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "id" + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-convertkit/source_convertkit/schemas/subscribers.json b/airbyte-integrations/connectors/source-convertkit/source_convertkit/schemas/subscribers.json new file mode 100644 index 000000000000..8e94599a7ed5 --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/source_convertkit/schemas/subscribers.json @@ -0,0 +1,49 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "first_name": { + "type": [ + "string", + "null" + ] + }, + "email_address": { + "type": [ + "string", + "null" + ] + }, + "state": { + "type": [ + "string", + "null" + ] + }, + "created_at": { + "type": [ + "string", + "null" + ] + }, + "fields": { + "type": [ + "object", + "null" + ], + "properties": { + "last_name": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "required": [ + "id" + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-convertkit/source_convertkit/schemas/tags.json b/airbyte-integrations/connectors/source-convertkit/source_convertkit/schemas/tags.json new file mode 100644 index 000000000000..66ccdc68f34e --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/source_convertkit/schemas/tags.json @@ -0,0 +1,23 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "created_at": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "id" + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-convertkit/source_convertkit/source.py b/airbyte-integrations/connectors/source-convertkit/source_convertkit/source.py new file mode 100644 index 000000000000..6fd59d24c59f --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/source_convertkit/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceConvertkit(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "convertkit.yaml"}) diff --git a/airbyte-integrations/connectors/source-convertkit/source_convertkit/spec.yaml b/airbyte-integrations/connectors/source-convertkit/source_convertkit/spec.yaml new file mode 100644 index 000000000000..a03e1252fb8a --- /dev/null +++ b/airbyte-integrations/connectors/source-convertkit/source_convertkit/spec.yaml @@ -0,0 +1,13 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/convertkit +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Convertkit Spec + type: object + required: + - api_secret + additionalProperties: true + properties: + api_secret: + type: string + description: API Secret + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile index 5c7b89d52ef8..9e503180003e 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.69 +LABEL io.airbyte.version=0.2.70 LABEL io.airbyte.name=airbyte/source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml index 5c35ca8f1472..86d6aa901dd6 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml @@ -4,8 +4,6 @@ connector_image: airbyte/source-facebook-marketing:dev tests: spec: - spec_path: "integration_tests/spec.json" - backward_compatibility_tests_config: - disable_for_version: "0.2.60" connection: - config_path: "secrets/config.json" status: "succeed" @@ -18,6 +16,8 @@ tests: empty_streams: ["videos"] incremental: - config_path: "secrets/config.json" + timeout_seconds: 2400 future_state_path: "integration_tests/future_state.json" + skip_comprehensive_incremental_tests: true full_refresh: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index efb67ce094b7..f27ad1bfc415 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -173,6 +173,7 @@ "social_spend", "spend", "total_postbacks", + "total_postbacks_detailed", "unique_actions", "unique_clicks", "unique_conversions", @@ -231,12 +232,15 @@ "hourly_stats_aggregated_by_audience_time_zone", "image_asset", "impression_device", + "is_conversion_id_modeled", "link_url_asset", + "mmm", "place_page_id", "platform_position", "product_id", "publisher_platform", "region", + "skan_campaign_id", "skan_conversion_id", "title_asset", "video_asset" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/setup.py b/airbyte-integrations/connectors/source-facebook-marketing/setup.py index fc9fc73c0a0b..8b9d7839bc03 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/setup.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/setup.py @@ -8,7 +8,7 @@ MAIN_REQUIREMENTS = [ "airbyte-cdk~=0.1", "cached_property==1.5.2", - "facebook_business==14.0.0", + "facebook_business==15.0.0", "pendulum>=2,<3", ] diff --git a/airbyte-integrations/connectors/source-file-secure/Dockerfile b/airbyte-integrations/connectors/source-file-secure/Dockerfile index e38177292b4c..0bfb5bbdbe95 100644 --- a/airbyte-integrations/connectors/source-file-secure/Dockerfile +++ b/airbyte-integrations/connectors/source-file-secure/Dockerfile @@ -1,4 +1,4 @@ -FROM airbyte/source-file:0.2.26 +FROM airbyte/source-file:0.2.28 WORKDIR /airbyte/integration_code COPY source_file_secure ./source_file_secure @@ -9,5 +9,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.26 +LABEL io.airbyte.version=0.2.28 LABEL io.airbyte.name=airbyte/source-file-secure diff --git a/airbyte-integrations/connectors/source-file-secure/README.md b/airbyte-integrations/connectors/source-file-secure/README.md index 6819056530a5..d1ab61dccb47 100644 --- a/airbyte-integrations/connectors/source-file-secure/README.md +++ b/airbyte-integrations/connectors/source-file-secure/README.md @@ -42,10 +42,10 @@ and place them into `secrets/config.json`. ### Locally running the connector ``` -python main_dev.py spec -python main_dev.py check --config secrets/config.json -python main_dev.py discover --config secrets/config.json -python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` ### Unit Tests @@ -59,7 +59,8 @@ python -m pytest unit_tests #### Build First, make sure you build the latest Docker image: ``` -docker build . -t airbyte/source-file-secure:dev +docker build . -t airbyte/source-file-secure:dev \ +&& python -m pytest -p source_acceptance_test.plugin ``` You can also build the connector image via Gradle: diff --git a/airbyte-integrations/connectors/source-file/Dockerfile b/airbyte-integrations/connectors/source-file/Dockerfile index b3f1178648ba..6a54c7114f16 100644 --- a/airbyte-integrations/connectors/source-file/Dockerfile +++ b/airbyte-integrations/connectors/source-file/Dockerfile @@ -17,5 +17,5 @@ COPY source_file ./source_file ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.26 +LABEL io.airbyte.version=0.2.28 LABEL io.airbyte.name=airbyte/source-file diff --git a/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py b/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py index 41aebb34fa66..692dea5594a1 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py +++ b/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py @@ -9,7 +9,7 @@ import pytest from airbyte_cdk import AirbyteLogger from source_file import SourceFile -from source_file.client import Client +from source_file.client import Client, ConfigurationError SAMPLE_DIRECTORY = Path(__file__).resolve().parent.joinpath("sample_files/formats") @@ -42,6 +42,28 @@ def test_local_file_read(file_format, extension, expected_columns, expected_rows check_read(configs, expected_columns, expected_rows) +@pytest.mark.parametrize( + "file_format, extension, wrong_format, filename", + [ + ("excel", "xls", "csv", "demo"), + ("excel", "xlsx", "csv", "demo"), + ("csv", "csv", "excel", "demo"), + ("csv", "csv", "excel", "demo"), + ("jsonl", "jsonl", "excel", "jsonl_nested"), + ("feather", "feather", "csv", "demo"), + ("parquet", "parquet", "feather", "demo"), + ("yaml", "yaml", "json", "demo"), + ], +) +def test_raises_file_wrong_format(file_format, extension, wrong_format, filename): + file_directory = SAMPLE_DIRECTORY.joinpath(file_format) + file_path = str(file_directory.joinpath(f"{filename}.{extension}")) + configs = {"dataset_name": "test", "format": wrong_format, "url": file_path, "provider": {"storage": "local"}} + client = Client(**configs) + with pytest.raises((TypeError, ValueError, ConfigurationError)): + list(client.read()) + + def run_load_dataframes(config, expected_columns=10, expected_rows=42): df_list = SourceFile.load_dataframes(config=config, logger=AirbyteLogger(), skip_data=False) assert len(df_list) == 1 # Properly load 1 DataFrame diff --git a/airbyte-integrations/connectors/source-file/setup.py b/airbyte-integrations/connectors/source-file/setup.py index 4d12358e6cce..16d0e65433bd 100644 --- a/airbyte-integrations/connectors/source-file/setup.py +++ b/airbyte-integrations/connectors/source-file/setup.py @@ -24,7 +24,7 @@ "pyxlsb==1.0.9", ] -TEST_REQUIREMENTS = ["pytest~=6.2", "pytest-docker==1.0.0", "pytest-mock~=3.6.1"] +TEST_REQUIREMENTS = ["pytest~=6.2", "pytest-docker~=1.0.0", "pytest-mock~=3.6.1"] setup( name="source_file", diff --git a/airbyte-integrations/connectors/source-file/source_file/client.py b/airbyte-integrations/connectors/source-file/source_file/client.py index 6a953dc4f558..43ec43001208 100644 --- a/airbyte-integrations/connectors/source-file/source_file/client.py +++ b/airbyte-integrations/connectors/source-file/source_file/client.py @@ -10,6 +10,7 @@ from typing import Iterable from urllib.parse import urlparse +import backoff import boto3 import botocore import google @@ -24,6 +25,8 @@ from google.oauth2 import service_account from yaml import safe_load +from .utils import backoff_handler + class ConfigurationError(Exception): """Client mis-configured""" @@ -351,25 +354,30 @@ def dtype_to_json_type(current_type: str, dtype) -> str: def reader(self) -> reader_class: return self.reader_class(url=self._url, provider=self._provider, binary=self.binary_source, encoding=self.encoding) + @backoff.on_exception(backoff.expo, ConnectionResetError, on_backoff=backoff_handler, max_tries=5, max_time=60) def read(self, fields: Iterable = None) -> Iterable[dict]: """Read data from the stream""" with self.reader.open() as fp: - if self._reader_format in ["json", "jsonl"]: - yield from self.load_nested_json(fp) - elif self._reader_format == "yaml": - fields = set(fields) if fields else None - df = self.load_yaml(fp) - columns = fields.intersection(set(df.columns)) if fields else df.columns - df = df.where(pd.notnull(df), None) - yield from df[columns].to_dict(orient="records") - else: - fields = set(fields) if fields else None - if self.binary_source: - fp = self._cache_stream(fp) - for df in self.load_dataframes(fp): + try: + if self._reader_format in ["json", "jsonl"]: + yield from self.load_nested_json(fp) + elif self._reader_format == "yaml": + fields = set(fields) if fields else None + df = self.load_yaml(fp) columns = fields.intersection(set(df.columns)) if fields else df.columns - df.replace({np.nan: None}, inplace=True) - yield from df[list(columns)].to_dict(orient="records") + df = df.where(pd.notnull(df), None) + yield from df[columns].to_dict(orient="records") + else: + fields = set(fields) if fields else None + if self.binary_source: + fp = self._cache_stream(fp) + for df in self.load_dataframes(fp): + columns = fields.intersection(set(df.columns)) if fields else df.columns + df.replace({np.nan: None}, inplace=True) + yield from df[list(columns)].to_dict(orient="records") + except ConnectionResetError: + logger.info(f"Catched `connection reset error - 104`, stream: {self.stream_name} ({self.reader.full_url})") + raise ConnectionResetError def _cache_stream(self, fp): """cache stream to file""" diff --git a/airbyte-integrations/connectors/source-file/source_file/source.py b/airbyte-integrations/connectors/source-file/source_file/source.py index 68d8592914cd..a9c51c0e8654 100644 --- a/airbyte-integrations/connectors/source-file/source_file/source.py +++ b/airbyte-integrations/connectors/source-file/source_file/source.py @@ -21,7 +21,7 @@ ) from airbyte_cdk.sources import Source -from .client import Client +from .client import Client, ConfigurationError from .utils import dropbox_force_download @@ -96,12 +96,25 @@ def check(self, logger, config: Mapping) -> AirbyteConnectionStatus: """ config = self._validate_and_transform(config) client = self._get_client(config) - logger.info(f"Checking access to {client.reader.full_url}...") + source_url = client.reader.full_url + logger.info(f"Checking access to {source_url}...") + if "docs.google.com/spreadsheets" in source_url: + reason = f"Failed to load {source_url}: please use the Official Google Sheets Source connector" + logger.error(reason) + return AirbyteConnectionStatus(status=Status.FAILED, message=reason) try: with client.reader.open(): + list(client.streams) return AirbyteConnectionStatus(status=Status.SUCCEEDED) + except (TypeError, ValueError, ConfigurationError) as err: + reason = ( + f"Failed to load {source_url}\n Please check File Format and Reader Options are set correctly" + f"\n{repr(err)}\n{traceback.format_exc()}" + ) + logger.error(reason) + return AirbyteConnectionStatus(status=Status.FAILED, message=reason) except Exception as err: - reason = f"Failed to load {client.reader.full_url}: {repr(err)}\n{traceback.format_exc()}" + reason = f"Failed to load {source_url}: {repr(err)}\n{traceback.format_exc()}" logger.error(reason) return AirbyteConnectionStatus(status=Status.FAILED, message=reason) diff --git a/airbyte-integrations/connectors/source-file/source_file/utils.py b/airbyte-integrations/connectors/source-file/source_file/utils.py index 7dd61daf63ae..a1b51d9e8248 100644 --- a/airbyte-integrations/connectors/source-file/source_file/utils.py +++ b/airbyte-integrations/connectors/source-file/source_file/utils.py @@ -2,9 +2,12 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # - +import logging from urllib.parse import parse_qs, urlencode, urlparse +# default logger +logger = logging.getLogger("airbyte") + def dropbox_force_download(url): """ @@ -17,3 +20,7 @@ def dropbox_force_download(url): qs["dl"] = "1" parse_result = parse_result._replace(query=urlencode(qs)) return parse_result.geturl() + + +def backoff_handler(details): + logger.info(f"Caught retryable error after {details['tries']} tries. Waiting {details['wait']} seconds then retrying...") diff --git a/airbyte-integrations/connectors/source-file/unit_tests/conftest.py b/airbyte-integrations/connectors/source-file/unit_tests/conftest.py index 4dab8b9f5def..890e795b4236 100644 --- a/airbyte-integrations/connectors/source-file/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-file/unit_tests/conftest.py @@ -64,3 +64,17 @@ def absolute_path(): @pytest.fixture def test_files(): return "../integration_tests/sample_files" + + +@pytest.fixture +def test_read_config(): + return { + "dataset_name": "integrationTestFile", + "format": "csv", + "url": "https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv", + "provider": { + "storage": "HTTPS", + "reader_impl": "gcsfs", + "user_agent": False, + }, + } diff --git a/airbyte-integrations/connectors/source-file/unit_tests/test_client.py b/airbyte-integrations/connectors/source-file/unit_tests/test_client.py index 95791d597d21..390b906f9b01 100644 --- a/airbyte-integrations/connectors/source-file/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-file/unit_tests/test_client.py @@ -3,6 +3,8 @@ # +from unittest.mock import patch + import pytest from pandas import read_csv, read_excel from source_file.client import Client, ConfigurationError, URLFile @@ -134,3 +136,14 @@ def test_open_gcs_url(): provider.update({"service_account_json": '{service_account_json": "service_account_json"}'}) with pytest.raises(ConfigurationError): assert URLFile(url="", provider=provider)._open_gcs_url() + + +def test_read(test_read_config): + client = Client(**test_read_config) + client.sleep_on_retry_sec = 0 # just for test + with patch.object(client, "load_dataframes", side_effect=ConnectionResetError) as mock_method: + try: + return client.read(["date", "key"]) + except ConnectionResetError: + print("Exception has been raised correctly!") + mock_method.assert_called() diff --git a/airbyte-integrations/connectors/source-freshdesk/setup.py b/airbyte-integrations/connectors/source-freshdesk/setup.py index 233deec4f97d..08f586a268bd 100644 --- a/airbyte-integrations/connectors/source-freshdesk/setup.py +++ b/airbyte-integrations/connectors/source-freshdesk/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk~=0.2", "backoff==1.10.0", "requests==2.25.1", "pendulum==2.1.2", diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py index de704263bc3d..00315a9bc5fa 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py @@ -4,6 +4,7 @@ import random from typing import Any, MutableMapping +from unittest.mock import PropertyMock, patch import pytest from airbyte_cdk.models import SyncMode @@ -125,18 +126,19 @@ def test_incremental(stream, resource, authenticator, config, requests_mock): highest_updated_at = "2022-04-25T22:00:00Z" other_updated_at = "2022-04-01T00:00:00Z" highest_index = random.randint(0, 24) - requests_mock.register_uri( - "GET", - f"/api/{resource}", - json=[{"id": x, "updated_at": highest_updated_at if x == highest_index else other_updated_at} for x in range(25)], - ) - - stream = stream(authenticator=authenticator, config=config) - records, state = _read_incremental(stream, {}) - - assert len(records) == 25 - assert "updated_at" in state - assert state["updated_at"] == highest_updated_at + with patch(f"source_freshdesk.streams.{stream.__name__}.use_cache", new_callable=PropertyMock, return_value=False): + requests_mock.register_uri( + "GET", + f"/api/{resource}", + json=[{"id": x, "updated_at": highest_updated_at if x == highest_index else other_updated_at} for x in range(25)], + ) + + stream = stream(authenticator=authenticator, config=config) + records, state = _read_incremental(stream, {}) + + assert len(records) == 25 + assert "updated_at" in state + assert state["updated_at"] == highest_updated_at @pytest.mark.parametrize( diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-config.yml index 4c3cb0ea4112..4b7cee681080 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-config.yml @@ -21,10 +21,10 @@ tests: expect_records: path: "integration_tests/expected_records.txt" incremental: - - config_path: "secrets/service_config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" - threshold_days: 2 + - config_path: "secrets/service_config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + threshold_days: 2 full_refresh: - config_path: "secrets/service_config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-google-search-console/Dockerfile b/airbyte-integrations/connectors/source-google-search-console/Dockerfile index 1276245292c8..c48b29620b6f 100755 --- a/airbyte-integrations/connectors/source-google-search-console/Dockerfile +++ b/airbyte-integrations/connectors/source-google-search-console/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.17 +LABEL io.airbyte.version=0.1.18 LABEL io.airbyte.name=airbyte/source-google-search-console diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py index 06729e9db31d..d05822969481 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py @@ -63,6 +63,10 @@ def _validate_and_transform(self, config: Mapping[str, Any]): except ValueError: raise Exception("custom_reports is not valid JSON") jsonschema.validate(config["custom_reports"], custom_reports_schema) + for report in config["custom_reports"]: + for dimension in report["dimensions"]: + if dimension not in SearchAnalyticsByCustomDimensions.dimension_to_property_schema_map: + raise Exception(f"dimension: '{dimension}' not found") pendulum.parse(config["start_date"]) end_date = config.get("end_date") diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py index 3ba2adb1fa07..c52d3b0599b2 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py @@ -298,6 +298,14 @@ class SearchAnalyticsAllFields(SearchAnalytics): class SearchAnalyticsByCustomDimensions(SearchAnalytics): + dimension_to_property_schema_map = { + "country": [{"country": {"type": ["null", "string"]}}], + "date": [], + "device": [{"device": {"type": ["null", "string"]}}], + "page": [{"page": {"type": ["null", "string"]}}], + "query": [{"query": {"type": ["null", "string"]}}], + } + def __init__(self, dimensions: List[str], *args, **kwargs): super(SearchAnalyticsByCustomDimensions, self).__init__(*args, **kwargs) self.dimensions = dimensions @@ -305,7 +313,7 @@ def __init__(self, dimensions: List[str], *args, **kwargs): def get_json_schema(self) -> Mapping[str, Any]: try: return super(SearchAnalyticsByCustomDimensions, self).get_json_schema() - except IOError: + except FileNotFoundError: schema: Mapping[str, Any] = { "$schema": "http://json-schema.org/draft-07/schema#", "type": ["null", "object"], @@ -327,16 +335,9 @@ def get_json_schema(self) -> Mapping[str, Any]: return schema def dimension_to_property_schema(self) -> dict: - dimension_to_property_schema_map = { - "country": [{"country": {"type": ["null", "string"]}}], - "date": [], - "device": [{"device": {"type": ["null", "string"]}}], - "page": [{"page": {"type": ["null", "string"]}}], - "query": [{"query": {"type": ["null", "string"]}}], - } properties = {} for dimension in sorted(self.dimensions): - fields = dimension_to_property_schema_map[dimension] + fields = self.dimension_to_property_schema_map[dimension] for field in fields: properties = {**properties, **field} return properties diff --git a/airbyte-integrations/connectors/source-google-webfonts/.dockerignore b/airbyte-integrations/connectors/source-google-webfonts/.dockerignore new file mode 100644 index 000000000000..b7c1ebe6c666 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_google_webfonts +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-google-webfonts/Dockerfile b/airbyte-integrations/connectors/source-google-webfonts/Dockerfile new file mode 100644 index 000000000000..56b13b3f8de2 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_google_webfonts ./source_google_webfonts + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-google-webfonts diff --git a/airbyte-integrations/connectors/source-google-webfonts/README.md b/airbyte-integrations/connectors/source-google-webfonts/README.md new file mode 100644 index 000000000000..f0e9d4f2ba72 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/README.md @@ -0,0 +1,103 @@ +# Google Webfonts Source + +This is the repository for the Google Webfonts configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/google-webfonts). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-google-webfonts:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/google-webfonts) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_webfonts/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source google-webfonts test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-google-webfonts:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-google-webfonts:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-google-webfonts:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-webfonts:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-webfonts:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-google-webfonts:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-google-webfonts:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-google-webfonts:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-google-webfonts/__init__.py b/airbyte-integrations/connectors/source-google-webfonts/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-google-webfonts/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-webfonts/acceptance-test-config.yml new file mode 100644 index 000000000000..7e69e954eac4 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/acceptance-test-config.yml @@ -0,0 +1,20 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-google-webfonts:dev +tests: + spec: + - spec_path: "source_google_webfonts/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-google-webfonts/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-google-webfonts/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-google-webfonts/bootstrap.md b/airbyte-integrations/connectors/source-google-webfonts/bootstrap.md new file mode 100644 index 000000000000..10cfa3b880b5 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/bootstrap.md @@ -0,0 +1,38 @@ +# Google-webfonts + +The connector uses the v1 API documented here: https://developers.google.com/fonts/docs/developer_api . It is +straightforward HTTP REST API with API authentication. + +## API key + +Api key is mandate for this connector to work, It could be generated by a gmail account for free at https://console.cloud.google.com/apis/dashboard. +Just pass the generated API key and optional parameters for establishing the connection. Example:123 + +## Implementation details + +## Setup guide + +### Step 1: Set up Google-webfonts connection + +- Generate an API key (Example: 12345) +- Params (If specific info is needed) +- Available params + - sort: SORT_UNDEFINED, ALPHA, DATE, STYLE, TRENDING, POPULARITY + - alt: json, media or proto + - prettyPrint: boolean + +## Step 2: Generate schema for the endpoint + +### Custom schema is generated and tested with different IDs + +## Step 3: Spec, Secrets, and connector yaml files are configured with reference to the Airbyte documentation. + +## In a nutshell: + +1. Navigate to the Airbyte Open Source dashboard. +2. Set the name for your source. +3. Enter your `api_key`. +5. Enter your config params if needed. (Optional) +6. Click **Set up source**. + + * We use only GET methods, towards the webfonts endpoints which is straightforward \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-google-webfonts/build.gradle b/airbyte-integrations/connectors/source-google-webfonts/build.gradle new file mode 100644 index 000000000000..8bf7fd748729 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_google_webfonts' +} diff --git a/airbyte-integrations/connectors/source-google-webfonts/integration_tests/__init__.py b/airbyte-integrations/connectors/source-google-webfonts/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-google-webfonts/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-webfonts/integration_tests/acceptance.py new file mode 100644 index 000000000000..1302b2f57e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-google-webfonts/integration_tests/catalog.json b/airbyte-integrations/connectors/source-google-webfonts/integration_tests/catalog.json new file mode 100644 index 000000000000..6799946a6851 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/integration_tests/catalog.json @@ -0,0 +1,39 @@ +{ + "streams": [ + { + "name": "TODO fix this file", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": "column1", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + }, + { + "name": "table1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-webfonts/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-webfonts/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..2875f8637421 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/integration_tests/configured_catalog.json @@ -0,0 +1,13 @@ +{ + "streams": [ + { + "stream": { + "name": "fonts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-webfonts/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-google-webfonts/integration_tests/invalid_config.json new file mode 100644 index 000000000000..316264c50d74 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/integration_tests/invalid_config.json @@ -0,0 +1,6 @@ +{ + "api_key": "", + "sort": "", + "prettyPrint": "", + "alt": "" +} diff --git a/airbyte-integrations/connectors/source-google-webfonts/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-google-webfonts/integration_tests/sample_config.json new file mode 100644 index 000000000000..93b02698274d --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/integration_tests/sample_config.json @@ -0,0 +1,6 @@ +{ + "api_key": "", + "sort": "SORT_UNDEFINED", + "prettyPrint": "true", + "alt": "json" +} diff --git a/airbyte-integrations/connectors/source-google-webfonts/main.py b/airbyte-integrations/connectors/source-google-webfonts/main.py new file mode 100644 index 000000000000..065faa3891bf --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_google_webfonts import SourceGoogleWebfonts + +if __name__ == "__main__": + source = SourceGoogleWebfonts() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-webfonts/requirements.txt b/airbyte-integrations/connectors/source-google-webfonts/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-google-webfonts/setup.py b/airbyte-integrations/connectors/source-google-webfonts/setup.py new file mode 100644 index 000000000000..f290fc2148d4 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_google_webfonts", + description="Source implementation for Google Webfonts.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/__init__.py b/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/__init__.py new file mode 100644 index 000000000000..4045cfdf0228 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceGoogleWebfonts + +__all__ = ["SourceGoogleWebfonts"] diff --git a/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/google_webfonts.yaml b/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/google_webfonts.yaml new file mode 100644 index 000000000000..61310dc2ac11 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/google_webfonts.yaml @@ -0,0 +1,38 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: ["items"] + requester: + url_base: "https://webfonts.googleapis.com/v1" + http_method: "GET" + authenticator: + type: ApiKeyAuthenticator + header: "apikey" + api_token: "{{ config['api_key'] }}" + + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + requester: + $ref: "*ref(definitions.requester)" + + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + + fonts_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "fonts" + path: "/webfonts?key={{ config['api_key'] }}&sort={{ config['sort'] or 'SORT_UNDEFINED'}}&prettyPrint={{ config['prettyPrint'] or 'true'}}&alt={{ config['alt'] or 'json'}}" + +streams: + - "*ref(definitions.fonts_stream)" + +check: + stream_names: + - "fonts" diff --git a/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/schemas/fonts.json b/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/schemas/fonts.json new file mode 100644 index 000000000000..9c62b02dc329 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/schemas/fonts.json @@ -0,0 +1,111 @@ +{ + "definitions": {}, + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://example.com/object1666796406.json", + "title": "Root", + "type": "object", + "properties": { + "kind": { + "$id": "#root/kind", + "title": "Kind", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "items": { + "$id": "#root/items", + "title": "Items", + "type": "array", + "default": [], + "items": { + "$id": "#root/items/items", + "title": "Items", + "type": "object", + "properties": { + "family": { + "$id": "#root/items/items/family", + "title": "Family", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "variants": { + "$id": "#root/items/items/variants", + "title": "Variants", + "type": "array", + "default": [], + "items": { + "$id": "#root/items/items/variants/items", + "title": "Items", + "type": "string", + "default": "", + "pattern": "^.*$" + } + }, + "subsets": { + "$id": "#root/items/items/subsets", + "title": "Subsets", + "type": "array", + "default": [], + "items": { + "$id": "#root/items/items/subsets/items", + "title": "Items", + "type": "string", + "default": "", + "pattern": "^.*$" + } + }, + "version": { + "$id": "#root/items/items/version", + "title": "Version", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "lastModified": { + "$id": "#root/items/items/lastModified", + "title": "Lastmodified", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "files": { + "$id": "#root/items/items/files", + "title": "Files", + "type": "object", + "properties": { + "regular": { + "$id": "#root/items/items/files/regular", + "title": "Regular", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "italic": { + "$id": "#root/items/items/files/italic", + "title": "Italic", + "type": "string", + "default": "", + "pattern": "^.*$" + } + } + }, + "category": { + "$id": "#root/items/items/category", + "title": "Category", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "kind": { + "$id": "#root/items/items/kind", + "title": "Kind", + "type": "string", + "default": "", + "pattern": "^.*$" + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/source.py b/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/source.py new file mode 100644 index 000000000000..353c6e585164 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceGoogleWebfonts(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "google_webfonts.yaml"}) diff --git a/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/spec.yaml b/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/spec.yaml new file mode 100644 index 000000000000..ebc80b05ad5a --- /dev/null +++ b/airbyte-integrations/connectors/source-google-webfonts/source_google_webfonts/spec.yaml @@ -0,0 +1,22 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/google-webfonts +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Google Webfonts Spec + type: object + required: + - api_key + additionalProperties: true + properties: + api_key: + type: string + description: API key is required to access google apis, For getting your's goto google console and generate api key for Webfonts + airbyte_secret: true + sort: + type: string + description: Optional, to find how to sort + prettyPrint: + type: string + description: Optional, boolean type + alt: + type: string + description: Optional, Available params- json, media, proto diff --git a/airbyte-integrations/connectors/source-greenhouse/Dockerfile b/airbyte-integrations/connectors/source-greenhouse/Dockerfile index f0c956236759..dedbaeca1b11 100644 --- a/airbyte-integrations/connectors/source-greenhouse/Dockerfile +++ b/airbyte-integrations/connectors/source-greenhouse/Dockerfile @@ -12,5 +12,5 @@ COPY main.py ./ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.11 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-greenhouse diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.txt index 105b06b32ab8..5e2019e3b30d 100644 --- a/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.txt @@ -39,11 +39,11 @@ {"stream": "offers", "data": {"id": 4154100003, "version": 1, "application_id": 19215333003, "created_at": "2020-11-24T23:32:25.760Z", "updated_at": "2020-11-24T23:32:25.772Z", "sent_at": null, "resolved_at": null, "starts_at": "2020-12-04", "status": "unresolved", "job_id": 4177048003, "candidate_id": 17130848003, "opening": {"id": 4320018003, "opening_id": "4-1", "status": "open", "opened_at": "2020-11-24T23:27:45.665Z", "closed_at": null, "application_id": null, "close_reason": null}, "custom_fields": {"employment_type": "Contract"}, "keyed_custom_fields": {"employment_type": {"name": "Employment Type", "type": "single_select", "value": "Contract"}}}, "emitted_at": 1664285618339} {"stream": "scorecards", "data": {"id": 5253031003, "updated_at": "2020-11-24T23:33:10.440Z", "created_at": "2020-11-24T23:33:10.440Z", "interview": "Application Review", "interview_step": {"id": 5628634003, "name": "Application Review"}, "candidate_id": 17130848003, "application_id": 19215333003, "interviewed_at": "2020-11-25T01:00:00.000Z", "submitted_by": {"id": 4218086003, "first_name": "John", "last_name": "Lafleur", "name": "John Lafleur", "employee_id": null}, "interviewer": {"id": 4218086003, "first_name": "John", "last_name": "Lafleur", "name": "John Lafleur", "employee_id": null}, "submitted_at": "2020-11-24T23:33:10.440Z", "overall_recommendation": "no_decision", "attributes": [{"name": "Willing to do required travel", "type": "Details", "note": null, "rating": "no_decision"}, {"name": "Three to five years of experience", "type": "Qualifications", "note": null, "rating": "no_decision"}, {"name": "Personable", "type": "Personality Traits", "note": null, "rating": "no_decision"}, {"name": "Passionate", "type": "Personality Traits", "note": null, "rating": "no_decision"}, {"name": "Organizational Skills", "type": "Skills", "note": null, "rating": "no_decision"}, {"name": "Manage competing priorities", "type": "Skills", "note": null, "rating": "no_decision"}, {"name": "Fits our salary range", "type": "Details", "note": null, "rating": "no_decision"}, {"name": "Empathetic", "type": "Personality Traits", "note": null, "rating": "no_decision"}, {"name": "Currently based locally", "type": "Details", "note": null, "rating": "no_decision"}, {"name": "Communication", "type": "Skills", "note": null, "rating": "no_decision"}], "ratings": {"definitely_not": [], "no": [], "mixed": [], "yes": [], "strong_yes": []}, "questions": [{"id": null, "question": "Key Take-Aways", "answer": ""}, {"id": null, "question": "Private Notes", "answer": ""}]}, "emitted_at": 1664285619010} {"stream": "scorecards", "data": {"id": 9664505003, "updated_at": "2021-09-29T17:23:11.468Z", "created_at": "2021-09-29T17:23:11.468Z", "interview": "Preliminary Screening Call", "interview_step": {"id": 5628615003, "name": "Preliminary Screening Call"}, "candidate_id": 40517966003, "application_id": 44937562003, "interviewed_at": "2021-09-29T01:00:00.000Z", "submitted_by": {"id": 4218086003, "first_name": "John", "last_name": "Lafleur", "name": "John Lafleur", "employee_id": null}, "interviewer": {"id": 4218086003, "first_name": "John", "last_name": "Lafleur", "name": "John Lafleur", "employee_id": null}, "submitted_at": "2021-09-29T17:23:11.468Z", "overall_recommendation": "no_decision", "attributes": [{"name": "Willing to do required travel", "type": "Details", "note": null, "rating": "yes"}, {"name": "Three to five years of experience", "type": "Qualifications", "note": null, "rating": "mixed"}, {"name": "Personable", "type": "Personality Traits", "note": null, "rating": "yes"}, {"name": "Passionate", "type": "Personality Traits", "note": null, "rating": "mixed"}, {"name": "Organizational Skills", "type": "Skills", "note": null, "rating": "yes"}, {"name": "Manage competing priorities", "type": "Skills", "note": null, "rating": "yes"}, {"name": "Fits our salary range", "type": "Details", "note": null, "rating": "yes"}, {"name": "Empathetic", "type": "Personality Traits", "note": null, "rating": "strong_yes"}, {"name": "Currently based locally", "type": "Details", "note": null, "rating": "mixed"}, {"name": "Communication", "type": "Skills", "note": null, "rating": "no"}], "ratings": {"definitely_not": [], "no": ["Communication"], "mixed": ["Three to five years of experience", "Passionate", "Currently based locally"], "yes": ["Willing to do required travel", "Personable", "Organizational Skills", "Manage competing priorities", "Fits our salary range"], "strong_yes": ["Empathetic"]}, "questions": [{"id": null, "question": "Key Take-Aways", "answer": "test"}, {"id": null, "question": "Private Notes", "answer": ""}]}, "emitted_at": 1664285619011} -{"stream": "users", "data": {"id": 4218085003, "name": "Greenhouse Admin", "first_name": "Greenhouse", "last_name": "Admin", "primary_email_address": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "updated_at": "2020-11-18T14:09:08.401Z", "created_at": "2020-11-18T14:09:08.401Z", "disabled": false, "site_admin": true, "emails": ["scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com"], "employee_id": null, "linked_candidate_ids": []}, "emitted_at": 1664285619554} -{"stream": "users", "data": {"id": 4218086003, "name": "John Lafleur", "first_name": "John", "last_name": "Lafleur", "primary_email_address": "integration-test@airbyte.io", "updated_at": "2022-09-27T13:00:12.685Z", "created_at": "2020-11-18T14:09:08.481Z", "disabled": false, "site_admin": true, "emails": ["integration-test@airbyte.io"], "employee_id": null, "linked_candidate_ids": []}, "emitted_at": 1664285619555} -{"stream": "users", "data": {"id": 4218087003, "name": "emily.brooks+airbyte_integration@greenhouse.io", "first_name": null, "last_name": null, "primary_email_address": "emily.brooks+airbyte_integration@greenhouse.io", "updated_at": "2020-11-18T14:09:08.991Z", "created_at": "2020-11-18T14:09:08.809Z", "disabled": false, "site_admin": true, "emails": ["emily.brooks+airbyte_integration@greenhouse.io"], "employee_id": null, "linked_candidate_ids": []}, "emitted_at": 1664285619555} -{"stream": "users", "data": {"id": 4460715003, "name": "Vadym Ratniuk", "first_name": "Vadym", "last_name": "Ratniuk", "primary_email_address": "vadym.ratniuk@globallogic.com", "updated_at": "2021-09-18T10:09:16.846Z", "created_at": "2021-09-14T14:03:01.050Z", "disabled": false, "site_admin": false, "emails": ["vadym.ratniuk@globallogic.com"], "employee_id": null, "linked_candidate_ids": []}, "emitted_at": 1664285619556} -{"stream": "users", "data": {"id": 4481107003, "name": "Vadym Hevlich", "first_name": "Vadym", "last_name": "Hevlich", "primary_email_address": "vadym.hevlich@zazmic.com", "updated_at": "2021-10-10T17:49:28.058Z", "created_at": "2021-10-10T17:48:41.978Z", "disabled": false, "site_admin": true, "emails": ["vadym.hevlich@zazmic.com"], "employee_id": null, "linked_candidate_ids": []}, "emitted_at": 1664285619556} +{"stream": "users", "data": {"id": 4218085003, "name": "Greenhouse Admin", "first_name": "Greenhouse", "last_name": "Admin", "primary_email_address": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "updated_at": "2020-11-18T14:09:08.401Z", "created_at": "2020-11-18T14:09:08.401Z", "disabled": false, "site_admin": true, "emails": ["scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com"], "employee_id": null, "linked_candidate_ids": [], "offices": [], "departments": []}, "emitted_at": 1664285619554} +{"stream": "users", "data": {"id": 4218086003, "name": "John Lafleur", "first_name": "John", "last_name": "Lafleur", "primary_email_address": "integration-test@airbyte.io", "updated_at": "2022-09-27T13:00:12.685Z", "created_at": "2020-11-18T14:09:08.481Z", "disabled": false, "site_admin": true, "emails": ["integration-test@airbyte.io"], "employee_id": null, "linked_candidate_ids": [], "offices": [], "departments": []}, "emitted_at": 1664285619555} +{"stream": "users", "data": {"id": 4218087003, "name": "emily.brooks+airbyte_integration@greenhouse.io", "first_name": null, "last_name": null, "primary_email_address": "emily.brooks+airbyte_integration@greenhouse.io", "updated_at": "2020-11-18T14:09:08.991Z", "created_at": "2020-11-18T14:09:08.809Z", "disabled": false, "site_admin": true, "emails": ["emily.brooks+airbyte_integration@greenhouse.io"], "employee_id": null, "linked_candidate_ids": [], "offices": [], "departments": []}, "emitted_at": 1664285619555} +{"stream": "users", "data": {"id": 4460715003, "name": "Vadym Ratniuk", "first_name": "Vadym", "last_name": "Ratniuk", "primary_email_address": "vadym.ratniuk@globallogic.com", "updated_at": "2021-09-18T10:09:16.846Z", "created_at": "2021-09-14T14:03:01.050Z", "disabled": false, "site_admin": false, "emails": ["vadym.ratniuk@globallogic.com"], "employee_id": null, "linked_candidate_ids": [], "offices": [], "departments": []}, "emitted_at": 1664285619556} +{"stream": "users", "data": {"id": 4481107003, "name": "Vadym Hevlich", "first_name": "Vadym", "last_name": "Hevlich", "primary_email_address": "vadym.hevlich@zazmic.com", "updated_at": "2021-10-10T17:49:28.058Z", "created_at": "2021-10-10T17:48:41.978Z", "disabled": false, "site_admin": true, "emails": ["vadym.hevlich@zazmic.com"], "employee_id": null, "linked_candidate_ids": [], "offices": [], "departments": []}, "emitted_at": 1664285619556} {"stream": "custom_fields", "data": {"id": 4680898003, "name": "School Name", "active": true, "field_type": "candidate", "priority": 0, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "school_name", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845822003, "name": "Abraham Baldwin Agricultural College", "priority": 0, "external_id": null}, {"id": 10845823003, "name": "Academy of Art University", "priority": 1, "external_id": null}, {"id": 10845824003, "name": "Acadia University", "priority": 2, "external_id": null}, {"id": 10845825003, "name": "Adams State University", "priority": 3, "external_id": null}, {"id": 10845826003, "name": "Adelphi University", "priority": 4, "external_id": null}, {"id": 10845827003, "name": "Adrian College", "priority": 5, "external_id": null}, {"id": 10845828003, "name": "Adventist University of Health Sciences", "priority": 6, "external_id": null}, {"id": 10845829003, "name": "Agnes Scott College", "priority": 7, "external_id": null}, {"id": 10845830003, "name": "AIB College of Business", "priority": 8, "external_id": null}, {"id": 10845831003, "name": "Alaska Pacific University", "priority": 9, "external_id": null}, {"id": 10845832003, "name": "Albany College of Pharmacy and Health Sciences", "priority": 10, "external_id": null}, {"id": 10845833003, "name": "Albany State University", "priority": 11, "external_id": null}, {"id": 10845834003, "name": "Albertus Magnus College", "priority": 12, "external_id": null}, {"id": 10845835003, "name": "Albion College", "priority": 13, "external_id": null}, {"id": 10845836003, "name": "Albright College", "priority": 14, "external_id": null}, {"id": 10845837003, "name": "Alderson Broaddus University", "priority": 15, "external_id": null}, {"id": 10845838003, "name": "Alfred University", "priority": 16, "external_id": null}, {"id": 10845839003, "name": "Alice Lloyd College", "priority": 17, "external_id": null}, {"id": 10845840003, "name": "Allegheny College", "priority": 18, "external_id": null}, {"id": 10845841003, "name": "Allen College", "priority": 19, "external_id": null}, {"id": 10845842003, "name": "Allen University", "priority": 20, "external_id": null}, {"id": 10845843003, "name": "Alliant International University", "priority": 21, "external_id": null}, {"id": 10845844003, "name": "Alma College", "priority": 22, "external_id": null}, {"id": 10845845003, "name": "Alvernia University", "priority": 23, "external_id": null}, {"id": 10845846003, "name": "Alverno College", "priority": 24, "external_id": null}, {"id": 10845847003, "name": "Amberton University", "priority": 25, "external_id": null}, {"id": 10845848003, "name": "American Academy of Art", "priority": 26, "external_id": null}, {"id": 10845849003, "name": "American Indian College of the Assemblies of God", "priority": 27, "external_id": null}, {"id": 10845850003, "name": "American InterContinental University", "priority": 28, "external_id": null}, {"id": 10845851003, "name": "American International College", "priority": 29, "external_id": null}, {"id": 10845852003, "name": "American Jewish University", "priority": 30, "external_id": null}, {"id": 10845853003, "name": "American Public University System", "priority": 31, "external_id": null}, {"id": 10845854003, "name": "American University", "priority": 32, "external_id": null}, {"id": 10845855003, "name": "American University in Bulgaria", "priority": 33, "external_id": null}, {"id": 10845856003, "name": "American University in Cairo", "priority": 34, "external_id": null}, {"id": 10845857003, "name": "American University of Beirut", "priority": 35, "external_id": null}, {"id": 10845858003, "name": "American University of Paris", "priority": 36, "external_id": null}, {"id": 10845859003, "name": "American University of Puerto Rico", "priority": 37, "external_id": null}, {"id": 10845860003, "name": "Amherst College", "priority": 38, "external_id": null}, {"id": 10845861003, "name": "Amridge University", "priority": 39, "external_id": null}, {"id": 10845862003, "name": "Anderson University", "priority": 40, "external_id": null}, {"id": 10845863003, "name": "Andrews University", "priority": 41, "external_id": null}, {"id": 10845864003, "name": "Angelo State University", "priority": 42, "external_id": null}, {"id": 10845865003, "name": "Anna Maria College", "priority": 43, "external_id": null}, {"id": 10845866003, "name": "Antioch University", "priority": 44, "external_id": null}, {"id": 10845867003, "name": "Appalachian Bible College", "priority": 45, "external_id": null}, {"id": 10845868003, "name": "Aquinas College", "priority": 46, "external_id": null}, {"id": 10845869003, "name": "Arcadia University", "priority": 47, "external_id": null}, {"id": 10845870003, "name": "Argosy University", "priority": 48, "external_id": null}, {"id": 10845871003, "name": "Arizona Christian University", "priority": 49, "external_id": null}, {"id": 10845872003, "name": "Arizona State University - West", "priority": 50, "external_id": null}, {"id": 10845873003, "name": "Arkansas Baptist College", "priority": 51, "external_id": null}, {"id": 10845874003, "name": "Arkansas Tech University", "priority": 52, "external_id": null}, {"id": 10845875003, "name": "Armstrong Atlantic State University", "priority": 53, "external_id": null}, {"id": 10845876003, "name": "Art Academy of Cincinnati", "priority": 54, "external_id": null}, {"id": 10845877003, "name": "Art Center College of Design", "priority": 55, "external_id": null}, {"id": 10845878003, "name": "Art Institute of Atlanta", "priority": 56, "external_id": null}, {"id": 10845879003, "name": "Art Institute of Colorado", "priority": 57, "external_id": null}, {"id": 10845880003, "name": "Art Institute of Houston", "priority": 58, "external_id": null}, {"id": 10845881003, "name": "Art Institute of Pittsburgh", "priority": 59, "external_id": null}, {"id": 10845882003, "name": "Art Institute of Portland", "priority": 60, "external_id": null}, {"id": 10845883003, "name": "Art Institute of Seattle", "priority": 61, "external_id": null}, {"id": 10845884003, "name": "Asbury University", "priority": 62, "external_id": null}, {"id": 10845885003, "name": "Ashford University", "priority": 63, "external_id": null}, {"id": 10845886003, "name": "Ashland University", "priority": 64, "external_id": null}, {"id": 10845887003, "name": "Assumption College", "priority": 65, "external_id": null}, {"id": 10845888003, "name": "Athens State University", "priority": 66, "external_id": null}, {"id": 10845889003, "name": "Auburn University - Montgomery", "priority": 67, "external_id": null}, {"id": 10845890003, "name": "Augsburg College", "priority": 68, "external_id": null}, {"id": 10845891003, "name": "Augustana College", "priority": 69, "external_id": null}, {"id": 10845892003, "name": "Aurora University", "priority": 70, "external_id": null}, {"id": 10845893003, "name": "Austin College", "priority": 71, "external_id": null}, {"id": 10845894003, "name": "Alcorn State University", "priority": 72, "external_id": null}, {"id": 10845895003, "name": "Ave Maria University", "priority": 73, "external_id": null}, {"id": 10845896003, "name": "Averett University", "priority": 74, "external_id": null}, {"id": 10845897003, "name": "Avila University", "priority": 75, "external_id": null}, {"id": 10845898003, "name": "Azusa Pacific University", "priority": 76, "external_id": null}, {"id": 10845899003, "name": "Babson College", "priority": 77, "external_id": null}, {"id": 10845900003, "name": "Bacone College", "priority": 78, "external_id": null}, {"id": 10845901003, "name": "Baker College of Flint", "priority": 79, "external_id": null}, {"id": 10845902003, "name": "Baker University", "priority": 80, "external_id": null}, {"id": 10845903003, "name": "Baldwin Wallace University", "priority": 81, "external_id": null}, {"id": 10845904003, "name": "Christian Brothers University", "priority": 82, "external_id": null}, {"id": 10845905003, "name": "Abilene Christian University", "priority": 83, "external_id": null}, {"id": 10845906003, "name": "Arizona State University", "priority": 84, "external_id": null}, {"id": 10845907003, "name": "Auburn University", "priority": 85, "external_id": null}, {"id": 10845908003, "name": "Alabama A&M University", "priority": 86, "external_id": null}, {"id": 10845909003, "name": "Alabama State University", "priority": 87, "external_id": null}, {"id": 10845910003, "name": "Arkansas State University", "priority": 88, "external_id": null}, {"id": 10845911003, "name": "Baptist Bible College", "priority": 89, "external_id": null}, {"id": 10845912003, "name": "Baptist Bible College and Seminary", "priority": 90, "external_id": null}, {"id": 10845913003, "name": "Baptist College of Florida", "priority": 91, "external_id": null}, {"id": 10845914003, "name": "Baptist Memorial College of Health Sciences", "priority": 92, "external_id": null}, {"id": 10845915003, "name": "Baptist Missionary Association Theological Seminary", "priority": 93, "external_id": null}, {"id": 10845916003, "name": "Bard College", "priority": 94, "external_id": null}, {"id": 10845917003, "name": "Bard College at Simon's Rock", "priority": 95, "external_id": null}, {"id": 10845918003, "name": "Barnard College", "priority": 96, "external_id": null}, {"id": 10845919003, "name": "Barry University", "priority": 97, "external_id": null}, {"id": 10845920003, "name": "Barton College", "priority": 98, "external_id": null}, {"id": 10845921003, "name": "Bastyr University", "priority": 99, "external_id": null}, {"id": 10845922003, "name": "Bates College", "priority": 100, "external_id": null}, {"id": 10845923003, "name": "Bauder College", "priority": 101, "external_id": null}, {"id": 10845924003, "name": "Bay Path College", "priority": 102, "external_id": null}, {"id": 10845925003, "name": "Bay State College", "priority": 103, "external_id": null}, {"id": 10845926003, "name": "Bayamon Central University", "priority": 104, "external_id": null}, {"id": 10845927003, "name": "Beacon College", "priority": 105, "external_id": null}, {"id": 10845928003, "name": "Becker College", "priority": 106, "external_id": null}, {"id": 10845929003, "name": "Belhaven University", "priority": 107, "external_id": null}, {"id": 10845930003, "name": "Bellarmine University", "priority": 108, "external_id": null}, {"id": 10845931003, "name": "Bellevue College", "priority": 109, "external_id": null}, {"id": 10845932003, "name": "Bellevue University", "priority": 110, "external_id": null}, {"id": 10845933003, "name": "Bellin College", "priority": 111, "external_id": null}, {"id": 10845934003, "name": "Belmont Abbey College", "priority": 112, "external_id": null}, {"id": 10845935003, "name": "Belmont University", "priority": 113, "external_id": null}, {"id": 10845936003, "name": "Beloit College", "priority": 114, "external_id": null}, {"id": 10845937003, "name": "Bemidji State University", "priority": 115, "external_id": null}, {"id": 10845938003, "name": "Benedict College", "priority": 116, "external_id": null}, {"id": 10845939003, "name": "Benedictine College", "priority": 117, "external_id": null}, {"id": 10845940003, "name": "Benedictine University", "priority": 118, "external_id": null}, {"id": 10845941003, "name": "Benjamin Franklin Institute of Technology", "priority": 119, "external_id": null}, {"id": 10845942003, "name": "Bennett College", "priority": 120, "external_id": null}, {"id": 10845943003, "name": "Bennington College", "priority": 121, "external_id": null}, {"id": 10845944003, "name": "Bentley University", "priority": 122, "external_id": null}, {"id": 10845945003, "name": "Berea College", "priority": 123, "external_id": null}, {"id": 10845946003, "name": "Berkeley College", "priority": 124, "external_id": null}, {"id": 10845947003, "name": "Berklee College of Music", "priority": 125, "external_id": null}, {"id": 10845948003, "name": "Berry College", "priority": 126, "external_id": null}, {"id": 10845949003, "name": "Bethany College", "priority": 127, "external_id": null}, {"id": 10845950003, "name": "Bethany Lutheran College", "priority": 128, "external_id": null}, {"id": 10845951003, "name": "Bethel College", "priority": 129, "external_id": null}, {"id": 10845952003, "name": "Bethel University", "priority": 130, "external_id": null}, {"id": 10845953003, "name": "BI Norwegian Business School", "priority": 131, "external_id": null}, {"id": 10845954003, "name": "Binghamton University - SUNY", "priority": 132, "external_id": null}, {"id": 10845955003, "name": "Biola University", "priority": 133, "external_id": null}, {"id": 10845956003, "name": "Birmingham-Southern College", "priority": 134, "external_id": null}, {"id": 10845957003, "name": "Bismarck State College", "priority": 135, "external_id": null}, {"id": 10845958003, "name": "Black Hills State University", "priority": 136, "external_id": null}, {"id": 10845959003, "name": "Blackburn College", "priority": 137, "external_id": null}, {"id": 10845960003, "name": "Blessing-Rieman College of Nursing", "priority": 138, "external_id": null}, {"id": 10845961003, "name": "Bloomfield College", "priority": 139, "external_id": null}, {"id": 10845962003, "name": "Bloomsburg University of Pennsylvania", "priority": 140, "external_id": null}, {"id": 10845963003, "name": "Blue Mountain College", "priority": 141, "external_id": null}, {"id": 10845964003, "name": "Bluefield College", "priority": 142, "external_id": null}, {"id": 10845965003, "name": "Bluefield State College", "priority": 143, "external_id": null}, {"id": 10845966003, "name": "Bluffton University", "priority": 144, "external_id": null}, {"id": 10845967003, "name": "Boricua College", "priority": 145, "external_id": null}, {"id": 10845968003, "name": "Boston Architectural College", "priority": 146, "external_id": null}, {"id": 10845969003, "name": "Boston Conservatory", "priority": 147, "external_id": null}, {"id": 10845970003, "name": "Boston University", "priority": 148, "external_id": null}, {"id": 10845971003, "name": "Bowdoin College", "priority": 149, "external_id": null}, {"id": 10845972003, "name": "Bowie State University", "priority": 150, "external_id": null}, {"id": 10845973003, "name": "Bradley University", "priority": 151, "external_id": null}, {"id": 10845974003, "name": "Brandeis University", "priority": 152, "external_id": null}, {"id": 10845975003, "name": "Brandman University", "priority": 153, "external_id": null}, {"id": 10845976003, "name": "Brazosport College", "priority": 154, "external_id": null}, {"id": 10845977003, "name": "Brenau University", "priority": 155, "external_id": null}, {"id": 10845978003, "name": "Brescia University", "priority": 156, "external_id": null}, {"id": 10845979003, "name": "Brevard College", "priority": 157, "external_id": null}, {"id": 10845980003, "name": "Brewton-Parker College", "priority": 158, "external_id": null}, {"id": 10845981003, "name": "Briar Cliff University", "priority": 159, "external_id": null}, {"id": 10845982003, "name": "Briarcliffe College", "priority": 160, "external_id": null}, {"id": 10845983003, "name": "Bridgewater College", "priority": 161, "external_id": null}, {"id": 10845984003, "name": "Bridgewater State University", "priority": 162, "external_id": null}, {"id": 10845985003, "name": "Brigham Young University - Hawaii", "priority": 163, "external_id": null}, {"id": 10845986003, "name": "Brigham Young University - Idaho", "priority": 164, "external_id": null}, {"id": 10845987003, "name": "Brock University", "priority": 165, "external_id": null}, {"id": 10845988003, "name": "Bryan College", "priority": 166, "external_id": null}, {"id": 10845989003, "name": "Bryn Athyn College of the New Church", "priority": 167, "external_id": null}, {"id": 10845990003, "name": "Bryn Mawr College", "priority": 168, "external_id": null}, {"id": 10845991003, "name": "Boston College", "priority": 169, "external_id": null}, {"id": 10845992003, "name": "Buena Vista University", "priority": 170, "external_id": null}, {"id": 10845993003, "name": "Burlington College", "priority": 171, "external_id": null}, {"id": 10845994003, "name": "Bowling Green State University", "priority": 172, "external_id": null}, {"id": 10845995003, "name": "Brown University", "priority": 173, "external_id": null}, {"id": 10845996003, "name": "Appalachian State University", "priority": 174, "external_id": null}, {"id": 10845997003, "name": "Brigham Young University - Provo", "priority": 175, "external_id": null}, {"id": 10845998003, "name": "Boise State University", "priority": 176, "external_id": null}, {"id": 10845999003, "name": "Bethune-Cookman University", "priority": 177, "external_id": null}, {"id": 10846000003, "name": "Bryant University", "priority": 178, "external_id": null}, {"id": 10846001003, "name": "Cabarrus College of Health Sciences", "priority": 179, "external_id": null}, {"id": 10846002003, "name": "Cabrini College", "priority": 180, "external_id": null}, {"id": 10846003003, "name": "Cairn University", "priority": 181, "external_id": null}, {"id": 10846004003, "name": "Caldwell College", "priority": 182, "external_id": null}, {"id": 10846005003, "name": "California Baptist University", "priority": 183, "external_id": null}, {"id": 10846006003, "name": "California College of the Arts", "priority": 184, "external_id": null}, {"id": 10846007003, "name": "California Institute of Integral Studies", "priority": 185, "external_id": null}, {"id": 10846008003, "name": "California Institute of Technology", "priority": 186, "external_id": null}, {"id": 10846009003, "name": "California Institute of the Arts", "priority": 187, "external_id": null}, {"id": 10846010003, "name": "California Lutheran University", "priority": 188, "external_id": null}, {"id": 10846011003, "name": "California Maritime Academy", "priority": 189, "external_id": null}, {"id": 10846012003, "name": "California State Polytechnic University - Pomona", "priority": 190, "external_id": null}, {"id": 10846013003, "name": "California State University - Bakersfield", "priority": 191, "external_id": null}, {"id": 10846014003, "name": "California State University - Channel Islands", "priority": 192, "external_id": null}, {"id": 10846015003, "name": "California State University - Chico", "priority": 193, "external_id": null}, {"id": 10846016003, "name": "California State University - Dominguez Hills", "priority": 194, "external_id": null}, {"id": 10846017003, "name": "California State University - East Bay", "priority": 195, "external_id": null}, {"id": 10846018003, "name": "California State University - Fullerton", "priority": 196, "external_id": null}, {"id": 10846019003, "name": "California State University - Los Angeles", "priority": 197, "external_id": null}, {"id": 10846020003, "name": "California State University - Monterey Bay", "priority": 198, "external_id": null}, {"id": 10846021003, "name": "California State University - Northridge", "priority": 199, "external_id": null}, {"id": 10846022003, "name": "California State University - San Bernardino", "priority": 200, "external_id": null}, {"id": 10846023003, "name": "California State University - San Marcos", "priority": 201, "external_id": null}, {"id": 10846024003, "name": "California State University - Stanislaus", "priority": 202, "external_id": null}, {"id": 10846025003, "name": "California University of Pennsylvania", "priority": 203, "external_id": null}, {"id": 10846026003, "name": "Calumet College of St. Joseph", "priority": 204, "external_id": null}, {"id": 10846027003, "name": "Calvary Bible College and Theological Seminary", "priority": 205, "external_id": null}, {"id": 10846028003, "name": "Calvin College", "priority": 206, "external_id": null}, {"id": 10846029003, "name": "Cambridge College", "priority": 207, "external_id": null}, {"id": 10846030003, "name": "Cameron University", "priority": 208, "external_id": null}, {"id": 10846031003, "name": "Campbellsville University", "priority": 209, "external_id": null}, {"id": 10846032003, "name": "Canisius College", "priority": 210, "external_id": null}, {"id": 10846033003, "name": "Capella University", "priority": 211, "external_id": null}, {"id": 10846034003, "name": "Capital University", "priority": 212, "external_id": null}, {"id": 10846035003, "name": "Capitol College", "priority": 213, "external_id": null}, {"id": 10846036003, "name": "Cardinal Stritch University", "priority": 214, "external_id": null}, {"id": 10846037003, "name": "Caribbean University", "priority": 215, "external_id": null}, {"id": 10846038003, "name": "Carleton College", "priority": 216, "external_id": null}, {"id": 10846039003, "name": "Carleton University", "priority": 217, "external_id": null}, {"id": 10846040003, "name": "Carlos Albizu University", "priority": 218, "external_id": null}, {"id": 10846041003, "name": "Carlow University", "priority": 219, "external_id": null}, {"id": 10846042003, "name": "Carnegie Mellon University", "priority": 220, "external_id": null}, {"id": 10846043003, "name": "Carroll College", "priority": 221, "external_id": null}, {"id": 10846044003, "name": "Carroll University", "priority": 222, "external_id": null}, {"id": 10846045003, "name": "Carson-Newman University", "priority": 223, "external_id": null}, {"id": 10846046003, "name": "Carthage College", "priority": 224, "external_id": null}, {"id": 10846047003, "name": "Case Western Reserve University", "priority": 225, "external_id": null}, {"id": 10846048003, "name": "Castleton State College", "priority": 226, "external_id": null}, {"id": 10846049003, "name": "Catawba College", "priority": 227, "external_id": null}, {"id": 10846050003, "name": "Cazenovia College", "priority": 228, "external_id": null}, {"id": 10846051003, "name": "Cedar Crest College", "priority": 229, "external_id": null}, {"id": 10846052003, "name": "Cedarville University", "priority": 230, "external_id": null}, {"id": 10846053003, "name": "Centenary College", "priority": 231, "external_id": null}, {"id": 10846054003, "name": "Centenary College of Louisiana", "priority": 232, "external_id": null}, {"id": 10846055003, "name": "Central Baptist College", "priority": 233, "external_id": null}, {"id": 10846056003, "name": "Central Bible College", "priority": 234, "external_id": null}, {"id": 10846057003, "name": "Central Christian College", "priority": 235, "external_id": null}, {"id": 10846058003, "name": "Central College", "priority": 236, "external_id": null}, {"id": 10846059003, "name": "Central Methodist University", "priority": 237, "external_id": null}, {"id": 10846060003, "name": "Central Penn College", "priority": 238, "external_id": null}, {"id": 10846061003, "name": "Central State University", "priority": 239, "external_id": null}, {"id": 10846062003, "name": "Central Washington University", "priority": 240, "external_id": null}, {"id": 10846063003, "name": "Centre College", "priority": 241, "external_id": null}, {"id": 10846064003, "name": "Chadron State College", "priority": 242, "external_id": null}, {"id": 10846065003, "name": "Chamberlain College of Nursing", "priority": 243, "external_id": null}, {"id": 10846066003, "name": "Chaminade University of Honolulu", "priority": 244, "external_id": null}, {"id": 10846067003, "name": "Champlain College", "priority": 245, "external_id": null}, {"id": 10846068003, "name": "Chancellor University", "priority": 246, "external_id": null}, {"id": 10846069003, "name": "Chapman University", "priority": 247, "external_id": null}, {"id": 10846070003, "name": "Charles R. Drew University of Medicine and Science", "priority": 248, "external_id": null}, {"id": 10846071003, "name": "Charter Oak State College", "priority": 249, "external_id": null}, {"id": 10846072003, "name": "Chatham University", "priority": 250, "external_id": null}, {"id": 10846073003, "name": "Chestnut Hill College", "priority": 251, "external_id": null}, {"id": 10846074003, "name": "Cheyney University of Pennsylvania", "priority": 252, "external_id": null}, {"id": 10846075003, "name": "Chicago State University", "priority": 253, "external_id": null}, {"id": 10846076003, "name": "Chipola College", "priority": 254, "external_id": null}, {"id": 10846077003, "name": "Chowan University", "priority": 255, "external_id": null}, {"id": 10846078003, "name": "Christendom College", "priority": 256, "external_id": null}, {"id": 10846079003, "name": "Baylor University", "priority": 257, "external_id": null}, {"id": 10846080003, "name": "Central Connecticut State University", "priority": 258, "external_id": null}, {"id": 10846081003, "name": "Central Michigan University", "priority": 259, "external_id": null}, {"id": 10846082003, "name": "Charleston Southern University", "priority": 260, "external_id": null}, {"id": 10846083003, "name": "California State University - Sacramento", "priority": 261, "external_id": null}, {"id": 10846084003, "name": "California State University - Fresno", "priority": 262, "external_id": null}, {"id": 10846085003, "name": "Campbell University", "priority": 263, "external_id": null}, {"id": 10846086003, "name": "Christopher Newport University", "priority": 264, "external_id": null}, {"id": 10846087003, "name": "Cincinnati Christian University", "priority": 265, "external_id": null}, {"id": 10846088003, "name": "Cincinnati College of Mortuary Science", "priority": 266, "external_id": null}, {"id": 10846089003, "name": "City University of Seattle", "priority": 267, "external_id": null}, {"id": 10846090003, "name": "Claflin University", "priority": 268, "external_id": null}, {"id": 10846091003, "name": "Claremont McKenna College", "priority": 269, "external_id": null}, {"id": 10846092003, "name": "Clarion University of Pennsylvania", "priority": 270, "external_id": null}, {"id": 10846093003, "name": "Clark Atlanta University", "priority": 271, "external_id": null}, {"id": 10846094003, "name": "Clark University", "priority": 272, "external_id": null}, {"id": 10846095003, "name": "Clarke University", "priority": 273, "external_id": null}, {"id": 10846096003, "name": "Clarkson College", "priority": 274, "external_id": null}, {"id": 10846097003, "name": "Clarkson University", "priority": 275, "external_id": null}, {"id": 10846098003, "name": "Clayton State University", "priority": 276, "external_id": null}, {"id": 10846099003, "name": "Clear Creek Baptist Bible College", "priority": 277, "external_id": null}, {"id": 10846100003, "name": "Clearwater Christian College", "priority": 278, "external_id": null}, {"id": 10846101003, "name": "Cleary University", "priority": 279, "external_id": null}, {"id": 10846102003, "name": "College of William and Mary", "priority": 280, "external_id": null}, {"id": 10846103003, "name": "Cleveland Chiropractic College", "priority": 281, "external_id": null}, {"id": 10846104003, "name": "Cleveland Institute of Art", "priority": 282, "external_id": null}, {"id": 10846105003, "name": "Cleveland Institute of Music", "priority": 283, "external_id": null}, {"id": 10846106003, "name": "Cleveland State University", "priority": 284, "external_id": null}, {"id": 10846107003, "name": "Coe College", "priority": 285, "external_id": null}, {"id": 10846108003, "name": "Cogswell Polytechnical College", "priority": 286, "external_id": null}, {"id": 10846109003, "name": "Coker College", "priority": 287, "external_id": null}, {"id": 10846110003, "name": "Colby College", "priority": 288, "external_id": null}, {"id": 10846111003, "name": "Colby-Sawyer College", "priority": 289, "external_id": null}, {"id": 10846112003, "name": "College at Brockport - SUNY", "priority": 290, "external_id": null}, {"id": 10846113003, "name": "College for Creative Studies", "priority": 291, "external_id": null}, {"id": 10846114003, "name": "College of Charleston", "priority": 292, "external_id": null}, {"id": 10846115003, "name": "College of Idaho", "priority": 293, "external_id": null}, {"id": 10846116003, "name": "College of Mount St. Joseph", "priority": 294, "external_id": null}, {"id": 10846117003, "name": "College of Mount St. Vincent", "priority": 295, "external_id": null}, {"id": 10846118003, "name": "College of New Jersey", "priority": 296, "external_id": null}, {"id": 10846119003, "name": "College of New Rochelle", "priority": 297, "external_id": null}, {"id": 10846120003, "name": "College of Our Lady of the Elms", "priority": 298, "external_id": null}, {"id": 10846121003, "name": "College of Saints John Fisher & Thomas More", "priority": 299, "external_id": null}, {"id": 10846122003, "name": "College of Southern Nevada", "priority": 300, "external_id": null}, {"id": 10846123003, "name": "College of St. Benedict", "priority": 301, "external_id": null}, {"id": 10846124003, "name": "College of St. Elizabeth", "priority": 302, "external_id": null}, {"id": 10846125003, "name": "College of St. Joseph", "priority": 303, "external_id": null}, {"id": 10846126003, "name": "College of St. Mary", "priority": 304, "external_id": null}, {"id": 10846127003, "name": "College of St. Rose", "priority": 305, "external_id": null}, {"id": 10846128003, "name": "College of St. Scholastica", "priority": 306, "external_id": null}, {"id": 10846129003, "name": "College of the Atlantic", "priority": 307, "external_id": null}, {"id": 10846130003, "name": "College of the Holy Cross", "priority": 308, "external_id": null}, {"id": 10846131003, "name": "College of the Ozarks", "priority": 309, "external_id": null}, {"id": 10846132003, "name": "College of Wooster", "priority": 310, "external_id": null}, {"id": 10846133003, "name": "Colorado Christian University", "priority": 311, "external_id": null}, {"id": 10846134003, "name": "Colorado College", "priority": 312, "external_id": null}, {"id": 10846135003, "name": "Colorado Mesa University", "priority": 313, "external_id": null}, {"id": 10846136003, "name": "Colorado School of Mines", "priority": 314, "external_id": null}, {"id": 10846137003, "name": "Colorado State University - Pueblo", "priority": 315, "external_id": null}, {"id": 10846138003, "name": "Colorado Technical University", "priority": 316, "external_id": null}, {"id": 10846139003, "name": "Columbia College", "priority": 317, "external_id": null}, {"id": 10846140003, "name": "Columbia College Chicago", "priority": 318, "external_id": null}, {"id": 10846141003, "name": "Columbia College of Nursing", "priority": 319, "external_id": null}, {"id": 10846142003, "name": "Columbia International University", "priority": 320, "external_id": null}, {"id": 10846143003, "name": "Columbus College of Art and Design", "priority": 321, "external_id": null}, {"id": 10846144003, "name": "Columbus State University", "priority": 322, "external_id": null}, {"id": 10846145003, "name": "Conception Seminary College", "priority": 323, "external_id": null}, {"id": 10846146003, "name": "Concord University", "priority": 324, "external_id": null}, {"id": 10846147003, "name": "Concordia College", "priority": 325, "external_id": null}, {"id": 10846148003, "name": "Concordia College - Moorhead", "priority": 326, "external_id": null}, {"id": 10846149003, "name": "Concordia University", "priority": 327, "external_id": null}, {"id": 10846150003, "name": "Concordia University Chicago", "priority": 328, "external_id": null}, {"id": 10846151003, "name": "Concordia University Texas", "priority": 329, "external_id": null}, {"id": 10846152003, "name": "Concordia University Wisconsin", "priority": 330, "external_id": null}, {"id": 10846153003, "name": "Concordia University - St. Paul", "priority": 331, "external_id": null}, {"id": 10846154003, "name": "Connecticut College", "priority": 332, "external_id": null}, {"id": 10846155003, "name": "Converse College", "priority": 333, "external_id": null}, {"id": 10846156003, "name": "Cooper Union", "priority": 334, "external_id": null}, {"id": 10846157003, "name": "Coppin State University", "priority": 335, "external_id": null}, {"id": 10846158003, "name": "Corban University", "priority": 336, "external_id": null}, {"id": 10846159003, "name": "Corcoran College of Art and Design", "priority": 337, "external_id": null}, {"id": 10846160003, "name": "Cornell College", "priority": 338, "external_id": null}, {"id": 10846161003, "name": "Cornerstone University", "priority": 339, "external_id": null}, {"id": 10846162003, "name": "Cornish College of the Arts", "priority": 340, "external_id": null}, {"id": 10846163003, "name": "Covenant College", "priority": 341, "external_id": null}, {"id": 10846164003, "name": "Cox College", "priority": 342, "external_id": null}, {"id": 10846165003, "name": "Creighton University", "priority": 343, "external_id": null}, {"id": 10846166003, "name": "Criswell College", "priority": 344, "external_id": null}, {"id": 10846167003, "name": "Crown College", "priority": 345, "external_id": null}, {"id": 10846168003, "name": "Culinary Institute of America", "priority": 346, "external_id": null}, {"id": 10846169003, "name": "Culver-Stockton College", "priority": 347, "external_id": null}, {"id": 10846170003, "name": "Cumberland University", "priority": 348, "external_id": null}, {"id": 10846171003, "name": "Columbia University", "priority": 349, "external_id": null}, {"id": 10846172003, "name": "Cornell University", "priority": 350, "external_id": null}, {"id": 10846173003, "name": "Colorado State University", "priority": 351, "external_id": null}, {"id": 10846174003, "name": "University of Virginia", "priority": 352, "external_id": null}, {"id": 10846175003, "name": "Colgate University", "priority": 353, "external_id": null}, {"id": 10846176003, "name": "CUNY - Baruch College", "priority": 354, "external_id": null}, {"id": 10846177003, "name": "CUNY - Brooklyn College", "priority": 355, "external_id": null}, {"id": 10846178003, "name": "CUNY - City College", "priority": 356, "external_id": null}, {"id": 10846179003, "name": "CUNY - College of Staten Island", "priority": 357, "external_id": null}, {"id": 10846180003, "name": "CUNY - Hunter College", "priority": 358, "external_id": null}, {"id": 10846181003, "name": "CUNY - John Jay College of Criminal Justice", "priority": 359, "external_id": null}, {"id": 10846182003, "name": "CUNY - Lehman College", "priority": 360, "external_id": null}, {"id": 10846183003, "name": "CUNY - Medgar Evers College", "priority": 361, "external_id": null}, {"id": 10846184003, "name": "CUNY - New York City College of Technology", "priority": 362, "external_id": null}, {"id": 10846185003, "name": "CUNY - Queens College", "priority": 363, "external_id": null}, {"id": 10846186003, "name": "CUNY - York College", "priority": 364, "external_id": null}, {"id": 10846187003, "name": "Curry College", "priority": 365, "external_id": null}, {"id": 10846188003, "name": "Curtis Institute of Music", "priority": 366, "external_id": null}, {"id": 10846189003, "name": "D'Youville College", "priority": 367, "external_id": null}, {"id": 10846190003, "name": "Daemen College", "priority": 368, "external_id": null}, {"id": 10846191003, "name": "Dakota State University", "priority": 369, "external_id": null}, {"id": 10846192003, "name": "Dakota Wesleyan University", "priority": 370, "external_id": null}, {"id": 10846193003, "name": "Dalhousie University", "priority": 371, "external_id": null}, {"id": 10846194003, "name": "Dallas Baptist University", "priority": 372, "external_id": null}, {"id": 10846195003, "name": "Dallas Christian College", "priority": 373, "external_id": null}, {"id": 10846196003, "name": "Dalton State College", "priority": 374, "external_id": null}, {"id": 10846197003, "name": "Daniel Webster College", "priority": 375, "external_id": null}, {"id": 10846198003, "name": "Davenport University", "priority": 376, "external_id": null}, {"id": 10846199003, "name": "Davis and Elkins College", "priority": 377, "external_id": null}, {"id": 10846200003, "name": "Davis College", "priority": 378, "external_id": null}, {"id": 10846201003, "name": "Daytona State College", "priority": 379, "external_id": null}, {"id": 10846202003, "name": "Dean College", "priority": 380, "external_id": null}, {"id": 10846203003, "name": "Defiance College", "priority": 381, "external_id": null}, {"id": 10846204003, "name": "Delaware Valley College", "priority": 382, "external_id": null}, {"id": 10846205003, "name": "Delta State University", "priority": 383, "external_id": null}, {"id": 10846206003, "name": "Denison University", "priority": 384, "external_id": null}, {"id": 10846207003, "name": "DePaul University", "priority": 385, "external_id": null}, {"id": 10846208003, "name": "DePauw University", "priority": 386, "external_id": null}, {"id": 10846209003, "name": "DEREE - The American College of Greece", "priority": 387, "external_id": null}, {"id": 10846210003, "name": "DeSales University", "priority": 388, "external_id": null}, {"id": 10846211003, "name": "DeVry University", "priority": 389, "external_id": null}, {"id": 10846212003, "name": "Dickinson College", "priority": 390, "external_id": null}, {"id": 10846213003, "name": "Dickinson State University", "priority": 391, "external_id": null}, {"id": 10846214003, "name": "Dillard University", "priority": 392, "external_id": null}, {"id": 10846215003, "name": "Divine Word College", "priority": 393, "external_id": null}, {"id": 10846216003, "name": "Dixie State College of Utah", "priority": 394, "external_id": null}, {"id": 10846217003, "name": "Doane College", "priority": 395, "external_id": null}, {"id": 10846218003, "name": "Dominican College", "priority": 396, "external_id": null}, {"id": 10846219003, "name": "Dominican University", "priority": 397, "external_id": null}, {"id": 10846220003, "name": "Dominican University of California", "priority": 398, "external_id": null}, {"id": 10846221003, "name": "Donnelly College", "priority": 399, "external_id": null}, {"id": 10846222003, "name": "Dordt College", "priority": 400, "external_id": null}, {"id": 10846223003, "name": "Dowling College", "priority": 401, "external_id": null}, {"id": 10846224003, "name": "Drew University", "priority": 402, "external_id": null}, {"id": 10846225003, "name": "Drexel University", "priority": 403, "external_id": null}, {"id": 10846226003, "name": "Drury University", "priority": 404, "external_id": null}, {"id": 10846227003, "name": "Dunwoody College of Technology", "priority": 405, "external_id": null}, {"id": 10846228003, "name": "Earlham College", "priority": 406, "external_id": null}, {"id": 10846229003, "name": "Drake University", "priority": 407, "external_id": null}, {"id": 10846230003, "name": "East Central University", "priority": 408, "external_id": null}, {"id": 10846231003, "name": "East Stroudsburg University of Pennsylvania", "priority": 409, "external_id": null}, {"id": 10846232003, "name": "East Tennessee State University", "priority": 410, "external_id": null}, {"id": 10846233003, "name": "East Texas Baptist University", "priority": 411, "external_id": null}, {"id": 10846234003, "name": "East-West University", "priority": 412, "external_id": null}, {"id": 10846235003, "name": "Eastern Connecticut State University", "priority": 413, "external_id": null}, {"id": 10846236003, "name": "Eastern Mennonite University", "priority": 414, "external_id": null}, {"id": 10846237003, "name": "Eastern Nazarene College", "priority": 415, "external_id": null}, {"id": 10846238003, "name": "Eastern New Mexico University", "priority": 416, "external_id": null}, {"id": 10846239003, "name": "Eastern Oregon University", "priority": 417, "external_id": null}, {"id": 10846240003, "name": "Eastern University", "priority": 418, "external_id": null}, {"id": 10846241003, "name": "Eckerd College", "priority": 419, "external_id": null}, {"id": 10846242003, "name": "ECPI University", "priority": 420, "external_id": null}, {"id": 10846243003, "name": "Edgewood College", "priority": 421, "external_id": null}, {"id": 10846244003, "name": "Edinboro University of Pennsylvania", "priority": 422, "external_id": null}, {"id": 10846245003, "name": "Edison State College", "priority": 423, "external_id": null}, {"id": 10846246003, "name": "Edward Waters College", "priority": 424, "external_id": null}, {"id": 10846247003, "name": "Elizabeth City State University", "priority": 425, "external_id": null}, {"id": 10846248003, "name": "Elizabethtown College", "priority": 426, "external_id": null}, {"id": 10846249003, "name": "Elmhurst College", "priority": 427, "external_id": null}, {"id": 10846250003, "name": "Elmira College", "priority": 428, "external_id": null}, {"id": 10846251003, "name": "Embry-Riddle Aeronautical University", "priority": 429, "external_id": null}, {"id": 10846252003, "name": "Embry-Riddle Aeronautical University - Prescott", "priority": 430, "external_id": null}, {"id": 10846253003, "name": "Emerson College", "priority": 431, "external_id": null}, {"id": 10846254003, "name": "Duquesne University", "priority": 432, "external_id": null}, {"id": 10846255003, "name": "Eastern Washington University", "priority": 433, "external_id": null}, {"id": 10846256003, "name": "Eastern Illinois University", "priority": 434, "external_id": null}, {"id": 10846257003, "name": "Eastern Kentucky University", "priority": 435, "external_id": null}, {"id": 10846258003, "name": "Eastern Michigan University", "priority": 436, "external_id": null}, {"id": 10846259003, "name": "Elon University", "priority": 437, "external_id": null}, {"id": 10846260003, "name": "Delaware State University", "priority": 438, "external_id": null}, {"id": 10846261003, "name": "Duke University", "priority": 439, "external_id": null}, {"id": 10846262003, "name": "California Polytechnic State University - San Luis Obispo", "priority": 440, "external_id": null}, {"id": 10846263003, "name": "Emmanuel College", "priority": 441, "external_id": null}, {"id": 10846264003, "name": "Emmaus Bible College", "priority": 442, "external_id": null}, {"id": 10846265003, "name": "Emory and Henry College", "priority": 443, "external_id": null}, {"id": 10846266003, "name": "Emory University", "priority": 444, "external_id": null}, {"id": 10846267003, "name": "Emporia State University", "priority": 445, "external_id": null}, {"id": 10846268003, "name": "Endicott College", "priority": 446, "external_id": null}, {"id": 10846269003, "name": "Erskine College", "priority": 447, "external_id": null}, {"id": 10846270003, "name": "Escuela de Artes Plasticas de Puerto Rico", "priority": 448, "external_id": null}, {"id": 10846271003, "name": "Eureka College", "priority": 449, "external_id": null}, {"id": 10846272003, "name": "Evangel University", "priority": 450, "external_id": null}, {"id": 10846273003, "name": "Everest College - Phoenix", "priority": 451, "external_id": null}, {"id": 10846274003, "name": "Everglades University", "priority": 452, "external_id": null}, {"id": 10846275003, "name": "Evergreen State College", "priority": 453, "external_id": null}, {"id": 10846276003, "name": "Excelsior College", "priority": 454, "external_id": null}, {"id": 10846277003, "name": "Fairfield University", "priority": 455, "external_id": null}, {"id": 10846278003, "name": "Fairleigh Dickinson University", "priority": 456, "external_id": null}, {"id": 10846279003, "name": "Fairmont State University", "priority": 457, "external_id": null}, {"id": 10846280003, "name": "Faith Baptist Bible College and Theological Seminary", "priority": 458, "external_id": null}, {"id": 10846281003, "name": "Farmingdale State College - SUNY", "priority": 459, "external_id": null}, {"id": 10846282003, "name": "Fashion Institute of Technology", "priority": 460, "external_id": null}, {"id": 10846283003, "name": "Faulkner University", "priority": 461, "external_id": null}, {"id": 10846284003, "name": "Fayetteville State University", "priority": 462, "external_id": null}, {"id": 10846285003, "name": "Felician College", "priority": 463, "external_id": null}, {"id": 10846286003, "name": "Ferris State University", "priority": 464, "external_id": null}, {"id": 10846287003, "name": "Ferrum College", "priority": 465, "external_id": null}, {"id": 10846288003, "name": "Finlandia University", "priority": 466, "external_id": null}, {"id": 10846289003, "name": "Fisher College", "priority": 467, "external_id": null}, {"id": 10846290003, "name": "Fisk University", "priority": 468, "external_id": null}, {"id": 10846291003, "name": "Fitchburg State University", "priority": 469, "external_id": null}, {"id": 10846292003, "name": "Five Towns College", "priority": 470, "external_id": null}, {"id": 10846293003, "name": "Flagler College", "priority": 471, "external_id": null}, {"id": 10846294003, "name": "Florida Christian College", "priority": 472, "external_id": null}, {"id": 10846295003, "name": "Florida College", "priority": 473, "external_id": null}, {"id": 10846296003, "name": "Florida Gulf Coast University", "priority": 474, "external_id": null}, {"id": 10846297003, "name": "Florida Institute of Technology", "priority": 475, "external_id": null}, {"id": 10846298003, "name": "Florida Memorial University", "priority": 476, "external_id": null}, {"id": 10846299003, "name": "Florida Southern College", "priority": 477, "external_id": null}, {"id": 10846300003, "name": "Florida State College - Jacksonville", "priority": 478, "external_id": null}, {"id": 10846301003, "name": "Fontbonne University", "priority": 479, "external_id": null}, {"id": 10846302003, "name": "Fort Hays State University", "priority": 480, "external_id": null}, {"id": 10846303003, "name": "Fort Lewis College", "priority": 481, "external_id": null}, {"id": 10846304003, "name": "Fort Valley State University", "priority": 482, "external_id": null}, {"id": 10846305003, "name": "Framingham State University", "priority": 483, "external_id": null}, {"id": 10846306003, "name": "Francis Marion University", "priority": 484, "external_id": null}, {"id": 10846307003, "name": "Franciscan University of Steubenville", "priority": 485, "external_id": null}, {"id": 10846308003, "name": "Frank Lloyd Wright School of Architecture", "priority": 486, "external_id": null}, {"id": 10846309003, "name": "Franklin and Marshall College", "priority": 487, "external_id": null}, {"id": 10846310003, "name": "Franklin College", "priority": 488, "external_id": null}, {"id": 10846311003, "name": "Franklin College Switzerland", "priority": 489, "external_id": null}, {"id": 10846312003, "name": "Franklin Pierce University", "priority": 490, "external_id": null}, {"id": 10846313003, "name": "Franklin University", "priority": 491, "external_id": null}, {"id": 10846314003, "name": "Franklin W. Olin College of Engineering", "priority": 492, "external_id": null}, {"id": 10846315003, "name": "Freed-Hardeman University", "priority": 493, "external_id": null}, {"id": 10846316003, "name": "Fresno Pacific University", "priority": 494, "external_id": null}, {"id": 10846317003, "name": "Friends University", "priority": 495, "external_id": null}, {"id": 10846318003, "name": "Frostburg State University", "priority": 496, "external_id": null}, {"id": 10846319003, "name": "Gallaudet University", "priority": 497, "external_id": null}, {"id": 10846320003, "name": "Gannon University", "priority": 498, "external_id": null}, {"id": 10846321003, "name": "Geneva College", "priority": 499, "external_id": null}, {"id": 10846322003, "name": "George Fox University", "priority": 500, "external_id": null}, {"id": 10846323003, "name": "George Mason University", "priority": 501, "external_id": null}, {"id": 10846324003, "name": "George Washington University", "priority": 502, "external_id": null}, {"id": 10846325003, "name": "Georgetown College", "priority": 503, "external_id": null}, {"id": 10846326003, "name": "Georgia College & State University", "priority": 504, "external_id": null}, {"id": 10846327003, "name": "Georgia Gwinnett College", "priority": 505, "external_id": null}, {"id": 10846328003, "name": "Georgia Regents University", "priority": 506, "external_id": null}, {"id": 10846329003, "name": "Georgia Southwestern State University", "priority": 507, "external_id": null}, {"id": 10846330003, "name": "Georgian Court University", "priority": 508, "external_id": null}, {"id": 10846331003, "name": "Gettysburg College", "priority": 509, "external_id": null}, {"id": 10846332003, "name": "Glenville State College", "priority": 510, "external_id": null}, {"id": 10846333003, "name": "God's Bible School and College", "priority": 511, "external_id": null}, {"id": 10846334003, "name": "Goddard College", "priority": 512, "external_id": null}, {"id": 10846335003, "name": "Golden Gate University", "priority": 513, "external_id": null}, {"id": 10846336003, "name": "Goldey-Beacom College", "priority": 514, "external_id": null}, {"id": 10846337003, "name": "Goldfarb School of Nursing at Barnes-Jewish College", "priority": 515, "external_id": null}, {"id": 10846338003, "name": "Gonzaga University", "priority": 516, "external_id": null}, {"id": 10846339003, "name": "Gordon College", "priority": 517, "external_id": null}, {"id": 10846340003, "name": "Fordham University", "priority": 518, "external_id": null}, {"id": 10846341003, "name": "Georgia Institute of Technology", "priority": 519, "external_id": null}, {"id": 10846342003, "name": "Gardner-Webb University", "priority": 520, "external_id": null}, {"id": 10846343003, "name": "Georgia Southern University", "priority": 521, "external_id": null}, {"id": 10846344003, "name": "Georgia State University", "priority": 522, "external_id": null}, {"id": 10846345003, "name": "Florida State University", "priority": 523, "external_id": null}, {"id": 10846346003, "name": "Dartmouth College", "priority": 524, "external_id": null}, {"id": 10846347003, "name": "Florida International University", "priority": 525, "external_id": null}, {"id": 10846348003, "name": "Georgetown University", "priority": 526, "external_id": null}, {"id": 10846349003, "name": "Furman University", "priority": 527, "external_id": null}, {"id": 10846350003, "name": "Gordon State College", "priority": 528, "external_id": null}, {"id": 10846351003, "name": "Goshen College", "priority": 529, "external_id": null}, {"id": 10846352003, "name": "Goucher College", "priority": 530, "external_id": null}, {"id": 10846353003, "name": "Governors State University", "priority": 531, "external_id": null}, {"id": 10846354003, "name": "Grace Bible College", "priority": 532, "external_id": null}, {"id": 10846355003, "name": "Grace College and Seminary", "priority": 533, "external_id": null}, {"id": 10846356003, "name": "Grace University", "priority": 534, "external_id": null}, {"id": 10846357003, "name": "Graceland University", "priority": 535, "external_id": null}, {"id": 10846358003, "name": "Grand Canyon University", "priority": 536, "external_id": null}, {"id": 10846359003, "name": "Grand Valley State University", "priority": 537, "external_id": null}, {"id": 10846360003, "name": "Grand View University", "priority": 538, "external_id": null}, {"id": 10846361003, "name": "Granite State College", "priority": 539, "external_id": null}, {"id": 10846362003, "name": "Gratz College", "priority": 540, "external_id": null}, {"id": 10846363003, "name": "Great Basin College", "priority": 541, "external_id": null}, {"id": 10846364003, "name": "Great Lakes Christian College", "priority": 542, "external_id": null}, {"id": 10846365003, "name": "Green Mountain College", "priority": 543, "external_id": null}, {"id": 10846366003, "name": "Greensboro College", "priority": 544, "external_id": null}, {"id": 10846367003, "name": "Greenville College", "priority": 545, "external_id": null}, {"id": 10846368003, "name": "Grinnell College", "priority": 546, "external_id": null}, {"id": 10846369003, "name": "Grove City College", "priority": 547, "external_id": null}, {"id": 10846370003, "name": "Guilford College", "priority": 548, "external_id": null}, {"id": 10846371003, "name": "Gustavus Adolphus College", "priority": 549, "external_id": null}, {"id": 10846372003, "name": "Gwynedd-Mercy College", "priority": 550, "external_id": null}, {"id": 10846373003, "name": "Hamilton College", "priority": 551, "external_id": null}, {"id": 10846374003, "name": "Hamline University", "priority": 552, "external_id": null}, {"id": 10846375003, "name": "Hampden-Sydney College", "priority": 553, "external_id": null}, {"id": 10846376003, "name": "Hampshire College", "priority": 554, "external_id": null}, {"id": 10846377003, "name": "Hannibal-LaGrange University", "priority": 555, "external_id": null}, {"id": 10846378003, "name": "Hanover College", "priority": 556, "external_id": null}, {"id": 10846379003, "name": "Hardin-Simmons University", "priority": 557, "external_id": null}, {"id": 10846380003, "name": "Harding University", "priority": 558, "external_id": null}, {"id": 10846381003, "name": "Harrington College of Design", "priority": 559, "external_id": null}, {"id": 10846382003, "name": "Harris-Stowe State University", "priority": 560, "external_id": null}, {"id": 10846383003, "name": "Harrisburg University of Science and Technology", "priority": 561, "external_id": null}, {"id": 10846384003, "name": "Hartwick College", "priority": 562, "external_id": null}, {"id": 10846385003, "name": "Harvey Mudd College", "priority": 563, "external_id": null}, {"id": 10846386003, "name": "Haskell Indian Nations University", "priority": 564, "external_id": null}, {"id": 10846387003, "name": "Hastings College", "priority": 565, "external_id": null}, {"id": 10846388003, "name": "Haverford College", "priority": 566, "external_id": null}, {"id": 10846389003, "name": "Hawaii Pacific University", "priority": 567, "external_id": null}, {"id": 10846390003, "name": "Hebrew Theological College", "priority": 568, "external_id": null}, {"id": 10846391003, "name": "Heidelberg University", "priority": 569, "external_id": null}, {"id": 10846392003, "name": "Hellenic College", "priority": 570, "external_id": null}, {"id": 10846393003, "name": "Henderson State University", "priority": 571, "external_id": null}, {"id": 10846394003, "name": "Hendrix College", "priority": 572, "external_id": null}, {"id": 10846395003, "name": "Heritage University", "priority": 573, "external_id": null}, {"id": 10846396003, "name": "Herzing University", "priority": 574, "external_id": null}, {"id": 10846397003, "name": "Hesser College", "priority": 575, "external_id": null}, {"id": 10846398003, "name": "High Point University", "priority": 576, "external_id": null}, {"id": 10846399003, "name": "Hilbert College", "priority": 577, "external_id": null}, {"id": 10846400003, "name": "Hillsdale College", "priority": 578, "external_id": null}, {"id": 10846401003, "name": "Hiram College", "priority": 579, "external_id": null}, {"id": 10846402003, "name": "Hobart and William Smith Colleges", "priority": 580, "external_id": null}, {"id": 10846403003, "name": "Hodges University", "priority": 581, "external_id": null}, {"id": 10846404003, "name": "Hofstra University", "priority": 582, "external_id": null}, {"id": 10846405003, "name": "Hollins University", "priority": 583, "external_id": null}, {"id": 10846406003, "name": "Holy Apostles College and Seminary", "priority": 584, "external_id": null}, {"id": 10846407003, "name": "Indiana State University", "priority": 585, "external_id": null}, {"id": 10846408003, "name": "Holy Family University", "priority": 586, "external_id": null}, {"id": 10846409003, "name": "Holy Names University", "priority": 587, "external_id": null}, {"id": 10846410003, "name": "Hood College", "priority": 588, "external_id": null}, {"id": 10846411003, "name": "Hope College", "priority": 589, "external_id": null}, {"id": 10846412003, "name": "Hope International University", "priority": 590, "external_id": null}, {"id": 10846413003, "name": "Houghton College", "priority": 591, "external_id": null}, {"id": 10846414003, "name": "Howard Payne University", "priority": 592, "external_id": null}, {"id": 10846415003, "name": "Hult International Business School", "priority": 593, "external_id": null}, {"id": 10846416003, "name": "Humboldt State University", "priority": 594, "external_id": null}, {"id": 10846417003, "name": "Humphreys College", "priority": 595, "external_id": null}, {"id": 10846418003, "name": "Huntingdon College", "priority": 596, "external_id": null}, {"id": 10846419003, "name": "Huntington University", "priority": 597, "external_id": null}, {"id": 10846420003, "name": "Husson University", "priority": 598, "external_id": null}, {"id": 10846421003, "name": "Huston-Tillotson University", "priority": 599, "external_id": null}, {"id": 10846422003, "name": "Illinois College", "priority": 600, "external_id": null}, {"id": 10846423003, "name": "Illinois Institute of Art at Chicago", "priority": 601, "external_id": null}, {"id": 10846424003, "name": "Illinois Institute of Technology", "priority": 602, "external_id": null}, {"id": 10846425003, "name": "Illinois Wesleyan University", "priority": 603, "external_id": null}, {"id": 10846426003, "name": "Immaculata University", "priority": 604, "external_id": null}, {"id": 10846427003, "name": "Indian River State College", "priority": 605, "external_id": null}, {"id": 10846428003, "name": "Indiana Institute of Technology", "priority": 606, "external_id": null}, {"id": 10846429003, "name": "Indiana University East", "priority": 607, "external_id": null}, {"id": 10846430003, "name": "Indiana University Northwest", "priority": 608, "external_id": null}, {"id": 10846431003, "name": "Indiana University of Pennsylvania", "priority": 609, "external_id": null}, {"id": 10846432003, "name": "Indiana University Southeast", "priority": 610, "external_id": null}, {"id": 10846433003, "name": "Illinois State University", "priority": 611, "external_id": null}, {"id": 10846434003, "name": "Indiana University - Bloomington", "priority": 612, "external_id": null}, {"id": 10846435003, "name": "Davidson College", "priority": 613, "external_id": null}, {"id": 10846436003, "name": "Idaho State University", "priority": 614, "external_id": null}, {"id": 10846437003, "name": "Harvard University", "priority": 615, "external_id": null}, {"id": 10846438003, "name": "Howard University", "priority": 616, "external_id": null}, {"id": 10846439003, "name": "Houston Baptist University", "priority": 617, "external_id": null}, {"id": 10846440003, "name": "Indiana University - Kokomo", "priority": 618, "external_id": null}, {"id": 10846441003, "name": "Indiana University - South Bend", "priority": 619, "external_id": null}, {"id": 10846442003, "name": "Indiana University-Purdue University - Fort Wayne", "priority": 620, "external_id": null}, {"id": 10846443003, "name": "Indiana University-Purdue University - Indianapolis", "priority": 621, "external_id": null}, {"id": 10846444003, "name": "Indiana Wesleyan University", "priority": 622, "external_id": null}, {"id": 10846445003, "name": "Institute of American Indian and Alaska Native Culture and Arts Development", "priority": 623, "external_id": null}, {"id": 10846446003, "name": "Inter American University of Puerto Rico - Aguadilla", "priority": 624, "external_id": null}, {"id": 10846447003, "name": "Inter American University of Puerto Rico - Arecibo", "priority": 625, "external_id": null}, {"id": 10846448003, "name": "Inter American University of Puerto Rico - Barranquitas", "priority": 626, "external_id": null}, {"id": 10846449003, "name": "Inter American University of Puerto Rico - Bayamon", "priority": 627, "external_id": null}, {"id": 10846450003, "name": "Inter American University of Puerto Rico - Fajardo", "priority": 628, "external_id": null}, {"id": 10846451003, "name": "Inter American University of Puerto Rico - Guayama", "priority": 629, "external_id": null}, {"id": 10846452003, "name": "Inter American University of Puerto Rico - Metropolitan Campus", "priority": 630, "external_id": null}, {"id": 10846453003, "name": "Inter American University of Puerto Rico - Ponce", "priority": 631, "external_id": null}, {"id": 10846454003, "name": "Inter American University of Puerto Rico - San German", "priority": 632, "external_id": null}, {"id": 10846455003, "name": "International College of the Cayman Islands", "priority": 633, "external_id": null}, {"id": 10846456003, "name": "Iona College", "priority": 634, "external_id": null}, {"id": 10846457003, "name": "Iowa Wesleyan College", "priority": 635, "external_id": null}, {"id": 10846458003, "name": "Ithaca College", "priority": 636, "external_id": null}, {"id": 10846459003, "name": "Jarvis Christian College", "priority": 637, "external_id": null}, {"id": 10846460003, "name": "Jewish Theological Seminary of America", "priority": 638, "external_id": null}, {"id": 10846461003, "name": "John Brown University", "priority": 639, "external_id": null}, {"id": 10846462003, "name": "John Carroll University", "priority": 640, "external_id": null}, {"id": 10846463003, "name": "John F. Kennedy University", "priority": 641, "external_id": null}, {"id": 10846464003, "name": "Johns Hopkins University", "priority": 642, "external_id": null}, {"id": 10846465003, "name": "Johnson & Wales University", "priority": 643, "external_id": null}, {"id": 10846466003, "name": "Johnson C. Smith University", "priority": 644, "external_id": null}, {"id": 10846467003, "name": "Johnson State College", "priority": 645, "external_id": null}, {"id": 10846468003, "name": "Johnson University", "priority": 646, "external_id": null}, {"id": 10846469003, "name": "Jones International University", "priority": 647, "external_id": null}, {"id": 10846470003, "name": "Judson College", "priority": 648, "external_id": null}, {"id": 10846471003, "name": "Judson University", "priority": 649, "external_id": null}, {"id": 10846472003, "name": "Juilliard School", "priority": 650, "external_id": null}, {"id": 10846473003, "name": "Juniata College", "priority": 651, "external_id": null}, {"id": 10846474003, "name": "Kalamazoo College", "priority": 652, "external_id": null}, {"id": 10846475003, "name": "Kansas City Art Institute", "priority": 653, "external_id": null}, {"id": 10846476003, "name": "Kansas Wesleyan University", "priority": 654, "external_id": null}, {"id": 10846477003, "name": "Kaplan University", "priority": 655, "external_id": null}, {"id": 10846478003, "name": "Kean University", "priority": 656, "external_id": null}, {"id": 10846479003, "name": "Keene State College", "priority": 657, "external_id": null}, {"id": 10846480003, "name": "Keiser University", "priority": 658, "external_id": null}, {"id": 10846481003, "name": "Kendall College", "priority": 659, "external_id": null}, {"id": 10846482003, "name": "Kennesaw State University", "priority": 660, "external_id": null}, {"id": 10846483003, "name": "Kentucky Christian University", "priority": 661, "external_id": null}, {"id": 10846484003, "name": "Kentucky State University", "priority": 662, "external_id": null}, {"id": 10846485003, "name": "Kentucky Wesleyan College", "priority": 663, "external_id": null}, {"id": 10846486003, "name": "Kenyon College", "priority": 664, "external_id": null}, {"id": 10846487003, "name": "Kettering College", "priority": 665, "external_id": null}, {"id": 10846488003, "name": "Kettering University", "priority": 666, "external_id": null}, {"id": 10846489003, "name": "Keuka College", "priority": 667, "external_id": null}, {"id": 10846490003, "name": "Keystone College", "priority": 668, "external_id": null}, {"id": 10846491003, "name": "King University", "priority": 669, "external_id": null}, {"id": 10846492003, "name": "King's College", "priority": 670, "external_id": null}, {"id": 10846493003, "name": "Knox College", "priority": 671, "external_id": null}, {"id": 10846494003, "name": "Kutztown University of Pennsylvania", "priority": 672, "external_id": null}, {"id": 10846495003, "name": "Kuyper College", "priority": 673, "external_id": null}, {"id": 10846496003, "name": "La Roche College", "priority": 674, "external_id": null}, {"id": 10846497003, "name": "La Salle University", "priority": 675, "external_id": null}, {"id": 10846498003, "name": "La Sierra University", "priority": 676, "external_id": null}, {"id": 10846499003, "name": "LaGrange College", "priority": 677, "external_id": null}, {"id": 10846500003, "name": "Laguna College of Art and Design", "priority": 678, "external_id": null}, {"id": 10846501003, "name": "Lake Erie College", "priority": 679, "external_id": null}, {"id": 10846502003, "name": "Lake Forest College", "priority": 680, "external_id": null}, {"id": 10846503003, "name": "Lake Superior State University", "priority": 681, "external_id": null}, {"id": 10846504003, "name": "Lakeland College", "priority": 682, "external_id": null}, {"id": 10846505003, "name": "Lakeview College of Nursing", "priority": 683, "external_id": null}, {"id": 10846506003, "name": "Lancaster Bible College", "priority": 684, "external_id": null}, {"id": 10846507003, "name": "Lander University", "priority": 685, "external_id": null}, {"id": 10846508003, "name": "Lane College", "priority": 686, "external_id": null}, {"id": 10846509003, "name": "Langston University", "priority": 687, "external_id": null}, {"id": 10846510003, "name": "Lasell College", "priority": 688, "external_id": null}, {"id": 10846511003, "name": "Lawrence Technological University", "priority": 689, "external_id": null}, {"id": 10846512003, "name": "Lawrence University", "priority": 690, "external_id": null}, {"id": 10846513003, "name": "Le Moyne College", "priority": 691, "external_id": null}, {"id": 10846514003, "name": "Lebanon Valley College", "priority": 692, "external_id": null}, {"id": 10846515003, "name": "Lee University", "priority": 693, "external_id": null}, {"id": 10846516003, "name": "Lees-McRae College", "priority": 694, "external_id": null}, {"id": 10846517003, "name": "Kansas State University", "priority": 695, "external_id": null}, {"id": 10846518003, "name": "James Madison University", "priority": 696, "external_id": null}, {"id": 10846519003, "name": "Lafayette College", "priority": 697, "external_id": null}, {"id": 10846520003, "name": "Jacksonville University", "priority": 698, "external_id": null}, {"id": 10846521003, "name": "Kent State University", "priority": 699, "external_id": null}, {"id": 10846522003, "name": "Lamar University", "priority": 700, "external_id": null}, {"id": 10846523003, "name": "Jackson State University", "priority": 701, "external_id": null}, {"id": 10846524003, "name": "Lehigh University", "priority": 702, "external_id": null}, {"id": 10846525003, "name": "Jacksonville State University", "priority": 703, "external_id": null}, {"id": 10846526003, "name": "LeMoyne-Owen College", "priority": 704, "external_id": null}, {"id": 10846527003, "name": "Lenoir-Rhyne University", "priority": 705, "external_id": null}, {"id": 10846528003, "name": "Lesley University", "priority": 706, "external_id": null}, {"id": 10846529003, "name": "LeTourneau University", "priority": 707, "external_id": null}, {"id": 10846530003, "name": "Lewis & Clark College", "priority": 708, "external_id": null}, {"id": 10846531003, "name": "Lewis University", "priority": 709, "external_id": null}, {"id": 10846532003, "name": "Lewis-Clark State College", "priority": 710, "external_id": null}, {"id": 10846533003, "name": "Lexington College", "priority": 711, "external_id": null}, {"id": 10846534003, "name": "Life Pacific College", "priority": 712, "external_id": null}, {"id": 10846535003, "name": "Life University", "priority": 713, "external_id": null}, {"id": 10846536003, "name": "LIM College", "priority": 714, "external_id": null}, {"id": 10846537003, "name": "Limestone College", "priority": 715, "external_id": null}, {"id": 10846538003, "name": "Lincoln Christian University", "priority": 716, "external_id": null}, {"id": 10846539003, "name": "Lincoln College", "priority": 717, "external_id": null}, {"id": 10846540003, "name": "Lincoln Memorial University", "priority": 718, "external_id": null}, {"id": 10846541003, "name": "Lincoln University", "priority": 719, "external_id": null}, {"id": 10846542003, "name": "Lindenwood University", "priority": 720, "external_id": null}, {"id": 10846543003, "name": "Lindsey Wilson College", "priority": 721, "external_id": null}, {"id": 10846544003, "name": "Linfield College", "priority": 722, "external_id": null}, {"id": 10846545003, "name": "Lipscomb University", "priority": 723, "external_id": null}, {"id": 10846546003, "name": "LIU Post", "priority": 724, "external_id": null}, {"id": 10846547003, "name": "Livingstone College", "priority": 725, "external_id": null}, {"id": 10846548003, "name": "Lock Haven University of Pennsylvania", "priority": 726, "external_id": null}, {"id": 10846549003, "name": "Loma Linda University", "priority": 727, "external_id": null}, {"id": 10846550003, "name": "Longwood University", "priority": 728, "external_id": null}, {"id": 10846551003, "name": "Loras College", "priority": 729, "external_id": null}, {"id": 10846552003, "name": "Louisiana College", "priority": 730, "external_id": null}, {"id": 10846553003, "name": "Louisiana State University Health Sciences Center", "priority": 731, "external_id": null}, {"id": 10846554003, "name": "Louisiana State University - Alexandria", "priority": 732, "external_id": null}, {"id": 10846555003, "name": "Louisiana State University - Shreveport", "priority": 733, "external_id": null}, {"id": 10846556003, "name": "Lourdes University", "priority": 734, "external_id": null}, {"id": 10846557003, "name": "Loyola Marymount University", "priority": 735, "external_id": null}, {"id": 10846558003, "name": "Loyola University Chicago", "priority": 736, "external_id": null}, {"id": 10846559003, "name": "Loyola University Maryland", "priority": 737, "external_id": null}, {"id": 10846560003, "name": "Loyola University New Orleans", "priority": 738, "external_id": null}, {"id": 10846561003, "name": "Lubbock Christian University", "priority": 739, "external_id": null}, {"id": 10846562003, "name": "Luther College", "priority": 740, "external_id": null}, {"id": 10846563003, "name": "Lycoming College", "priority": 741, "external_id": null}, {"id": 10846564003, "name": "Lyme Academy College of Fine Arts", "priority": 742, "external_id": null}, {"id": 10846565003, "name": "Lynchburg College", "priority": 743, "external_id": null}, {"id": 10846566003, "name": "Lyndon State College", "priority": 744, "external_id": null}, {"id": 10846567003, "name": "Lynn University", "priority": 745, "external_id": null}, {"id": 10846568003, "name": "Lyon College", "priority": 746, "external_id": null}, {"id": 10846569003, "name": "Macalester College", "priority": 747, "external_id": null}, {"id": 10846570003, "name": "MacMurray College", "priority": 748, "external_id": null}, {"id": 10846571003, "name": "Madonna University", "priority": 749, "external_id": null}, {"id": 10846572003, "name": "Maharishi University of Management", "priority": 750, "external_id": null}, {"id": 10846573003, "name": "Maine College of Art", "priority": 751, "external_id": null}, {"id": 10846574003, "name": "Maine Maritime Academy", "priority": 752, "external_id": null}, {"id": 10846575003, "name": "Malone University", "priority": 753, "external_id": null}, {"id": 10846576003, "name": "Manchester University", "priority": 754, "external_id": null}, {"id": 10846577003, "name": "Manhattan Christian College", "priority": 755, "external_id": null}, {"id": 10846578003, "name": "Manhattan College", "priority": 756, "external_id": null}, {"id": 10846579003, "name": "Manhattan School of Music", "priority": 757, "external_id": null}, {"id": 10846580003, "name": "Manhattanville College", "priority": 758, "external_id": null}, {"id": 10846581003, "name": "Mansfield University of Pennsylvania", "priority": 759, "external_id": null}, {"id": 10846582003, "name": "Maranatha Baptist Bible College", "priority": 760, "external_id": null}, {"id": 10846583003, "name": "Marian University", "priority": 761, "external_id": null}, {"id": 10846584003, "name": "Marietta College", "priority": 762, "external_id": null}, {"id": 10846585003, "name": "Marlboro College", "priority": 763, "external_id": null}, {"id": 10846586003, "name": "Marquette University", "priority": 764, "external_id": null}, {"id": 10846587003, "name": "Mars Hill University", "priority": 765, "external_id": null}, {"id": 10846588003, "name": "Martin Luther College", "priority": 766, "external_id": null}, {"id": 10846589003, "name": "Martin Methodist College", "priority": 767, "external_id": null}, {"id": 10846590003, "name": "Martin University", "priority": 768, "external_id": null}, {"id": 10846591003, "name": "Mary Baldwin College", "priority": 769, "external_id": null}, {"id": 10846592003, "name": "Marygrove College", "priority": 770, "external_id": null}, {"id": 10846593003, "name": "Maryland Institute College of Art", "priority": 771, "external_id": null}, {"id": 10846594003, "name": "Marylhurst University", "priority": 772, "external_id": null}, {"id": 10846595003, "name": "Marymount Manhattan College", "priority": 773, "external_id": null}, {"id": 10846596003, "name": "Marymount University", "priority": 774, "external_id": null}, {"id": 10846597003, "name": "Maryville College", "priority": 775, "external_id": null}, {"id": 10846598003, "name": "Maryville University of St. Louis", "priority": 776, "external_id": null}, {"id": 10846599003, "name": "Marywood University", "priority": 777, "external_id": null}, {"id": 10846600003, "name": "Massachusetts College of Art and Design", "priority": 778, "external_id": null}, {"id": 10846601003, "name": "Massachusetts College of Liberal Arts", "priority": 779, "external_id": null}, {"id": 10846602003, "name": "Massachusetts College of Pharmacy and Health Sciences", "priority": 780, "external_id": null}, {"id": 10846603003, "name": "Massachusetts Institute of Technology", "priority": 781, "external_id": null}, {"id": 10846604003, "name": "Massachusetts Maritime Academy", "priority": 782, "external_id": null}, {"id": 10846605003, "name": "Master's College and Seminary", "priority": 783, "external_id": null}, {"id": 10846606003, "name": "Mayville State University", "priority": 784, "external_id": null}, {"id": 10846607003, "name": "McDaniel College", "priority": 785, "external_id": null}, {"id": 10846608003, "name": "McGill University", "priority": 786, "external_id": null}, {"id": 10846609003, "name": "McKendree University", "priority": 787, "external_id": null}, {"id": 10846610003, "name": "McMurry University", "priority": 788, "external_id": null}, {"id": 10846611003, "name": "McPherson College", "priority": 789, "external_id": null}, {"id": 10846612003, "name": "Medaille College", "priority": 790, "external_id": null}, {"id": 10846613003, "name": "Marist College", "priority": 791, "external_id": null}, {"id": 10846614003, "name": "McNeese State University", "priority": 792, "external_id": null}, {"id": 10846615003, "name": "Louisiana Tech University", "priority": 793, "external_id": null}, {"id": 10846616003, "name": "Marshall University", "priority": 794, "external_id": null}, {"id": 10846617003, "name": "Medical University of South Carolina", "priority": 795, "external_id": null}, {"id": 10846618003, "name": "Memorial University of Newfoundland", "priority": 796, "external_id": null}, {"id": 10846619003, "name": "Memphis College of Art", "priority": 797, "external_id": null}, {"id": 10846620003, "name": "Menlo College", "priority": 798, "external_id": null}, {"id": 10846621003, "name": "Mercy College", "priority": 799, "external_id": null}, {"id": 10846622003, "name": "Mercy College of Health Sciences", "priority": 800, "external_id": null}, {"id": 10846623003, "name": "Mercy College of Ohio", "priority": 801, "external_id": null}, {"id": 10846624003, "name": "Mercyhurst University", "priority": 802, "external_id": null}, {"id": 10846625003, "name": "Meredith College", "priority": 803, "external_id": null}, {"id": 10846626003, "name": "Merrimack College", "priority": 804, "external_id": null}, {"id": 10846627003, "name": "Messiah College", "priority": 805, "external_id": null}, {"id": 10846628003, "name": "Methodist University", "priority": 806, "external_id": null}, {"id": 10846629003, "name": "Metropolitan College of New York", "priority": 807, "external_id": null}, {"id": 10846630003, "name": "Metropolitan State University", "priority": 808, "external_id": null}, {"id": 10846631003, "name": "Metropolitan State University of Denver", "priority": 809, "external_id": null}, {"id": 10846632003, "name": "Miami Dade College", "priority": 810, "external_id": null}, {"id": 10846633003, "name": "Miami International University of Art & Design", "priority": 811, "external_id": null}, {"id": 10846634003, "name": "Michigan Technological University", "priority": 812, "external_id": null}, {"id": 10846635003, "name": "Mid-America Christian University", "priority": 813, "external_id": null}, {"id": 10846636003, "name": "Mid-Atlantic Christian University", "priority": 814, "external_id": null}, {"id": 10846637003, "name": "Mid-Continent University", "priority": 815, "external_id": null}, {"id": 10846638003, "name": "MidAmerica Nazarene University", "priority": 816, "external_id": null}, {"id": 10846639003, "name": "Middle Georgia State College", "priority": 817, "external_id": null}, {"id": 10846640003, "name": "Middlebury College", "priority": 818, "external_id": null}, {"id": 10846641003, "name": "Midland College", "priority": 819, "external_id": null}, {"id": 10846642003, "name": "Midland University", "priority": 820, "external_id": null}, {"id": 10846643003, "name": "Midstate College", "priority": 821, "external_id": null}, {"id": 10846644003, "name": "Midway College", "priority": 822, "external_id": null}, {"id": 10846645003, "name": "Midwestern State University", "priority": 823, "external_id": null}, {"id": 10846646003, "name": "Miles College", "priority": 824, "external_id": null}, {"id": 10846647003, "name": "Millersville University of Pennsylvania", "priority": 825, "external_id": null}, {"id": 10846648003, "name": "Milligan College", "priority": 826, "external_id": null}, {"id": 10846649003, "name": "Millikin University", "priority": 827, "external_id": null}, {"id": 10846650003, "name": "Mills College", "priority": 828, "external_id": null}, {"id": 10846651003, "name": "Millsaps College", "priority": 829, "external_id": null}, {"id": 10846652003, "name": "Milwaukee Institute of Art and Design", "priority": 830, "external_id": null}, {"id": 10846653003, "name": "Milwaukee School of Engineering", "priority": 831, "external_id": null}, {"id": 10846654003, "name": "Minneapolis College of Art and Design", "priority": 832, "external_id": null}, {"id": 10846655003, "name": "Minnesota State University - Mankato", "priority": 833, "external_id": null}, {"id": 10846656003, "name": "Minnesota State University - Moorhead", "priority": 834, "external_id": null}, {"id": 10846657003, "name": "Minot State University", "priority": 835, "external_id": null}, {"id": 10846658003, "name": "Misericordia University", "priority": 836, "external_id": null}, {"id": 10846659003, "name": "Mississippi College", "priority": 837, "external_id": null}, {"id": 10846660003, "name": "Mississippi University for Women", "priority": 838, "external_id": null}, {"id": 10846661003, "name": "Missouri Baptist University", "priority": 839, "external_id": null}, {"id": 10846662003, "name": "Missouri Southern State University", "priority": 840, "external_id": null}, {"id": 10846663003, "name": "Missouri University of Science & Technology", "priority": 841, "external_id": null}, {"id": 10846664003, "name": "Missouri Valley College", "priority": 842, "external_id": null}, {"id": 10846665003, "name": "Missouri Western State University", "priority": 843, "external_id": null}, {"id": 10846666003, "name": "Mitchell College", "priority": 844, "external_id": null}, {"id": 10846667003, "name": "Molloy College", "priority": 845, "external_id": null}, {"id": 10846668003, "name": "Monmouth College", "priority": 846, "external_id": null}, {"id": 10846669003, "name": "Monroe College", "priority": 847, "external_id": null}, {"id": 10846670003, "name": "Montana State University - Billings", "priority": 848, "external_id": null}, {"id": 10846671003, "name": "Montana State University - Northern", "priority": 849, "external_id": null}, {"id": 10846672003, "name": "Montana Tech of the University of Montana", "priority": 850, "external_id": null}, {"id": 10846673003, "name": "Montclair State University", "priority": 851, "external_id": null}, {"id": 10846674003, "name": "Monterrey Institute of Technology and Higher Education - Monterrey", "priority": 852, "external_id": null}, {"id": 10846675003, "name": "Montreat College", "priority": 853, "external_id": null}, {"id": 10846676003, "name": "Montserrat College of Art", "priority": 854, "external_id": null}, {"id": 10846677003, "name": "Moody Bible Institute", "priority": 855, "external_id": null}, {"id": 10846678003, "name": "Moore College of Art & Design", "priority": 856, "external_id": null}, {"id": 10846679003, "name": "Moravian College", "priority": 857, "external_id": null}, {"id": 10846680003, "name": "Morehouse College", "priority": 858, "external_id": null}, {"id": 10846681003, "name": "Morningside College", "priority": 859, "external_id": null}, {"id": 10846682003, "name": "Morris College", "priority": 860, "external_id": null}, {"id": 10846683003, "name": "Morrisville State College", "priority": 861, "external_id": null}, {"id": 10846684003, "name": "Mount Aloysius College", "priority": 862, "external_id": null}, {"id": 10846685003, "name": "Mount Angel Seminary", "priority": 863, "external_id": null}, {"id": 10846686003, "name": "Mount Carmel College of Nursing", "priority": 864, "external_id": null}, {"id": 10846687003, "name": "Mount Holyoke College", "priority": 865, "external_id": null}, {"id": 10846688003, "name": "Mount Ida College", "priority": 866, "external_id": null}, {"id": 10846689003, "name": "Mount Marty College", "priority": 867, "external_id": null}, {"id": 10846690003, "name": "Mount Mary University", "priority": 868, "external_id": null}, {"id": 10846691003, "name": "Mount Mercy University", "priority": 869, "external_id": null}, {"id": 10846692003, "name": "Mount Olive College", "priority": 870, "external_id": null}, {"id": 10846693003, "name": "Mississippi State University", "priority": 871, "external_id": null}, {"id": 10846694003, "name": "Montana State University", "priority": 872, "external_id": null}, {"id": 10846695003, "name": "Mississippi Valley State University", "priority": 873, "external_id": null}, {"id": 10846696003, "name": "Monmouth University", "priority": 874, "external_id": null}, {"id": 10846697003, "name": "Morehead State University", "priority": 875, "external_id": null}, {"id": 10846698003, "name": "Miami University - Oxford", "priority": 876, "external_id": null}, {"id": 10846699003, "name": "Morgan State University", "priority": 877, "external_id": null}, {"id": 10846700003, "name": "Missouri State University", "priority": 878, "external_id": null}, {"id": 10846701003, "name": "Michigan State University", "priority": 879, "external_id": null}, {"id": 10846702003, "name": "Mount St. Mary College", "priority": 880, "external_id": null}, {"id": 10846703003, "name": "Mount St. Mary's College", "priority": 881, "external_id": null}, {"id": 10846704003, "name": "Mount St. Mary's University", "priority": 882, "external_id": null}, {"id": 10846705003, "name": "Mount Vernon Nazarene University", "priority": 883, "external_id": null}, {"id": 10846706003, "name": "Muhlenberg College", "priority": 884, "external_id": null}, {"id": 10846707003, "name": "Multnomah University", "priority": 885, "external_id": null}, {"id": 10846708003, "name": "Muskingum University", "priority": 886, "external_id": null}, {"id": 10846709003, "name": "Naropa University", "priority": 887, "external_id": null}, {"id": 10846710003, "name": "National American University", "priority": 888, "external_id": null}, {"id": 10846711003, "name": "National Graduate School of Quality Management", "priority": 889, "external_id": null}, {"id": 10846712003, "name": "National Hispanic University", "priority": 890, "external_id": null}, {"id": 10846713003, "name": "National Labor College", "priority": 891, "external_id": null}, {"id": 10846714003, "name": "National University", "priority": 892, "external_id": null}, {"id": 10846715003, "name": "National-Louis University", "priority": 893, "external_id": null}, {"id": 10846716003, "name": "Nazarene Bible College", "priority": 894, "external_id": null}, {"id": 10846717003, "name": "Nazareth College", "priority": 895, "external_id": null}, {"id": 10846718003, "name": "Nebraska Methodist College", "priority": 896, "external_id": null}, {"id": 10846719003, "name": "Nebraska Wesleyan University", "priority": 897, "external_id": null}, {"id": 10846720003, "name": "Neumann University", "priority": 898, "external_id": null}, {"id": 10846721003, "name": "Nevada State College", "priority": 899, "external_id": null}, {"id": 10846722003, "name": "New College of Florida", "priority": 900, "external_id": null}, {"id": 10846723003, "name": "New England College", "priority": 901, "external_id": null}, {"id": 10846724003, "name": "New England Conservatory of Music", "priority": 902, "external_id": null}, {"id": 10846725003, "name": "New England Institute of Art", "priority": 903, "external_id": null}, {"id": 10846726003, "name": "New England Institute of Technology", "priority": 904, "external_id": null}, {"id": 10846727003, "name": "New Jersey City University", "priority": 905, "external_id": null}, {"id": 10846728003, "name": "New Jersey Institute of Technology", "priority": 906, "external_id": null}, {"id": 10846729003, "name": "New Mexico Highlands University", "priority": 907, "external_id": null}, {"id": 10846730003, "name": "New Mexico Institute of Mining and Technology", "priority": 908, "external_id": null}, {"id": 10846731003, "name": "New Orleans Baptist Theological Seminary", "priority": 909, "external_id": null}, {"id": 10846732003, "name": "New School", "priority": 910, "external_id": null}, {"id": 10846733003, "name": "New York Institute of Technology", "priority": 911, "external_id": null}, {"id": 10846734003, "name": "New York University", "priority": 912, "external_id": null}, {"id": 10846735003, "name": "Newberry College", "priority": 913, "external_id": null}, {"id": 10846736003, "name": "Newbury College", "priority": 914, "external_id": null}, {"id": 10846737003, "name": "Newman University", "priority": 915, "external_id": null}, {"id": 10846738003, "name": "Niagara University", "priority": 916, "external_id": null}, {"id": 10846739003, "name": "Nichols College", "priority": 917, "external_id": null}, {"id": 10846740003, "name": "North Carolina Wesleyan College", "priority": 918, "external_id": null}, {"id": 10846741003, "name": "North Central College", "priority": 919, "external_id": null}, {"id": 10846742003, "name": "North Central University", "priority": 920, "external_id": null}, {"id": 10846743003, "name": "North Greenville University", "priority": 921, "external_id": null}, {"id": 10846744003, "name": "North Park University", "priority": 922, "external_id": null}, {"id": 10846745003, "name": "Northcentral University", "priority": 923, "external_id": null}, {"id": 10846746003, "name": "Northeastern Illinois University", "priority": 924, "external_id": null}, {"id": 10846747003, "name": "Northeastern State University", "priority": 925, "external_id": null}, {"id": 10846748003, "name": "Northeastern University", "priority": 926, "external_id": null}, {"id": 10846749003, "name": "Northern Kentucky University", "priority": 927, "external_id": null}, {"id": 10846750003, "name": "Northern Michigan University", "priority": 928, "external_id": null}, {"id": 10846751003, "name": "Northern New Mexico College", "priority": 929, "external_id": null}, {"id": 10846752003, "name": "Northern State University", "priority": 930, "external_id": null}, {"id": 10846753003, "name": "Northland College", "priority": 931, "external_id": null}, {"id": 10846754003, "name": "Northwest Christian University", "priority": 932, "external_id": null}, {"id": 10846755003, "name": "Northwest Florida State College", "priority": 933, "external_id": null}, {"id": 10846756003, "name": "Northwest Missouri State University", "priority": 934, "external_id": null}, {"id": 10846757003, "name": "Northwest Nazarene University", "priority": 935, "external_id": null}, {"id": 10846758003, "name": "Northwest University", "priority": 936, "external_id": null}, {"id": 10846759003, "name": "Northwestern College", "priority": 937, "external_id": null}, {"id": 10846760003, "name": "Northwestern Health Sciences University", "priority": 938, "external_id": null}, {"id": 10846761003, "name": "Northwestern Oklahoma State University", "priority": 939, "external_id": null}, {"id": 10846762003, "name": "Northwood University", "priority": 940, "external_id": null}, {"id": 10846763003, "name": "Norwich University", "priority": 941, "external_id": null}, {"id": 10846764003, "name": "Notre Dame College of Ohio", "priority": 942, "external_id": null}, {"id": 10846765003, "name": "Notre Dame de Namur University", "priority": 943, "external_id": null}, {"id": 10846766003, "name": "Notre Dame of Maryland University", "priority": 944, "external_id": null}, {"id": 10846767003, "name": "Nova Scotia College of Art and Design", "priority": 945, "external_id": null}, {"id": 10846768003, "name": "Nova Southeastern University", "priority": 946, "external_id": null}, {"id": 10846769003, "name": "Nyack College", "priority": 947, "external_id": null}, {"id": 10846770003, "name": "Oakland City University", "priority": 948, "external_id": null}, {"id": 10846771003, "name": "Oakland University", "priority": 949, "external_id": null}, {"id": 10846772003, "name": "Oakwood University", "priority": 950, "external_id": null}, {"id": 10846773003, "name": "Oberlin College", "priority": 951, "external_id": null}, {"id": 10846774003, "name": "Occidental College", "priority": 952, "external_id": null}, {"id": 10846775003, "name": "Oglala Lakota College", "priority": 953, "external_id": null}, {"id": 10846776003, "name": "North Carolina A&T State University", "priority": 954, "external_id": null}, {"id": 10846777003, "name": "Northern Illinois University", "priority": 955, "external_id": null}, {"id": 10846778003, "name": "North Dakota State University", "priority": 956, "external_id": null}, {"id": 10846779003, "name": "Nicholls State University", "priority": 957, "external_id": null}, {"id": 10846780003, "name": "North Carolina Central University", "priority": 958, "external_id": null}, {"id": 10846781003, "name": "Norfolk State University", "priority": 959, "external_id": null}, {"id": 10846782003, "name": "Northwestern State University of Louisiana", "priority": 960, "external_id": null}, {"id": 10846783003, "name": "Northern Arizona University", "priority": 961, "external_id": null}, {"id": 10846784003, "name": "North Carolina State University - Raleigh", "priority": 962, "external_id": null}, {"id": 10846785003, "name": "Northwestern University", "priority": 963, "external_id": null}, {"id": 10846786003, "name": "Oglethorpe University", "priority": 964, "external_id": null}, {"id": 10846787003, "name": "Ohio Christian University", "priority": 965, "external_id": null}, {"id": 10846788003, "name": "Ohio Dominican University", "priority": 966, "external_id": null}, {"id": 10846789003, "name": "Ohio Northern University", "priority": 967, "external_id": null}, {"id": 10846790003, "name": "Ohio Valley University", "priority": 968, "external_id": null}, {"id": 10846791003, "name": "Ohio Wesleyan University", "priority": 969, "external_id": null}, {"id": 10846792003, "name": "Oklahoma Baptist University", "priority": 970, "external_id": null}, {"id": 10846793003, "name": "Oklahoma Christian University", "priority": 971, "external_id": null}, {"id": 10846794003, "name": "Oklahoma City University", "priority": 972, "external_id": null}, {"id": 10846795003, "name": "Oklahoma Panhandle State University", "priority": 973, "external_id": null}, {"id": 10846796003, "name": "Oklahoma State University Institute of Technology - Okmulgee", "priority": 974, "external_id": null}, {"id": 10846797003, "name": "Oklahoma State University - Oklahoma City", "priority": 975, "external_id": null}, {"id": 10846798003, "name": "Oklahoma Wesleyan University", "priority": 976, "external_id": null}, {"id": 10846799003, "name": "Olivet College", "priority": 977, "external_id": null}, {"id": 10846800003, "name": "Olivet Nazarene University", "priority": 978, "external_id": null}, {"id": 10846801003, "name": "Olympic College", "priority": 979, "external_id": null}, {"id": 10846802003, "name": "Oral Roberts University", "priority": 980, "external_id": null}, {"id": 10846803003, "name": "Oregon College of Art and Craft", "priority": 981, "external_id": null}, {"id": 10846804003, "name": "Oregon Health and Science University", "priority": 982, "external_id": null}, {"id": 10846805003, "name": "Oregon Institute of Technology", "priority": 983, "external_id": null}, {"id": 10846806003, "name": "Otis College of Art and Design", "priority": 984, "external_id": null}, {"id": 10846807003, "name": "Ottawa University", "priority": 985, "external_id": null}, {"id": 10846808003, "name": "Otterbein University", "priority": 986, "external_id": null}, {"id": 10846809003, "name": "Ouachita Baptist University", "priority": 987, "external_id": null}, {"id": 10846810003, "name": "Our Lady of Holy Cross College", "priority": 988, "external_id": null}, {"id": 10846811003, "name": "Our Lady of the Lake College", "priority": 989, "external_id": null}, {"id": 10846812003, "name": "Our Lady of the Lake University", "priority": 990, "external_id": null}, {"id": 10846813003, "name": "Pace University", "priority": 991, "external_id": null}, {"id": 10846814003, "name": "Pacific Lutheran University", "priority": 992, "external_id": null}, {"id": 10846815003, "name": "Pacific Northwest College of Art", "priority": 993, "external_id": null}, {"id": 10846816003, "name": "Pacific Oaks College", "priority": 994, "external_id": null}, {"id": 10846817003, "name": "Pacific Union College", "priority": 995, "external_id": null}, {"id": 10846818003, "name": "Pacific University", "priority": 996, "external_id": null}, {"id": 10846819003, "name": "Paine College", "priority": 997, "external_id": null}, {"id": 10846820003, "name": "Palm Beach Atlantic University", "priority": 998, "external_id": null}, {"id": 10846821003, "name": "Palmer College of Chiropractic", "priority": 999, "external_id": null}, {"id": 10846822003, "name": "Park University", "priority": 1000, "external_id": null}, {"id": 10846823003, "name": "Parker University", "priority": 1001, "external_id": null}, {"id": 10846824003, "name": "Patten University", "priority": 1002, "external_id": null}, {"id": 10846825003, "name": "Paul Smith's College", "priority": 1003, "external_id": null}, {"id": 10846826003, "name": "Peirce College", "priority": 1004, "external_id": null}, {"id": 10846827003, "name": "Peninsula College", "priority": 1005, "external_id": null}, {"id": 10846828003, "name": "Pennsylvania College of Art and Design", "priority": 1006, "external_id": null}, {"id": 10846829003, "name": "Pennsylvania College of Technology", "priority": 1007, "external_id": null}, {"id": 10846830003, "name": "Pennsylvania State University - Erie, The Behrend College", "priority": 1008, "external_id": null}, {"id": 10846831003, "name": "Pennsylvania State University - Harrisburg", "priority": 1009, "external_id": null}, {"id": 10846832003, "name": "Pepperdine University", "priority": 1010, "external_id": null}, {"id": 10846833003, "name": "Peru State College", "priority": 1011, "external_id": null}, {"id": 10846834003, "name": "Pfeiffer University", "priority": 1012, "external_id": null}, {"id": 10846835003, "name": "Philadelphia University", "priority": 1013, "external_id": null}, {"id": 10846836003, "name": "Philander Smith College", "priority": 1014, "external_id": null}, {"id": 10846837003, "name": "Piedmont College", "priority": 1015, "external_id": null}, {"id": 10846838003, "name": "Pine Manor College", "priority": 1016, "external_id": null}, {"id": 10846839003, "name": "Pittsburg State University", "priority": 1017, "external_id": null}, {"id": 10846840003, "name": "Pitzer College", "priority": 1018, "external_id": null}, {"id": 10846841003, "name": "Plaza College", "priority": 1019, "external_id": null}, {"id": 10846842003, "name": "Plymouth State University", "priority": 1020, "external_id": null}, {"id": 10846843003, "name": "Point Loma Nazarene University", "priority": 1021, "external_id": null}, {"id": 10846844003, "name": "Point Park University", "priority": 1022, "external_id": null}, {"id": 10846845003, "name": "Point University", "priority": 1023, "external_id": null}, {"id": 10846846003, "name": "Polytechnic Institute of New York University", "priority": 1024, "external_id": null}, {"id": 10846847003, "name": "Pomona College", "priority": 1025, "external_id": null}, {"id": 10846848003, "name": "Pontifical Catholic University of Puerto Rico", "priority": 1026, "external_id": null}, {"id": 10846849003, "name": "Pontifical College Josephinum", "priority": 1027, "external_id": null}, {"id": 10846850003, "name": "Post University", "priority": 1028, "external_id": null}, {"id": 10846851003, "name": "Potomac College", "priority": 1029, "external_id": null}, {"id": 10846852003, "name": "Pratt Institute", "priority": 1030, "external_id": null}, {"id": 10846853003, "name": "Prescott College", "priority": 1031, "external_id": null}, {"id": 10846854003, "name": "Presentation College", "priority": 1032, "external_id": null}, {"id": 10846855003, "name": "Principia College", "priority": 1033, "external_id": null}, {"id": 10846856003, "name": "Providence College", "priority": 1034, "external_id": null}, {"id": 10846857003, "name": "Puerto Rico Conservatory of Music", "priority": 1035, "external_id": null}, {"id": 10846858003, "name": "Purchase College - SUNY", "priority": 1036, "external_id": null}, {"id": 10846859003, "name": "Purdue University - Calumet", "priority": 1037, "external_id": null}, {"id": 10846860003, "name": "Purdue University - North Central", "priority": 1038, "external_id": null}, {"id": 10846861003, "name": "Queens University of Charlotte", "priority": 1039, "external_id": null}, {"id": 10846862003, "name": "Oklahoma State University", "priority": 1040, "external_id": null}, {"id": 10846863003, "name": "Oregon State University", "priority": 1041, "external_id": null}, {"id": 10846864003, "name": "Portland State University", "priority": 1042, "external_id": null}, {"id": 10846865003, "name": "Old Dominion University", "priority": 1043, "external_id": null}, {"id": 10846866003, "name": "Prairie View A&M University", "priority": 1044, "external_id": null}, {"id": 10846867003, "name": "Presbyterian College", "priority": 1045, "external_id": null}, {"id": 10846868003, "name": "Purdue University - West Lafayette", "priority": 1046, "external_id": null}, {"id": 10846869003, "name": "Ohio University", "priority": 1047, "external_id": null}, {"id": 10846870003, "name": "Princeton University", "priority": 1048, "external_id": null}, {"id": 10846871003, "name": "Quincy University", "priority": 1049, "external_id": null}, {"id": 10846872003, "name": "Quinnipiac University", "priority": 1050, "external_id": null}, {"id": 10846873003, "name": "Radford University", "priority": 1051, "external_id": null}, {"id": 10846874003, "name": "Ramapo College of New Jersey", "priority": 1052, "external_id": null}, {"id": 10846875003, "name": "Randolph College", "priority": 1053, "external_id": null}, {"id": 10846876003, "name": "Randolph-Macon College", "priority": 1054, "external_id": null}, {"id": 10846877003, "name": "Ranken Technical College", "priority": 1055, "external_id": null}, {"id": 10846878003, "name": "Reed College", "priority": 1056, "external_id": null}, {"id": 10846879003, "name": "Regent University", "priority": 1057, "external_id": null}, {"id": 10846880003, "name": "Regent's American College London", "priority": 1058, "external_id": null}, {"id": 10846881003, "name": "Regis College", "priority": 1059, "external_id": null}, {"id": 10846882003, "name": "Regis University", "priority": 1060, "external_id": null}, {"id": 10846883003, "name": "Reinhardt University", "priority": 1061, "external_id": null}, {"id": 10846884003, "name": "Rensselaer Polytechnic Institute", "priority": 1062, "external_id": null}, {"id": 10846885003, "name": "Research College of Nursing", "priority": 1063, "external_id": null}, {"id": 10846886003, "name": "Resurrection University", "priority": 1064, "external_id": null}, {"id": 10846887003, "name": "Rhode Island College", "priority": 1065, "external_id": null}, {"id": 10846888003, "name": "Rhode Island School of Design", "priority": 1066, "external_id": null}, {"id": 10846889003, "name": "Rhodes College", "priority": 1067, "external_id": null}, {"id": 10846890003, "name": "Richard Stockton College of New Jersey", "priority": 1068, "external_id": null}, {"id": 10846891003, "name": "Richmond - The American International University in London", "priority": 1069, "external_id": null}, {"id": 10846892003, "name": "Rider University", "priority": 1070, "external_id": null}, {"id": 10846893003, "name": "Ringling College of Art and Design", "priority": 1071, "external_id": null}, {"id": 10846894003, "name": "Ripon College", "priority": 1072, "external_id": null}, {"id": 10846895003, "name": "Rivier University", "priority": 1073, "external_id": null}, {"id": 10846896003, "name": "Roanoke College", "priority": 1074, "external_id": null}, {"id": 10846897003, "name": "Robert B. Miller College", "priority": 1075, "external_id": null}, {"id": 10846898003, "name": "Roberts Wesleyan College", "priority": 1076, "external_id": null}, {"id": 10846899003, "name": "Rochester College", "priority": 1077, "external_id": null}, {"id": 10846900003, "name": "Rochester Institute of Technology", "priority": 1078, "external_id": null}, {"id": 10846901003, "name": "Rockford University", "priority": 1079, "external_id": null}, {"id": 10846902003, "name": "Rockhurst University", "priority": 1080, "external_id": null}, {"id": 10846903003, "name": "Rocky Mountain College", "priority": 1081, "external_id": null}, {"id": 10846904003, "name": "Rocky Mountain College of Art and Design", "priority": 1082, "external_id": null}, {"id": 10846905003, "name": "Roger Williams University", "priority": 1083, "external_id": null}, {"id": 10846906003, "name": "Rogers State University", "priority": 1084, "external_id": null}, {"id": 10846907003, "name": "Rollins College", "priority": 1085, "external_id": null}, {"id": 10846908003, "name": "Roosevelt University", "priority": 1086, "external_id": null}, {"id": 10846909003, "name": "Rosalind Franklin University of Medicine and Science", "priority": 1087, "external_id": null}, {"id": 10846910003, "name": "Rose-Hulman Institute of Technology", "priority": 1088, "external_id": null}, {"id": 10846911003, "name": "Rosemont College", "priority": 1089, "external_id": null}, {"id": 10846912003, "name": "Rowan University", "priority": 1090, "external_id": null}, {"id": 10846913003, "name": "Rush University", "priority": 1091, "external_id": null}, {"id": 10846914003, "name": "Rust College", "priority": 1092, "external_id": null}, {"id": 10846915003, "name": "Rutgers, the State University of New Jersey - Camden", "priority": 1093, "external_id": null}, {"id": 10846916003, "name": "Rutgers, the State University of New Jersey - Newark", "priority": 1094, "external_id": null}, {"id": 10846917003, "name": "Ryerson University", "priority": 1095, "external_id": null}, {"id": 10846918003, "name": "Sacred Heart Major Seminary", "priority": 1096, "external_id": null}, {"id": 10846919003, "name": "Saginaw Valley State University", "priority": 1097, "external_id": null}, {"id": 10846920003, "name": "Salem College", "priority": 1098, "external_id": null}, {"id": 10846921003, "name": "Salem International University", "priority": 1099, "external_id": null}, {"id": 10846922003, "name": "Salem State University", "priority": 1100, "external_id": null}, {"id": 10846923003, "name": "Salisbury University", "priority": 1101, "external_id": null}, {"id": 10846924003, "name": "Salish Kootenai College", "priority": 1102, "external_id": null}, {"id": 10846925003, "name": "Salve Regina University", "priority": 1103, "external_id": null}, {"id": 10846926003, "name": "Samuel Merritt University", "priority": 1104, "external_id": null}, {"id": 10846927003, "name": "San Diego Christian College", "priority": 1105, "external_id": null}, {"id": 10846928003, "name": "San Francisco Art Institute", "priority": 1106, "external_id": null}, {"id": 10846929003, "name": "San Francisco Conservatory of Music", "priority": 1107, "external_id": null}, {"id": 10846930003, "name": "San Francisco State University", "priority": 1108, "external_id": null}, {"id": 10846931003, "name": "Sanford College of Nursing", "priority": 1109, "external_id": null}, {"id": 10846932003, "name": "Santa Clara University", "priority": 1110, "external_id": null}, {"id": 10846933003, "name": "Santa Fe University of Art and Design", "priority": 1111, "external_id": null}, {"id": 10846934003, "name": "Sarah Lawrence College", "priority": 1112, "external_id": null}, {"id": 10846935003, "name": "Savannah College of Art and Design", "priority": 1113, "external_id": null}, {"id": 10846936003, "name": "School of the Art Institute of Chicago", "priority": 1114, "external_id": null}, {"id": 10846937003, "name": "School of Visual Arts", "priority": 1115, "external_id": null}, {"id": 10846938003, "name": "Schreiner University", "priority": 1116, "external_id": null}, {"id": 10846939003, "name": "Scripps College", "priority": 1117, "external_id": null}, {"id": 10846940003, "name": "Seattle Pacific University", "priority": 1118, "external_id": null}, {"id": 10846941003, "name": "Seattle University", "priority": 1119, "external_id": null}, {"id": 10846942003, "name": "Seton Hall University", "priority": 1120, "external_id": null}, {"id": 10846943003, "name": "Seton Hill University", "priority": 1121, "external_id": null}, {"id": 10846944003, "name": "Sewanee - University of the South", "priority": 1122, "external_id": null}, {"id": 10846945003, "name": "Shaw University", "priority": 1123, "external_id": null}, {"id": 10846946003, "name": "Shawnee State University", "priority": 1124, "external_id": null}, {"id": 10846947003, "name": "Shenandoah University", "priority": 1125, "external_id": null}, {"id": 10846948003, "name": "Shepherd University", "priority": 1126, "external_id": null}, {"id": 10846949003, "name": "Shimer College", "priority": 1127, "external_id": null}, {"id": 10846950003, "name": "Sacred Heart University", "priority": 1128, "external_id": null}, {"id": 10846951003, "name": "Robert Morris University", "priority": 1129, "external_id": null}, {"id": 10846952003, "name": "Sam Houston State University", "priority": 1130, "external_id": null}, {"id": 10846953003, "name": "Samford University", "priority": 1131, "external_id": null}, {"id": 10846954003, "name": "Savannah State University", "priority": 1132, "external_id": null}, {"id": 10846955003, "name": "San Jose State University", "priority": 1133, "external_id": null}, {"id": 10846956003, "name": "Rutgers, the State University of New Jersey - New Brunswick", "priority": 1134, "external_id": null}, {"id": 10846957003, "name": "San Diego State University", "priority": 1135, "external_id": null}, {"id": 10846958003, "name": "Shippensburg University of Pennsylvania", "priority": 1136, "external_id": null}, {"id": 10846959003, "name": "Shorter University", "priority": 1137, "external_id": null}, {"id": 10846960003, "name": "Siena College", "priority": 1138, "external_id": null}, {"id": 10846961003, "name": "Siena Heights University", "priority": 1139, "external_id": null}, {"id": 10846962003, "name": "Sierra Nevada College", "priority": 1140, "external_id": null}, {"id": 10846963003, "name": "Silver Lake College", "priority": 1141, "external_id": null}, {"id": 10846964003, "name": "Simmons College", "priority": 1142, "external_id": null}, {"id": 10846965003, "name": "Simon Fraser University", "priority": 1143, "external_id": null}, {"id": 10846966003, "name": "Simpson College", "priority": 1144, "external_id": null}, {"id": 10846967003, "name": "Simpson University", "priority": 1145, "external_id": null}, {"id": 10846968003, "name": "Sinte Gleska University", "priority": 1146, "external_id": null}, {"id": 10846969003, "name": "Sitting Bull College", "priority": 1147, "external_id": null}, {"id": 10846970003, "name": "Skidmore College", "priority": 1148, "external_id": null}, {"id": 10846971003, "name": "Slippery Rock University of Pennsylvania", "priority": 1149, "external_id": null}, {"id": 10846972003, "name": "Smith College", "priority": 1150, "external_id": null}, {"id": 10846973003, "name": "Sojourner-Douglass College", "priority": 1151, "external_id": null}, {"id": 10846974003, "name": "Soka University of America", "priority": 1152, "external_id": null}, {"id": 10846975003, "name": "Sonoma State University", "priority": 1153, "external_id": null}, {"id": 10846976003, "name": "South College", "priority": 1154, "external_id": null}, {"id": 10846977003, "name": "South Dakota School of Mines and Technology", "priority": 1155, "external_id": null}, {"id": 10846978003, "name": "South Seattle Community College", "priority": 1156, "external_id": null}, {"id": 10846979003, "name": "South Texas College", "priority": 1157, "external_id": null}, {"id": 10846980003, "name": "South University", "priority": 1158, "external_id": null}, {"id": 10846981003, "name": "Southeastern Oklahoma State University", "priority": 1159, "external_id": null}, {"id": 10846982003, "name": "Southeastern University", "priority": 1160, "external_id": null}, {"id": 10846983003, "name": "Southern Adventist University", "priority": 1161, "external_id": null}, {"id": 10846984003, "name": "Southern Arkansas University", "priority": 1162, "external_id": null}, {"id": 10846985003, "name": "Southern Baptist Theological Seminary", "priority": 1163, "external_id": null}, {"id": 10846986003, "name": "Southern California Institute of Architecture", "priority": 1164, "external_id": null}, {"id": 10846987003, "name": "Southern Connecticut State University", "priority": 1165, "external_id": null}, {"id": 10846988003, "name": "Southern Illinois University - Edwardsville", "priority": 1166, "external_id": null}, {"id": 10846989003, "name": "Southern Nazarene University", "priority": 1167, "external_id": null}, {"id": 10846990003, "name": "Southern New Hampshire University", "priority": 1168, "external_id": null}, {"id": 10846991003, "name": "Southern Oregon University", "priority": 1169, "external_id": null}, {"id": 10846992003, "name": "Southern Polytechnic State University", "priority": 1170, "external_id": null}, {"id": 10846993003, "name": "Southern University - New Orleans", "priority": 1171, "external_id": null}, {"id": 10846994003, "name": "Southern Vermont College", "priority": 1172, "external_id": null}, {"id": 10846995003, "name": "Southern Wesleyan University", "priority": 1173, "external_id": null}, {"id": 10846996003, "name": "Southwest Baptist University", "priority": 1174, "external_id": null}, {"id": 10846997003, "name": "Southwest Minnesota State University", "priority": 1175, "external_id": null}, {"id": 10846998003, "name": "Southwest University of Visual Arts", "priority": 1176, "external_id": null}, {"id": 10846999003, "name": "Southwestern Adventist University", "priority": 1177, "external_id": null}, {"id": 10847000003, "name": "Southwestern Assemblies of God University", "priority": 1178, "external_id": null}, {"id": 10847001003, "name": "Southwestern Christian College", "priority": 1179, "external_id": null}, {"id": 10847002003, "name": "Southwestern Christian University", "priority": 1180, "external_id": null}, {"id": 10847003003, "name": "Southwestern College", "priority": 1181, "external_id": null}, {"id": 10847004003, "name": "Southwestern Oklahoma State University", "priority": 1182, "external_id": null}, {"id": 10847005003, "name": "Southwestern University", "priority": 1183, "external_id": null}, {"id": 10847006003, "name": "Spalding University", "priority": 1184, "external_id": null}, {"id": 10847007003, "name": "Spelman College", "priority": 1185, "external_id": null}, {"id": 10847008003, "name": "Spring Arbor University", "priority": 1186, "external_id": null}, {"id": 10847009003, "name": "Spring Hill College", "priority": 1187, "external_id": null}, {"id": 10847010003, "name": "Springfield College", "priority": 1188, "external_id": null}, {"id": 10847011003, "name": "St. Ambrose University", "priority": 1189, "external_id": null}, {"id": 10847012003, "name": "St. Anselm College", "priority": 1190, "external_id": null}, {"id": 10847013003, "name": "St. Anthony College of Nursing", "priority": 1191, "external_id": null}, {"id": 10847014003, "name": "St. Augustine College", "priority": 1192, "external_id": null}, {"id": 10847015003, "name": "St. Augustine's University", "priority": 1193, "external_id": null}, {"id": 10847016003, "name": "St. Bonaventure University", "priority": 1194, "external_id": null}, {"id": 10847017003, "name": "St. Catharine College", "priority": 1195, "external_id": null}, {"id": 10847018003, "name": "St. Catherine University", "priority": 1196, "external_id": null}, {"id": 10847019003, "name": "St. Charles Borromeo Seminary", "priority": 1197, "external_id": null}, {"id": 10847020003, "name": "St. Cloud State University", "priority": 1198, "external_id": null}, {"id": 10847021003, "name": "St. Edward's University", "priority": 1199, "external_id": null}, {"id": 10847022003, "name": "St. Francis College", "priority": 1200, "external_id": null}, {"id": 10847023003, "name": "St. Francis Medical Center College of Nursing", "priority": 1201, "external_id": null}, {"id": 10847024003, "name": "St. Gregory's University", "priority": 1202, "external_id": null}, {"id": 10847025003, "name": "St. John Fisher College", "priority": 1203, "external_id": null}, {"id": 10847026003, "name": "St. John Vianney College Seminary", "priority": 1204, "external_id": null}, {"id": 10847027003, "name": "St. John's College", "priority": 1205, "external_id": null}, {"id": 10847028003, "name": "St. John's University", "priority": 1206, "external_id": null}, {"id": 10847029003, "name": "St. Joseph Seminary College", "priority": 1207, "external_id": null}, {"id": 10847030003, "name": "St. Joseph's College", "priority": 1208, "external_id": null}, {"id": 10847031003, "name": "St. Joseph's College New York", "priority": 1209, "external_id": null}, {"id": 10847032003, "name": "St. Joseph's University", "priority": 1210, "external_id": null}, {"id": 10847033003, "name": "St. Lawrence University", "priority": 1211, "external_id": null}, {"id": 10847034003, "name": "St. Leo University", "priority": 1212, "external_id": null}, {"id": 10847035003, "name": "Southern University and A&M College", "priority": 1213, "external_id": null}, {"id": 10847036003, "name": "Southern Methodist University", "priority": 1214, "external_id": null}, {"id": 10847037003, "name": "Southeast Missouri State University", "priority": 1215, "external_id": null}, {"id": 10847038003, "name": "Southern Utah University", "priority": 1216, "external_id": null}, {"id": 10847039003, "name": "South Dakota State University", "priority": 1217, "external_id": null}, {"id": 10847040003, "name": "St. Francis University", "priority": 1218, "external_id": null}, {"id": 10847041003, "name": "Southeastern Louisiana University", "priority": 1219, "external_id": null}, {"id": 10847042003, "name": "Southern Illinois University - Carbondale", "priority": 1220, "external_id": null}, {"id": 10847043003, "name": "St. Louis College of Pharmacy", "priority": 1221, "external_id": null}, {"id": 10847044003, "name": "St. Louis University", "priority": 1222, "external_id": null}, {"id": 10847045003, "name": "St. Luke's College of Health Sciences", "priority": 1223, "external_id": null}, {"id": 10847046003, "name": "St. Martin's University", "priority": 1224, "external_id": null}, {"id": 10847047003, "name": "St. Mary's College", "priority": 1225, "external_id": null}, {"id": 10847048003, "name": "St. Mary's College of California", "priority": 1226, "external_id": null}, {"id": 10847049003, "name": "St. Mary's College of Maryland", "priority": 1227, "external_id": null}, {"id": 10847050003, "name": "St. Mary's Seminary and University", "priority": 1228, "external_id": null}, {"id": 10847051003, "name": "St. Mary's University of Minnesota", "priority": 1229, "external_id": null}, {"id": 10847052003, "name": "St. Mary's University of San Antonio", "priority": 1230, "external_id": null}, {"id": 10847053003, "name": "St. Mary-of-the-Woods College", "priority": 1231, "external_id": null}, {"id": 10847054003, "name": "St. Michael's College", "priority": 1232, "external_id": null}, {"id": 10847055003, "name": "St. Norbert College", "priority": 1233, "external_id": null}, {"id": 10847056003, "name": "St. Olaf College", "priority": 1234, "external_id": null}, {"id": 10847057003, "name": "St. Paul's College", "priority": 1235, "external_id": null}, {"id": 10847058003, "name": "St. Peter's University", "priority": 1236, "external_id": null}, {"id": 10847059003, "name": "St. Petersburg College", "priority": 1237, "external_id": null}, {"id": 10847060003, "name": "St. Thomas Aquinas College", "priority": 1238, "external_id": null}, {"id": 10847061003, "name": "St. Thomas University", "priority": 1239, "external_id": null}, {"id": 10847062003, "name": "St. Vincent College", "priority": 1240, "external_id": null}, {"id": 10847063003, "name": "St. Xavier University", "priority": 1241, "external_id": null}, {"id": 10847064003, "name": "Stephens College", "priority": 1242, "external_id": null}, {"id": 10847065003, "name": "Sterling College", "priority": 1243, "external_id": null}, {"id": 10847066003, "name": "Stevens Institute of Technology", "priority": 1244, "external_id": null}, {"id": 10847067003, "name": "Stevenson University", "priority": 1245, "external_id": null}, {"id": 10847068003, "name": "Stillman College", "priority": 1246, "external_id": null}, {"id": 10847069003, "name": "Stonehill College", "priority": 1247, "external_id": null}, {"id": 10847070003, "name": "Strayer University", "priority": 1248, "external_id": null}, {"id": 10847071003, "name": "Suffolk University", "priority": 1249, "external_id": null}, {"id": 10847072003, "name": "Sul Ross State University", "priority": 1250, "external_id": null}, {"id": 10847073003, "name": "Sullivan University", "priority": 1251, "external_id": null}, {"id": 10847074003, "name": "SUNY Buffalo State", "priority": 1252, "external_id": null}, {"id": 10847075003, "name": "SUNY College of Agriculture and Technology - Cobleskill", "priority": 1253, "external_id": null}, {"id": 10847076003, "name": "SUNY College of Environmental Science and Forestry", "priority": 1254, "external_id": null}, {"id": 10847077003, "name": "SUNY College of Technology - Alfred", "priority": 1255, "external_id": null}, {"id": 10847078003, "name": "SUNY College of Technology - Canton", "priority": 1256, "external_id": null}, {"id": 10847079003, "name": "SUNY College of Technology - Delhi", "priority": 1257, "external_id": null}, {"id": 10847080003, "name": "SUNY College - Cortland", "priority": 1258, "external_id": null}, {"id": 10847081003, "name": "SUNY College - Old Westbury", "priority": 1259, "external_id": null}, {"id": 10847082003, "name": "SUNY College - Oneonta", "priority": 1260, "external_id": null}, {"id": 10847083003, "name": "SUNY College - Potsdam", "priority": 1261, "external_id": null}, {"id": 10847084003, "name": "SUNY Downstate Medical Center", "priority": 1262, "external_id": null}, {"id": 10847085003, "name": "SUNY Empire State College", "priority": 1263, "external_id": null}, {"id": 10847086003, "name": "SUNY Institute of Technology - Utica/Rome", "priority": 1264, "external_id": null}, {"id": 10847087003, "name": "SUNY Maritime College", "priority": 1265, "external_id": null}, {"id": 10847088003, "name": "SUNY Upstate Medical University", "priority": 1266, "external_id": null}, {"id": 10847089003, "name": "SUNY - Fredonia", "priority": 1267, "external_id": null}, {"id": 10847090003, "name": "SUNY - Geneseo", "priority": 1268, "external_id": null}, {"id": 10847091003, "name": "SUNY - New Paltz", "priority": 1269, "external_id": null}, {"id": 10847092003, "name": "SUNY - Oswego", "priority": 1270, "external_id": null}, {"id": 10847093003, "name": "SUNY - Plattsburgh", "priority": 1271, "external_id": null}, {"id": 10847094003, "name": "Swarthmore College", "priority": 1272, "external_id": null}, {"id": 10847095003, "name": "Sweet Briar College", "priority": 1273, "external_id": null}, {"id": 10847096003, "name": "Tabor College", "priority": 1274, "external_id": null}, {"id": 10847097003, "name": "Talladega College", "priority": 1275, "external_id": null}, {"id": 10847098003, "name": "Tarleton State University", "priority": 1276, "external_id": null}, {"id": 10847099003, "name": "Taylor University", "priority": 1277, "external_id": null}, {"id": 10847100003, "name": "Tennessee Wesleyan College", "priority": 1278, "external_id": null}, {"id": 10847101003, "name": "Texas A&M International University", "priority": 1279, "external_id": null}, {"id": 10847102003, "name": "Texas A&M University - Commerce", "priority": 1280, "external_id": null}, {"id": 10847103003, "name": "Texas A&M University - Corpus Christi", "priority": 1281, "external_id": null}, {"id": 10847104003, "name": "Texas A&M University - Galveston", "priority": 1282, "external_id": null}, {"id": 10847105003, "name": "Texas A&M University - Kingsville", "priority": 1283, "external_id": null}, {"id": 10847106003, "name": "Texas A&M University - Texarkana", "priority": 1284, "external_id": null}, {"id": 10847107003, "name": "Texas College", "priority": 1285, "external_id": null}, {"id": 10847108003, "name": "Texas Lutheran University", "priority": 1286, "external_id": null}, {"id": 10847109003, "name": "Bucknell University", "priority": 1287, "external_id": null}, {"id": 10847110003, "name": "Butler University", "priority": 1288, "external_id": null}, {"id": 10847111003, "name": "Stephen F. Austin State University", "priority": 1289, "external_id": null}, {"id": 10847112003, "name": "Texas A&M University - College Station", "priority": 1290, "external_id": null}, {"id": 10847113003, "name": "Stanford University", "priority": 1291, "external_id": null}, {"id": 10847114003, "name": "Stetson University", "priority": 1292, "external_id": null}, {"id": 10847115003, "name": "Stony Brook University - SUNY", "priority": 1293, "external_id": null}, {"id": 10847116003, "name": "Syracuse University", "priority": 1294, "external_id": null}, {"id": 10847117003, "name": "Texas Christian University", "priority": 1295, "external_id": null}, {"id": 10847118003, "name": "Temple University", "priority": 1296, "external_id": null}, {"id": 10847119003, "name": "Clemson University", "priority": 1297, "external_id": null}, {"id": 10847120003, "name": "Texas Southern University", "priority": 1298, "external_id": null}, {"id": 10847121003, "name": "Austin Peay State University", "priority": 1299, "external_id": null}, {"id": 10847122003, "name": "Tennessee State University", "priority": 1300, "external_id": null}, {"id": 10847123003, "name": "Ball State University", "priority": 1301, "external_id": null}, {"id": 10847124003, "name": "Texas Tech University Health Sciences Center", "priority": 1302, "external_id": null}, {"id": 10847125003, "name": "Texas Wesleyan University", "priority": 1303, "external_id": null}, {"id": 10847126003, "name": "Texas Woman's University", "priority": 1304, "external_id": null}, {"id": 10847127003, "name": "The Catholic University of America", "priority": 1305, "external_id": null}, {"id": 10847128003, "name": "The Sage Colleges", "priority": 1306, "external_id": null}, {"id": 10847129003, "name": "Thiel College", "priority": 1307, "external_id": null}, {"id": 10847130003, "name": "Thomas Aquinas College", "priority": 1308, "external_id": null}, {"id": 10847131003, "name": "Thomas College", "priority": 1309, "external_id": null}, {"id": 10847132003, "name": "Thomas Edison State College", "priority": 1310, "external_id": null}, {"id": 10847133003, "name": "Thomas Jefferson University", "priority": 1311, "external_id": null}, {"id": 10847134003, "name": "Thomas More College", "priority": 1312, "external_id": null}, {"id": 10847135003, "name": "Thomas More College of Liberal Arts", "priority": 1313, "external_id": null}, {"id": 10847136003, "name": "Thomas University", "priority": 1314, "external_id": null}, {"id": 10847137003, "name": "Tiffin University", "priority": 1315, "external_id": null}, {"id": 10847138003, "name": "Tilburg University", "priority": 1316, "external_id": null}, {"id": 10847139003, "name": "Toccoa Falls College", "priority": 1317, "external_id": null}, {"id": 10847140003, "name": "Tougaloo College", "priority": 1318, "external_id": null}, {"id": 10847141003, "name": "Touro College", "priority": 1319, "external_id": null}, {"id": 10847142003, "name": "Transylvania University", "priority": 1320, "external_id": null}, {"id": 10847143003, "name": "Trent University", "priority": 1321, "external_id": null}, {"id": 10847144003, "name": "Trevecca Nazarene University", "priority": 1322, "external_id": null}, {"id": 10847145003, "name": "Trident University International", "priority": 1323, "external_id": null}, {"id": 10847146003, "name": "Trine University", "priority": 1324, "external_id": null}, {"id": 10847147003, "name": "Trinity Christian College", "priority": 1325, "external_id": null}, {"id": 10847148003, "name": "Trinity College", "priority": 1326, "external_id": null}, {"id": 10847149003, "name": "Trinity College of Nursing & Health Sciences", "priority": 1327, "external_id": null}, {"id": 10847150003, "name": "Trinity International University", "priority": 1328, "external_id": null}, {"id": 10847151003, "name": "Trinity Lutheran College", "priority": 1329, "external_id": null}, {"id": 10847152003, "name": "Trinity University", "priority": 1330, "external_id": null}, {"id": 10847153003, "name": "Trinity Western University", "priority": 1331, "external_id": null}, {"id": 10847154003, "name": "Truett McConnell College", "priority": 1332, "external_id": null}, {"id": 10847155003, "name": "Truman State University", "priority": 1333, "external_id": null}, {"id": 10847156003, "name": "Tufts University", "priority": 1334, "external_id": null}, {"id": 10847157003, "name": "Tusculum College", "priority": 1335, "external_id": null}, {"id": 10847158003, "name": "Tuskegee University", "priority": 1336, "external_id": null}, {"id": 10847159003, "name": "Union College", "priority": 1337, "external_id": null}, {"id": 10847160003, "name": "Union Institute and University", "priority": 1338, "external_id": null}, {"id": 10847161003, "name": "Union University", "priority": 1339, "external_id": null}, {"id": 10847162003, "name": "United States Coast Guard Academy", "priority": 1340, "external_id": null}, {"id": 10847163003, "name": "United States International University - Kenya", "priority": 1341, "external_id": null}, {"id": 10847164003, "name": "United States Merchant Marine Academy", "priority": 1342, "external_id": null}, {"id": 10847165003, "name": "United States Sports Academy", "priority": 1343, "external_id": null}, {"id": 10847166003, "name": "Unity College", "priority": 1344, "external_id": null}, {"id": 10847167003, "name": "Universidad Adventista de las Antillas", "priority": 1345, "external_id": null}, {"id": 10847168003, "name": "Universidad del Este", "priority": 1346, "external_id": null}, {"id": 10847169003, "name": "Universidad del Turabo", "priority": 1347, "external_id": null}, {"id": 10847170003, "name": "Universidad Metropolitana", "priority": 1348, "external_id": null}, {"id": 10847171003, "name": "Universidad Politecnica De Puerto Rico", "priority": 1349, "external_id": null}, {"id": 10847172003, "name": "University of Advancing Technology", "priority": 1350, "external_id": null}, {"id": 10847173003, "name": "University of Alabama - Huntsville", "priority": 1351, "external_id": null}, {"id": 10847174003, "name": "University of Alaska - Anchorage", "priority": 1352, "external_id": null}, {"id": 10847175003, "name": "University of Alaska - Fairbanks", "priority": 1353, "external_id": null}, {"id": 10847176003, "name": "University of Alaska - Southeast", "priority": 1354, "external_id": null}, {"id": 10847177003, "name": "University of Alberta", "priority": 1355, "external_id": null}, {"id": 10847178003, "name": "University of Arkansas for Medical Sciences", "priority": 1356, "external_id": null}, {"id": 10847179003, "name": "University of Arkansas - Fort Smith", "priority": 1357, "external_id": null}, {"id": 10847180003, "name": "University of Arkansas - Little Rock", "priority": 1358, "external_id": null}, {"id": 10847181003, "name": "University of Arkansas - Monticello", "priority": 1359, "external_id": null}, {"id": 10847182003, "name": "University of Baltimore", "priority": 1360, "external_id": null}, {"id": 10847183003, "name": "University of Bridgeport", "priority": 1361, "external_id": null}, {"id": 10847184003, "name": "University of British Columbia", "priority": 1362, "external_id": null}, {"id": 10847185003, "name": "University of Calgary", "priority": 1363, "external_id": null}, {"id": 10847186003, "name": "University of California - Riverside", "priority": 1364, "external_id": null}, {"id": 10847187003, "name": "Holy Cross College", "priority": 1365, "external_id": null}, {"id": 10847188003, "name": "Towson University", "priority": 1366, "external_id": null}, {"id": 10847189003, "name": "United States Military Academy", "priority": 1367, "external_id": null}, {"id": 10847190003, "name": "The Citadel", "priority": 1368, "external_id": null}, {"id": 10847191003, "name": "Troy University", "priority": 1369, "external_id": null}, {"id": 10847192003, "name": "University of California - Davis", "priority": 1370, "external_id": null}, {"id": 10847193003, "name": "Grambling State University", "priority": 1371, "external_id": null}, {"id": 10847194003, "name": "University at Albany - SUNY", "priority": 1372, "external_id": null}, {"id": 10847195003, "name": "University at Buffalo - SUNY", "priority": 1373, "external_id": null}, {"id": 10847196003, "name": "United States Naval Academy", "priority": 1374, "external_id": null}, {"id": 10847197003, "name": "University of Arizona", "priority": 1375, "external_id": null}, {"id": 10847198003, "name": "University of California - Los Angeles", "priority": 1376, "external_id": null}, {"id": 10847199003, "name": "Florida A&M University", "priority": 1377, "external_id": null}, {"id": 10847200003, "name": "Texas State University", "priority": 1378, "external_id": null}, {"id": 10847201003, "name": "University of Alabama - Birmingham", "priority": 1379, "external_id": null}, {"id": 10847202003, "name": "University of California - Santa Cruz", "priority": 1380, "external_id": null}, {"id": 10847203003, "name": "University of Central Missouri", "priority": 1381, "external_id": null}, {"id": 10847204003, "name": "University of Central Oklahoma", "priority": 1382, "external_id": null}, {"id": 10847205003, "name": "University of Charleston", "priority": 1383, "external_id": null}, {"id": 10847206003, "name": "University of Chicago", "priority": 1384, "external_id": null}, {"id": 10847207003, "name": "University of Cincinnati - UC Blue Ash College", "priority": 1385, "external_id": null}, {"id": 10847208003, "name": "University of Colorado - Colorado Springs", "priority": 1386, "external_id": null}, {"id": 10847209003, "name": "University of Colorado - Denver", "priority": 1387, "external_id": null}, {"id": 10847210003, "name": "University of Dallas", "priority": 1388, "external_id": null}, {"id": 10847211003, "name": "University of Denver", "priority": 1389, "external_id": null}, {"id": 10847212003, "name": "University of Detroit Mercy", "priority": 1390, "external_id": null}, {"id": 10847213003, "name": "University of Dubuque", "priority": 1391, "external_id": null}, {"id": 10847214003, "name": "University of Evansville", "priority": 1392, "external_id": null}, {"id": 10847215003, "name": "University of Findlay", "priority": 1393, "external_id": null}, {"id": 10847216003, "name": "University of Great Falls", "priority": 1394, "external_id": null}, {"id": 10847217003, "name": "University of Guam", "priority": 1395, "external_id": null}, {"id": 10847218003, "name": "University of Guelph", "priority": 1396, "external_id": null}, {"id": 10847219003, "name": "University of Hartford", "priority": 1397, "external_id": null}, {"id": 10847220003, "name": "University of Hawaii - Hilo", "priority": 1398, "external_id": null}, {"id": 10847221003, "name": "University of Hawaii - Maui College", "priority": 1399, "external_id": null}, {"id": 10847222003, "name": "University of Hawaii - West Oahu", "priority": 1400, "external_id": null}, {"id": 10847223003, "name": "University of Houston - Clear Lake", "priority": 1401, "external_id": null}, {"id": 10847224003, "name": "University of Houston - Downtown", "priority": 1402, "external_id": null}, {"id": 10847225003, "name": "University of Houston - Victoria", "priority": 1403, "external_id": null}, {"id": 10847226003, "name": "University of Illinois - Chicago", "priority": 1404, "external_id": null}, {"id": 10847227003, "name": "University of Illinois - Springfield", "priority": 1405, "external_id": null}, {"id": 10847228003, "name": "University of Indianapolis", "priority": 1406, "external_id": null}, {"id": 10847229003, "name": "University of Jamestown", "priority": 1407, "external_id": null}, {"id": 10847230003, "name": "University of La Verne", "priority": 1408, "external_id": null}, {"id": 10847231003, "name": "University of Maine - Augusta", "priority": 1409, "external_id": null}, {"id": 10847232003, "name": "University of Maine - Farmington", "priority": 1410, "external_id": null}, {"id": 10847233003, "name": "University of Maine - Fort Kent", "priority": 1411, "external_id": null}, {"id": 10847234003, "name": "University of Maine - Machias", "priority": 1412, "external_id": null}, {"id": 10847235003, "name": "University of Maine - Presque Isle", "priority": 1413, "external_id": null}, {"id": 10847236003, "name": "University of Mary", "priority": 1414, "external_id": null}, {"id": 10847237003, "name": "University of Mary Hardin-Baylor", "priority": 1415, "external_id": null}, {"id": 10847238003, "name": "University of Mary Washington", "priority": 1416, "external_id": null}, {"id": 10847239003, "name": "University of Maryland - Baltimore", "priority": 1417, "external_id": null}, {"id": 10847240003, "name": "University of Maryland - Baltimore County", "priority": 1418, "external_id": null}, {"id": 10847241003, "name": "University of Maryland - Eastern Shore", "priority": 1419, "external_id": null}, {"id": 10847242003, "name": "University of Maryland - University College", "priority": 1420, "external_id": null}, {"id": 10847243003, "name": "University of Massachusetts - Boston", "priority": 1421, "external_id": null}, {"id": 10847244003, "name": "University of Massachusetts - Dartmouth", "priority": 1422, "external_id": null}, {"id": 10847245003, "name": "University of Massachusetts - Lowell", "priority": 1423, "external_id": null}, {"id": 10847246003, "name": "University of Medicine and Dentistry of New Jersey", "priority": 1424, "external_id": null}, {"id": 10847247003, "name": "University of Michigan - Dearborn", "priority": 1425, "external_id": null}, {"id": 10847248003, "name": "University of Michigan - Flint", "priority": 1426, "external_id": null}, {"id": 10847249003, "name": "University of Minnesota - Crookston", "priority": 1427, "external_id": null}, {"id": 10847250003, "name": "University of Minnesota - Duluth", "priority": 1428, "external_id": null}, {"id": 10847251003, "name": "University of Minnesota - Morris", "priority": 1429, "external_id": null}, {"id": 10847252003, "name": "University of Mississippi Medical Center", "priority": 1430, "external_id": null}, {"id": 10847253003, "name": "University of Missouri - Kansas City", "priority": 1431, "external_id": null}, {"id": 10847254003, "name": "University of Missouri - St. Louis", "priority": 1432, "external_id": null}, {"id": 10847255003, "name": "University of Mobile", "priority": 1433, "external_id": null}, {"id": 10847256003, "name": "University of Montana - Western", "priority": 1434, "external_id": null}, {"id": 10847257003, "name": "University of Montevallo", "priority": 1435, "external_id": null}, {"id": 10847258003, "name": "University of Mount Union", "priority": 1436, "external_id": null}, {"id": 10847259003, "name": "University of Nebraska Medical Center", "priority": 1437, "external_id": null}, {"id": 10847260003, "name": "University of Nebraska - Kearney", "priority": 1438, "external_id": null}, {"id": 10847261003, "name": "University of Dayton", "priority": 1439, "external_id": null}, {"id": 10847262003, "name": "University of Delaware", "priority": 1440, "external_id": null}, {"id": 10847263003, "name": "University of Florida", "priority": 1441, "external_id": null}, {"id": 10847264003, "name": "University of Iowa", "priority": 1442, "external_id": null}, {"id": 10847265003, "name": "University of Idaho", "priority": 1443, "external_id": null}, {"id": 10847266003, "name": "University of Kentucky", "priority": 1444, "external_id": null}, {"id": 10847267003, "name": "University of Massachusetts - Amherst", "priority": 1445, "external_id": null}, {"id": 10847268003, "name": "University of Maine", "priority": 1446, "external_id": null}, {"id": 10847269003, "name": "University of Michigan - Ann Arbor", "priority": 1447, "external_id": null}, {"id": 10847270003, "name": "University of Cincinnati", "priority": 1448, "external_id": null}, {"id": 10847271003, "name": "University of Miami", "priority": 1449, "external_id": null}, {"id": 10847272003, "name": "University of Louisiana - Monroe", "priority": 1450, "external_id": null}, {"id": 10847273003, "name": "University of Missouri", "priority": 1451, "external_id": null}, {"id": 10847274003, "name": "University of Mississippi", "priority": 1452, "external_id": null}, {"id": 10847275003, "name": "University of Memphis", "priority": 1453, "external_id": null}, {"id": 10847276003, "name": "University of Houston", "priority": 1454, "external_id": null}, {"id": 10847277003, "name": "University of Colorado - Boulder", "priority": 1455, "external_id": null}, {"id": 10847278003, "name": "University of Nebraska - Omaha", "priority": 1456, "external_id": null}, {"id": 10847279003, "name": "University of New Brunswick", "priority": 1457, "external_id": null}, {"id": 10847280003, "name": "University of New England", "priority": 1458, "external_id": null}, {"id": 10847281003, "name": "University of New Haven", "priority": 1459, "external_id": null}, {"id": 10847282003, "name": "University of New Orleans", "priority": 1460, "external_id": null}, {"id": 10847283003, "name": "University of North Alabama", "priority": 1461, "external_id": null}, {"id": 10847284003, "name": "University of North Carolina School of the Arts", "priority": 1462, "external_id": null}, {"id": 10847285003, "name": "University of North Carolina - Asheville", "priority": 1463, "external_id": null}, {"id": 10847286003, "name": "University of North Carolina - Greensboro", "priority": 1464, "external_id": null}, {"id": 10847287003, "name": "University of North Carolina - Pembroke", "priority": 1465, "external_id": null}, {"id": 10847288003, "name": "University of North Carolina - Wilmington", "priority": 1466, "external_id": null}, {"id": 10847289003, "name": "University of North Florida", "priority": 1467, "external_id": null}, {"id": 10847290003, "name": "University of North Georgia", "priority": 1468, "external_id": null}, {"id": 10847291003, "name": "University of Northwestern Ohio", "priority": 1469, "external_id": null}, {"id": 10847292003, "name": "University of Northwestern - St. Paul", "priority": 1470, "external_id": null}, {"id": 10847293003, "name": "University of Ottawa", "priority": 1471, "external_id": null}, {"id": 10847294003, "name": "University of Phoenix", "priority": 1472, "external_id": null}, {"id": 10847295003, "name": "University of Pikeville", "priority": 1473, "external_id": null}, {"id": 10847296003, "name": "University of Portland", "priority": 1474, "external_id": null}, {"id": 10847297003, "name": "University of Prince Edward Island", "priority": 1475, "external_id": null}, {"id": 10847298003, "name": "University of Puerto Rico - Aguadilla", "priority": 1476, "external_id": null}, {"id": 10847299003, "name": "University of Puerto Rico - Arecibo", "priority": 1477, "external_id": null}, {"id": 10847300003, "name": "University of Puerto Rico - Bayamon", "priority": 1478, "external_id": null}, {"id": 10847301003, "name": "University of Puerto Rico - Cayey", "priority": 1479, "external_id": null}, {"id": 10847302003, "name": "University of Puerto Rico - Humacao", "priority": 1480, "external_id": null}, {"id": 10847303003, "name": "University of Puerto Rico - Mayaguez", "priority": 1481, "external_id": null}, {"id": 10847304003, "name": "University of Puerto Rico - Medical Sciences Campus", "priority": 1482, "external_id": null}, {"id": 10847305003, "name": "University of Puerto Rico - Ponce", "priority": 1483, "external_id": null}, {"id": 10847306003, "name": "University of Puerto Rico - Rio Piedras", "priority": 1484, "external_id": null}, {"id": 10847307003, "name": "University of Puget Sound", "priority": 1485, "external_id": null}, {"id": 10847308003, "name": "University of Redlands", "priority": 1486, "external_id": null}, {"id": 10847309003, "name": "University of Regina", "priority": 1487, "external_id": null}, {"id": 10847310003, "name": "University of Rio Grande", "priority": 1488, "external_id": null}, {"id": 10847311003, "name": "University of Rochester", "priority": 1489, "external_id": null}, {"id": 10847312003, "name": "University of San Francisco", "priority": 1490, "external_id": null}, {"id": 10847313003, "name": "University of Saskatchewan", "priority": 1491, "external_id": null}, {"id": 10847314003, "name": "University of Science and Arts of Oklahoma", "priority": 1492, "external_id": null}, {"id": 10847315003, "name": "University of Scranton", "priority": 1493, "external_id": null}, {"id": 10847316003, "name": "University of Sioux Falls", "priority": 1494, "external_id": null}, {"id": 10847317003, "name": "University of South Carolina - Aiken", "priority": 1495, "external_id": null}, {"id": 10847318003, "name": "University of South Carolina - Beaufort", "priority": 1496, "external_id": null}, {"id": 10847319003, "name": "University of South Carolina - Upstate", "priority": 1497, "external_id": null}, {"id": 10847320003, "name": "University of South Florida - St. Petersburg", "priority": 1498, "external_id": null}, {"id": 10847321003, "name": "University of Southern Indiana", "priority": 1499, "external_id": null}, {"id": 10847322003, "name": "University of Southern Maine", "priority": 1500, "external_id": null}, {"id": 10847323003, "name": "University of St. Francis", "priority": 1501, "external_id": null}, {"id": 10847324003, "name": "University of St. Joseph", "priority": 1502, "external_id": null}, {"id": 10847325003, "name": "University of St. Mary", "priority": 1503, "external_id": null}, {"id": 10847326003, "name": "University of St. Thomas", "priority": 1504, "external_id": null}, {"id": 10847327003, "name": "University of Tampa", "priority": 1505, "external_id": null}, {"id": 10847328003, "name": "University of Texas Health Science Center - Houston", "priority": 1506, "external_id": null}, {"id": 10847329003, "name": "University of Texas Health Science Center - San Antonio", "priority": 1507, "external_id": null}, {"id": 10847330003, "name": "University of Texas Medical Branch - Galveston", "priority": 1508, "external_id": null}, {"id": 10847331003, "name": "University of Texas of the Permian Basin", "priority": 1509, "external_id": null}, {"id": 10847332003, "name": "University of Texas - Arlington", "priority": 1510, "external_id": null}, {"id": 10847333003, "name": "University of Texas - Brownsville", "priority": 1511, "external_id": null}, {"id": 10847334003, "name": "University of Texas - Pan American", "priority": 1512, "external_id": null}, {"id": 10847335003, "name": "University of Oregon", "priority": 1513, "external_id": null}, {"id": 10847336003, "name": "University of New Mexico", "priority": 1514, "external_id": null}, {"id": 10847337003, "name": "University of Pennsylvania", "priority": 1515, "external_id": null}, {"id": 10847338003, "name": "University of North Dakota", "priority": 1516, "external_id": null}, {"id": 10847339003, "name": "University of Nevada - Reno", "priority": 1517, "external_id": null}, {"id": 10847340003, "name": "University of New Hampshire", "priority": 1518, "external_id": null}, {"id": 10847341003, "name": "University of Texas - Austin", "priority": 1519, "external_id": null}, {"id": 10847342003, "name": "University of Southern Mississippi", "priority": 1520, "external_id": null}, {"id": 10847343003, "name": "University of Rhode Island", "priority": 1521, "external_id": null}, {"id": 10847344003, "name": "University of South Dakota", "priority": 1522, "external_id": null}, {"id": 10847345003, "name": "University of Tennessee", "priority": 1523, "external_id": null}, {"id": 10847346003, "name": "University of North Texas", "priority": 1524, "external_id": null}, {"id": 10847347003, "name": "University of North Carolina - Charlotte", "priority": 1525, "external_id": null}, {"id": 10847348003, "name": "University of Texas - San Antonio", "priority": 1526, "external_id": null}, {"id": 10847349003, "name": "University of Notre Dame", "priority": 1527, "external_id": null}, {"id": 10847350003, "name": "University of Southern California", "priority": 1528, "external_id": null}, {"id": 10847351003, "name": "University of Texas - Tyler", "priority": 1529, "external_id": null}, {"id": 10847352003, "name": "University of the Arts", "priority": 1530, "external_id": null}, {"id": 10847353003, "name": "University of the Cumberlands", "priority": 1531, "external_id": null}, {"id": 10847354003, "name": "University of the District of Columbia", "priority": 1532, "external_id": null}, {"id": 10847355003, "name": "University of the Ozarks", "priority": 1533, "external_id": null}, {"id": 10847356003, "name": "University of the Pacific", "priority": 1534, "external_id": null}, {"id": 10847357003, "name": "University of the Sacred Heart", "priority": 1535, "external_id": null}, {"id": 10847358003, "name": "University of the Sciences", "priority": 1536, "external_id": null}, {"id": 10847359003, "name": "University of the Southwest", "priority": 1537, "external_id": null}, {"id": 10847360003, "name": "University of the Virgin Islands", "priority": 1538, "external_id": null}, {"id": 10847361003, "name": "University of the West", "priority": 1539, "external_id": null}, {"id": 10847362003, "name": "University of Toronto", "priority": 1540, "external_id": null}, {"id": 10847363003, "name": "University of Vermont", "priority": 1541, "external_id": null}, {"id": 10847364003, "name": "University of Victoria", "priority": 1542, "external_id": null}, {"id": 10847365003, "name": "University of Virginia - Wise", "priority": 1543, "external_id": null}, {"id": 10847366003, "name": "University of Waterloo", "priority": 1544, "external_id": null}, {"id": 10847367003, "name": "University of West Alabama", "priority": 1545, "external_id": null}, {"id": 10847368003, "name": "University of West Florida", "priority": 1546, "external_id": null}, {"id": 10847369003, "name": "University of West Georgia", "priority": 1547, "external_id": null}, {"id": 10847370003, "name": "University of Windsor", "priority": 1548, "external_id": null}, {"id": 10847371003, "name": "University of Winnipeg", "priority": 1549, "external_id": null}, {"id": 10847372003, "name": "University of Wisconsin - Eau Claire", "priority": 1550, "external_id": null}, {"id": 10847373003, "name": "University of Wisconsin - Green Bay", "priority": 1551, "external_id": null}, {"id": 10847374003, "name": "University of Wisconsin - La Crosse", "priority": 1552, "external_id": null}, {"id": 10847375003, "name": "University of Wisconsin - Milwaukee", "priority": 1553, "external_id": null}, {"id": 10847376003, "name": "University of Wisconsin - Oshkosh", "priority": 1554, "external_id": null}, {"id": 10847377003, "name": "University of Wisconsin - Parkside", "priority": 1555, "external_id": null}, {"id": 10847378003, "name": "University of Wisconsin - Platteville", "priority": 1556, "external_id": null}, {"id": 10847379003, "name": "University of Wisconsin - River Falls", "priority": 1557, "external_id": null}, {"id": 10847380003, "name": "University of Wisconsin - Stevens Point", "priority": 1558, "external_id": null}, {"id": 10847381003, "name": "University of Wisconsin - Stout", "priority": 1559, "external_id": null}, {"id": 10847382003, "name": "University of Wisconsin - Superior", "priority": 1560, "external_id": null}, {"id": 10847383003, "name": "University of Wisconsin - Whitewater", "priority": 1561, "external_id": null}, {"id": 10847384003, "name": "Upper Iowa University", "priority": 1562, "external_id": null}, {"id": 10847385003, "name": "Urbana University", "priority": 1563, "external_id": null}, {"id": 10847386003, "name": "Ursinus College", "priority": 1564, "external_id": null}, {"id": 10847387003, "name": "Ursuline College", "priority": 1565, "external_id": null}, {"id": 10847388003, "name": "Utah Valley University", "priority": 1566, "external_id": null}, {"id": 10847389003, "name": "Utica College", "priority": 1567, "external_id": null}, {"id": 10847390003, "name": "Valdosta State University", "priority": 1568, "external_id": null}, {"id": 10847391003, "name": "Valley City State University", "priority": 1569, "external_id": null}, {"id": 10847392003, "name": "Valley Forge Christian College", "priority": 1570, "external_id": null}, {"id": 10847393003, "name": "VanderCook College of Music", "priority": 1571, "external_id": null}, {"id": 10847394003, "name": "Vanguard University of Southern California", "priority": 1572, "external_id": null}, {"id": 10847395003, "name": "Vassar College", "priority": 1573, "external_id": null}, {"id": 10847396003, "name": "Vaughn College of Aeronautics and Technology", "priority": 1574, "external_id": null}, {"id": 10847397003, "name": "Vermont Technical College", "priority": 1575, "external_id": null}, {"id": 10847398003, "name": "Victory University", "priority": 1576, "external_id": null}, {"id": 10847399003, "name": "Vincennes University", "priority": 1577, "external_id": null}, {"id": 10847400003, "name": "Virginia Commonwealth University", "priority": 1578, "external_id": null}, {"id": 10847401003, "name": "Virginia Intermont College", "priority": 1579, "external_id": null}, {"id": 10847402003, "name": "Virginia State University", "priority": 1580, "external_id": null}, {"id": 10847403003, "name": "Virginia Union University", "priority": 1581, "external_id": null}, {"id": 10847404003, "name": "Virginia Wesleyan College", "priority": 1582, "external_id": null}, {"id": 10847405003, "name": "Viterbo University", "priority": 1583, "external_id": null}, {"id": 10847406003, "name": "Voorhees College", "priority": 1584, "external_id": null}, {"id": 10847407003, "name": "Wabash College", "priority": 1585, "external_id": null}, {"id": 10847408003, "name": "Walden University", "priority": 1586, "external_id": null}, {"id": 10847409003, "name": "Waldorf College", "priority": 1587, "external_id": null}, {"id": 10847410003, "name": "Walla Walla University", "priority": 1588, "external_id": null}, {"id": 10847411003, "name": "Walsh College of Accountancy and Business Administration", "priority": 1589, "external_id": null}, {"id": 10847412003, "name": "Walsh University", "priority": 1590, "external_id": null}, {"id": 10847413003, "name": "Warner Pacific College", "priority": 1591, "external_id": null}, {"id": 10847414003, "name": "Warner University", "priority": 1592, "external_id": null}, {"id": 10847415003, "name": "Warren Wilson College", "priority": 1593, "external_id": null}, {"id": 10847416003, "name": "Wartburg College", "priority": 1594, "external_id": null}, {"id": 10847417003, "name": "Washburn University", "priority": 1595, "external_id": null}, {"id": 10847418003, "name": "Washington Adventist University", "priority": 1596, "external_id": null}, {"id": 10847419003, "name": "Washington and Jefferson College", "priority": 1597, "external_id": null}, {"id": 10847420003, "name": "Washington and Lee University", "priority": 1598, "external_id": null}, {"id": 10847421003, "name": "Washington College", "priority": 1599, "external_id": null}, {"id": 10847422003, "name": "Washington University in St. Louis", "priority": 1600, "external_id": null}, {"id": 10847423003, "name": "Watkins College of Art, Design & Film", "priority": 1601, "external_id": null}, {"id": 10847424003, "name": "Wayland Baptist University", "priority": 1602, "external_id": null}, {"id": 10847425003, "name": "Wayne State College", "priority": 1603, "external_id": null}, {"id": 10847426003, "name": "Wayne State University", "priority": 1604, "external_id": null}, {"id": 10847427003, "name": "Waynesburg University", "priority": 1605, "external_id": null}, {"id": 10847428003, "name": "Valparaiso University", "priority": 1606, "external_id": null}, {"id": 10847429003, "name": "Villanova University", "priority": 1607, "external_id": null}, {"id": 10847430003, "name": "Virginia Tech", "priority": 1608, "external_id": null}, {"id": 10847431003, "name": "Washington State University", "priority": 1609, "external_id": null}, {"id": 10847432003, "name": "University of Toledo", "priority": 1610, "external_id": null}, {"id": 10847433003, "name": "Wagner College", "priority": 1611, "external_id": null}, {"id": 10847434003, "name": "University of Wyoming", "priority": 1612, "external_id": null}, {"id": 10847435003, "name": "University of Wisconsin - Madison", "priority": 1613, "external_id": null}, {"id": 10847436003, "name": "University of Tulsa", "priority": 1614, "external_id": null}, {"id": 10847437003, "name": "Webb Institute", "priority": 1615, "external_id": null}, {"id": 10847438003, "name": "Webber International University", "priority": 1616, "external_id": null}, {"id": 10847439003, "name": "Webster University", "priority": 1617, "external_id": null}, {"id": 10847440003, "name": "Welch College", "priority": 1618, "external_id": null}, {"id": 10847441003, "name": "Wellesley College", "priority": 1619, "external_id": null}, {"id": 10847442003, "name": "Wells College", "priority": 1620, "external_id": null}, {"id": 10847443003, "name": "Wentworth Institute of Technology", "priority": 1621, "external_id": null}, {"id": 10847444003, "name": "Wesley College", "priority": 1622, "external_id": null}, {"id": 10847445003, "name": "Wesleyan College", "priority": 1623, "external_id": null}, {"id": 10847446003, "name": "Wesleyan University", "priority": 1624, "external_id": null}, {"id": 10847447003, "name": "West Chester University of Pennsylvania", "priority": 1625, "external_id": null}, {"id": 10847448003, "name": "West Liberty University", "priority": 1626, "external_id": null}, {"id": 10847449003, "name": "West Texas A&M University", "priority": 1627, "external_id": null}, {"id": 10847450003, "name": "West Virginia State University", "priority": 1628, "external_id": null}, {"id": 10847451003, "name": "West Virginia University Institute of Technology", "priority": 1629, "external_id": null}, {"id": 10847452003, "name": "West Virginia University - Parkersburg", "priority": 1630, "external_id": null}, {"id": 10847453003, "name": "West Virginia Wesleyan College", "priority": 1631, "external_id": null}, {"id": 10847454003, "name": "Western Connecticut State University", "priority": 1632, "external_id": null}, {"id": 10847455003, "name": "Western Governors University", "priority": 1633, "external_id": null}, {"id": 10847456003, "name": "Western International University", "priority": 1634, "external_id": null}, {"id": 10847457003, "name": "Western Nevada College", "priority": 1635, "external_id": null}, {"id": 10847458003, "name": "Western New England University", "priority": 1636, "external_id": null}, {"id": 10847459003, "name": "Western New Mexico University", "priority": 1637, "external_id": null}, {"id": 10847460003, "name": "Western Oregon University", "priority": 1638, "external_id": null}, {"id": 10847461003, "name": "Western State Colorado University", "priority": 1639, "external_id": null}, {"id": 10847462003, "name": "Western University", "priority": 1640, "external_id": null}, {"id": 10847463003, "name": "Western Washington University", "priority": 1641, "external_id": null}, {"id": 10847464003, "name": "Westfield State University", "priority": 1642, "external_id": null}, {"id": 10847465003, "name": "Westminster College", "priority": 1643, "external_id": null}, {"id": 10847466003, "name": "Westmont College", "priority": 1644, "external_id": null}, {"id": 10847467003, "name": "Wheaton College", "priority": 1645, "external_id": null}, {"id": 10847468003, "name": "Wheeling Jesuit University", "priority": 1646, "external_id": null}, {"id": 10847469003, "name": "Wheelock College", "priority": 1647, "external_id": null}, {"id": 10847470003, "name": "Whitman College", "priority": 1648, "external_id": null}, {"id": 10847471003, "name": "Whittier College", "priority": 1649, "external_id": null}, {"id": 10847472003, "name": "Whitworth University", "priority": 1650, "external_id": null}, {"id": 10847473003, "name": "Wichita State University", "priority": 1651, "external_id": null}, {"id": 10847474003, "name": "Widener University", "priority": 1652, "external_id": null}, {"id": 10847475003, "name": "Wilberforce University", "priority": 1653, "external_id": null}, {"id": 10847476003, "name": "Wiley College", "priority": 1654, "external_id": null}, {"id": 10847477003, "name": "Wilkes University", "priority": 1655, "external_id": null}, {"id": 10847478003, "name": "Willamette University", "priority": 1656, "external_id": null}, {"id": 10847479003, "name": "William Carey University", "priority": 1657, "external_id": null}, {"id": 10847480003, "name": "William Jessup University", "priority": 1658, "external_id": null}, {"id": 10847481003, "name": "William Jewell College", "priority": 1659, "external_id": null}, {"id": 10847482003, "name": "William Paterson University of New Jersey", "priority": 1660, "external_id": null}, {"id": 10847483003, "name": "William Peace University", "priority": 1661, "external_id": null}, {"id": 10847484003, "name": "William Penn University", "priority": 1662, "external_id": null}, {"id": 10847485003, "name": "William Woods University", "priority": 1663, "external_id": null}, {"id": 10847486003, "name": "Williams Baptist College", "priority": 1664, "external_id": null}, {"id": 10847487003, "name": "Williams College", "priority": 1665, "external_id": null}, {"id": 10847488003, "name": "Wilmington College", "priority": 1666, "external_id": null}, {"id": 10847489003, "name": "Wilmington University", "priority": 1667, "external_id": null}, {"id": 10847490003, "name": "Wilson College", "priority": 1668, "external_id": null}, {"id": 10847491003, "name": "Wingate University", "priority": 1669, "external_id": null}, {"id": 10847492003, "name": "Winona State University", "priority": 1670, "external_id": null}, {"id": 10847493003, "name": "Winston-Salem State University", "priority": 1671, "external_id": null}, {"id": 10847494003, "name": "Winthrop University", "priority": 1672, "external_id": null}, {"id": 10847495003, "name": "Wisconsin Lutheran College", "priority": 1673, "external_id": null}, {"id": 10847496003, "name": "Wittenberg University", "priority": 1674, "external_id": null}, {"id": 10847497003, "name": "Woodbury University", "priority": 1675, "external_id": null}, {"id": 10847498003, "name": "Worcester Polytechnic Institute", "priority": 1676, "external_id": null}, {"id": 10847499003, "name": "Worcester State University", "priority": 1677, "external_id": null}, {"id": 10847500003, "name": "Wright State University", "priority": 1678, "external_id": null}, {"id": 10847501003, "name": "Xavier University", "priority": 1679, "external_id": null}, {"id": 10847502003, "name": "Xavier University of Louisiana", "priority": 1680, "external_id": null}, {"id": 10847503003, "name": "Yeshiva University", "priority": 1681, "external_id": null}, {"id": 10847504003, "name": "York College", "priority": 1682, "external_id": null}, {"id": 10847505003, "name": "York College of Pennsylvania", "priority": 1683, "external_id": null}, {"id": 10847506003, "name": "York University", "priority": 1684, "external_id": null}, {"id": 10847507003, "name": "University of Cambridge", "priority": 1685, "external_id": null}, {"id": 10847508003, "name": "UCL (University College London)", "priority": 1686, "external_id": null}, {"id": 10847509003, "name": "Imperial College London", "priority": 1687, "external_id": null}, {"id": 10847510003, "name": "University of Oxford", "priority": 1688, "external_id": null}, {"id": 10847511003, "name": "ETH Zurich (Swiss Federal Institute of Technology)", "priority": 1689, "external_id": null}, {"id": 10847512003, "name": "University of Edinburgh", "priority": 1690, "external_id": null}, {"id": 10847513003, "name": "Ecole Polytechnique F\u00e9d\u00e9rale de Lausanne", "priority": 1691, "external_id": null}, {"id": 10847514003, "name": "King's College London (KCL)", "priority": 1692, "external_id": null}, {"id": 10847515003, "name": "National University of Singapore (NUS)", "priority": 1693, "external_id": null}, {"id": 10847516003, "name": "University of Hong Kong", "priority": 1694, "external_id": null}, {"id": 10847517003, "name": "Australian National University", "priority": 1695, "external_id": null}, {"id": 10847518003, "name": "Ecole normale sup\u00e9rieure, Paris", "priority": 1696, "external_id": null}, {"id": 10847519003, "name": "University of Bristol", "priority": 1697, "external_id": null}, {"id": 10847520003, "name": "The University of Melbourne", "priority": 1698, "external_id": null}, {"id": 10847521003, "name": "The University of Tokyo", "priority": 1699, "external_id": null}, {"id": 10847522003, "name": "The University of Manchester", "priority": 1700, "external_id": null}, {"id": 10847523003, "name": "Western Illinois University", "priority": 1701, "external_id": null}, {"id": 10847524003, "name": "Wofford College", "priority": 1702, "external_id": null}, {"id": 10847525003, "name": "Western Carolina University", "priority": 1703, "external_id": null}, {"id": 10847526003, "name": "West Virginia University", "priority": 1704, "external_id": null}, {"id": 10847527003, "name": "Yale University", "priority": 1705, "external_id": null}, {"id": 10847528003, "name": "The Hong Kong University of Science and Technology", "priority": 1706, "external_id": null}, {"id": 10847529003, "name": "Kyoto University", "priority": 1707, "external_id": null}, {"id": 10847530003, "name": "Seoul National University", "priority": 1708, "external_id": null}, {"id": 10847531003, "name": "The University of Sydney", "priority": 1709, "external_id": null}, {"id": 10847532003, "name": "The Chinese University of Hong Kong", "priority": 1710, "external_id": null}, {"id": 10847533003, "name": "Ecole Polytechnique", "priority": 1711, "external_id": null}, {"id": 10847534003, "name": "Nanyang Technological University (NTU)", "priority": 1712, "external_id": null}, {"id": 10847535003, "name": "The University of Queensland", "priority": 1713, "external_id": null}, {"id": 10847536003, "name": "University of Copenhagen", "priority": 1714, "external_id": null}, {"id": 10847537003, "name": "Peking University", "priority": 1715, "external_id": null}, {"id": 10847538003, "name": "Tsinghua University", "priority": 1716, "external_id": null}, {"id": 10847539003, "name": "Ruprecht-Karls-Universit\u00e4t Heidelberg", "priority": 1717, "external_id": null}, {"id": 10847540003, "name": "University of Glasgow", "priority": 1718, "external_id": null}, {"id": 10847541003, "name": "The University of New South Wales", "priority": 1719, "external_id": null}, {"id": 10847542003, "name": "Technische Universit\u00e4t M\u00fcnchen", "priority": 1720, "external_id": null}, {"id": 10847543003, "name": "Osaka University", "priority": 1721, "external_id": null}, {"id": 10847544003, "name": "University of Amsterdam", "priority": 1722, "external_id": null}, {"id": 10847545003, "name": "KAIST - Korea Advanced Institute of Science & Technology", "priority": 1723, "external_id": null}, {"id": 10847546003, "name": "Trinity College Dublin", "priority": 1724, "external_id": null}, {"id": 10847547003, "name": "University of Birmingham", "priority": 1725, "external_id": null}, {"id": 10847548003, "name": "The University of Warwick", "priority": 1726, "external_id": null}, {"id": 10847549003, "name": "Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen", "priority": 1727, "external_id": null}, {"id": 10847550003, "name": "Tokyo Institute of Technology", "priority": 1728, "external_id": null}, {"id": 10847551003, "name": "Lund University", "priority": 1729, "external_id": null}, {"id": 10847552003, "name": "London School of Economics and Political Science (LSE)", "priority": 1730, "external_id": null}, {"id": 10847553003, "name": "Monash University", "priority": 1731, "external_id": null}, {"id": 10847554003, "name": "University of Helsinki", "priority": 1732, "external_id": null}, {"id": 10847555003, "name": "The University of Sheffield", "priority": 1733, "external_id": null}, {"id": 10847556003, "name": "University of Geneva", "priority": 1734, "external_id": null}, {"id": 10847557003, "name": "Leiden University", "priority": 1735, "external_id": null}, {"id": 10847558003, "name": "The University of Nottingham", "priority": 1736, "external_id": null}, {"id": 10847559003, "name": "Tohoku University", "priority": 1737, "external_id": null}, {"id": 10847560003, "name": "KU Leuven", "priority": 1738, "external_id": null}, {"id": 10847561003, "name": "University of Zurich", "priority": 1739, "external_id": null}, {"id": 10847562003, "name": "Uppsala University", "priority": 1740, "external_id": null}, {"id": 10847563003, "name": "Utrecht University", "priority": 1741, "external_id": null}, {"id": 10847564003, "name": "National Taiwan University (NTU)", "priority": 1742, "external_id": null}, {"id": 10847565003, "name": "University of St Andrews", "priority": 1743, "external_id": null}, {"id": 10847566003, "name": "The University of Western Australia", "priority": 1744, "external_id": null}, {"id": 10847567003, "name": "University of Southampton", "priority": 1745, "external_id": null}, {"id": 10847568003, "name": "Fudan University", "priority": 1746, "external_id": null}, {"id": 10847569003, "name": "University of Oslo", "priority": 1747, "external_id": null}, {"id": 10847570003, "name": "Durham University", "priority": 1748, "external_id": null}, {"id": 10847571003, "name": "Aarhus University", "priority": 1749, "external_id": null}, {"id": 10847572003, "name": "Erasmus University Rotterdam", "priority": 1750, "external_id": null}, {"id": 10847573003, "name": "Universit\u00e9 de Montr\u00e9al", "priority": 1751, "external_id": null}, {"id": 10847574003, "name": "The University of Auckland", "priority": 1752, "external_id": null}, {"id": 10847575003, "name": "Delft University of Technology", "priority": 1753, "external_id": null}, {"id": 10847576003, "name": "University of Groningen", "priority": 1754, "external_id": null}, {"id": 10847577003, "name": "University of Leeds", "priority": 1755, "external_id": null}, {"id": 10847578003, "name": "Nagoya University", "priority": 1756, "external_id": null}, {"id": 10847579003, "name": "Universit\u00e4t Freiburg", "priority": 1757, "external_id": null}, {"id": 10847580003, "name": "City University of Hong Kong", "priority": 1758, "external_id": null}, {"id": 10847581003, "name": "The University of Adelaide", "priority": 1759, "external_id": null}, {"id": 10847582003, "name": "Pohang University of Science And Technology (POSTECH)", "priority": 1760, "external_id": null}, {"id": 10847583003, "name": "Freie Universit\u00e4t Berlin", "priority": 1761, "external_id": null}, {"id": 10847584003, "name": "University of Basel", "priority": 1762, "external_id": null}, {"id": 10847585003, "name": "University of Lausanne", "priority": 1763, "external_id": null}, {"id": 10847586003, "name": "Universit\u00e9 Pierre et Marie Curie (UPMC)", "priority": 1764, "external_id": null}, {"id": 10847587003, "name": "Yonsei University", "priority": 1765, "external_id": null}, {"id": 10847588003, "name": "University of York", "priority": 1766, "external_id": null}, {"id": 10847589003, "name": "Queen Mary, University of London (QMUL)", "priority": 1767, "external_id": null}, {"id": 10847590003, "name": "Karlsruhe Institute of Technology (KIT)", "priority": 1768, "external_id": null}, {"id": 10847591003, "name": "KTH, Royal Institute of Technology", "priority": 1769, "external_id": null}, {"id": 10847592003, "name": "Lomonosov Moscow State University", "priority": 1770, "external_id": null}, {"id": 10847593003, "name": "Maastricht University", "priority": 1771, "external_id": null}, {"id": 10847594003, "name": "University of Ghent", "priority": 1772, "external_id": null}, {"id": 10847595003, "name": "Shanghai Jiao Tong University", "priority": 1773, "external_id": null}, {"id": 10847596003, "name": "Humboldt-Universit\u00e4t zu Berlin", "priority": 1774, "external_id": null}, {"id": 10847597003, "name": "Universidade de S\u00e3o Paulo (USP)", "priority": 1775, "external_id": null}, {"id": 10847598003, "name": "Georg-August-Universit\u00e4t G\u00f6ttingen", "priority": 1776, "external_id": null}, {"id": 10847599003, "name": "Newcastle University", "priority": 1777, "external_id": null}, {"id": 10847600003, "name": "University of Liverpool", "priority": 1778, "external_id": null}, {"id": 10847601003, "name": "Kyushu University", "priority": 1779, "external_id": null}, {"id": 10847602003, "name": "Eberhard Karls Universit\u00e4t T\u00fcbingen", "priority": 1780, "external_id": null}, {"id": 10847603003, "name": "Technical University of Denmark", "priority": 1781, "external_id": null}, {"id": 10847604003, "name": "Cardiff University", "priority": 1782, "external_id": null}, {"id": 10847605003, "name": "Universit\u00e9 Catholique de Louvain (UCL)", "priority": 1783, "external_id": null}, {"id": 10847606003, "name": "University College Dublin", "priority": 1784, "external_id": null}, {"id": 10847607003, "name": "McMaster University", "priority": 1785, "external_id": null}, {"id": 10847608003, "name": "Hebrew University of Jerusalem", "priority": 1786, "external_id": null}, {"id": 10847609003, "name": "Radboud University Nijmegen", "priority": 1787, "external_id": null}, {"id": 10847610003, "name": "Hokkaido University", "priority": 1788, "external_id": null}, {"id": 10847611003, "name": "Korea University", "priority": 1789, "external_id": null}, {"id": 10847612003, "name": "University of Cape Town", "priority": 1790, "external_id": null}, {"id": 10847613003, "name": "Rheinisch-Westf\u00e4lische Technische Hochschule Aachen", "priority": 1791, "external_id": null}, {"id": 10847614003, "name": "University of Aberdeen", "priority": 1792, "external_id": null}, {"id": 10847615003, "name": "Wageningen University", "priority": 1793, "external_id": null}, {"id": 10847616003, "name": "University of Bergen", "priority": 1794, "external_id": null}, {"id": 10847617003, "name": "University of Bern", "priority": 1795, "external_id": null}, {"id": 10847618003, "name": "University of Otago", "priority": 1796, "external_id": null}, {"id": 10847619003, "name": "Lancaster University", "priority": 1797, "external_id": null}, {"id": 10847620003, "name": "Eindhoven University of Technology", "priority": 1798, "external_id": null}, {"id": 10847621003, "name": "Ecole Normale Sup\u00e9rieure de Lyon", "priority": 1799, "external_id": null}, {"id": 10847622003, "name": "University of Vienna", "priority": 1800, "external_id": null}, {"id": 10847623003, "name": "The Hong Kong Polytechnic University", "priority": 1801, "external_id": null}, {"id": 10847624003, "name": "Sungkyunkwan University", "priority": 1802, "external_id": null}, {"id": 10847625003, "name": "Rheinische Friedrich-Wilhelms-Universit\u00e4t Bonn", "priority": 1803, "external_id": null}, {"id": 10847626003, "name": "Universidad Nacional Aut\u00f3noma de M\u00e9xico (UNAM)", "priority": 1804, "external_id": null}, {"id": 10847627003, "name": "Zhejiang University", "priority": 1805, "external_id": null}, {"id": 10847628003, "name": "Pontificia Universidad Cat\u00f3lica de Chile", "priority": 1806, "external_id": null}, {"id": 10847629003, "name": "Universiti Malaya (UM)", "priority": 1807, "external_id": null}, {"id": 10847630003, "name": "Universit\u00e9 Libre de Bruxelles (ULB)", "priority": 1808, "external_id": null}, {"id": 10847631003, "name": "University of Exeter", "priority": 1809, "external_id": null}, {"id": 10847632003, "name": "Stockholm University", "priority": 1810, "external_id": null}, {"id": 10847633003, "name": "Queen's University of Belfast", "priority": 1811, "external_id": null}, {"id": 10847634003, "name": "Vrije Universiteit Brussel (VUB)", "priority": 1812, "external_id": null}, {"id": 10847635003, "name": "University of Science and Technology of China", "priority": 1813, "external_id": null}, {"id": 10847636003, "name": "Nanjing University", "priority": 1814, "external_id": null}, {"id": 10847637003, "name": "Universitat Aut\u00f3noma de Barcelona", "priority": 1815, "external_id": null}, {"id": 10847638003, "name": "University of Barcelona", "priority": 1816, "external_id": null}, {"id": 10847639003, "name": "VU University Amsterdam", "priority": 1817, "external_id": null}, {"id": 10847640003, "name": "Technion - Israel Institute of Technology", "priority": 1818, "external_id": null}, {"id": 10847641003, "name": "Technische Universit\u00e4t Berlin", "priority": 1819, "external_id": null}, {"id": 10847642003, "name": "University of Antwerp", "priority": 1820, "external_id": null}, {"id": 10847643003, "name": "Universit\u00e4t Hamburg", "priority": 1821, "external_id": null}, {"id": 10847644003, "name": "University of Bath", "priority": 1822, "external_id": null}, {"id": 10847645003, "name": "University of Bologna", "priority": 1823, "external_id": null}, {"id": 10847646003, "name": "Queen's University, Ontario", "priority": 1824, "external_id": null}, {"id": 10847647003, "name": "Universit\u00e9 Paris-Sud 11", "priority": 1825, "external_id": null}, {"id": 10847648003, "name": "Keio University", "priority": 1826, "external_id": null}, {"id": 10847649003, "name": "University of Sussex", "priority": 1827, "external_id": null}, {"id": 10847650003, "name": "Universidad Aut\u00f3noma de Madrid", "priority": 1828, "external_id": null}, {"id": 10847651003, "name": "Aalto University", "priority": 1829, "external_id": null}, {"id": 10847652003, "name": "Sapienza University of Rome", "priority": 1830, "external_id": null}, {"id": 10847653003, "name": "Tel Aviv University", "priority": 1831, "external_id": null}, {"id": 10847654003, "name": "National Tsing Hua University", "priority": 1832, "external_id": null}, {"id": 10847655003, "name": "Chalmers University of Technology", "priority": 1833, "external_id": null}, {"id": 10847656003, "name": "University of Leicester", "priority": 1834, "external_id": null}, {"id": 10847657003, "name": "Universit\u00e9 Paris Diderot - Paris 7", "priority": 1835, "external_id": null}, {"id": 10847658003, "name": "University of Gothenburg", "priority": 1836, "external_id": null}, {"id": 10847659003, "name": "University of Turku", "priority": 1837, "external_id": null}, {"id": 10847660003, "name": "Universit\u00e4t Frankfurt am Main", "priority": 1838, "external_id": null}, {"id": 10847661003, "name": "Universidad de Buenos Aires", "priority": 1839, "external_id": null}, {"id": 10847662003, "name": "University College Cork", "priority": 1840, "external_id": null}, {"id": 10847663003, "name": "University of Tsukuba", "priority": 1841, "external_id": null}, {"id": 10847664003, "name": "University of Reading", "priority": 1842, "external_id": null}, {"id": 10847665003, "name": "Sciences Po Paris", "priority": 1843, "external_id": null}, {"id": 10847666003, "name": "Universidade Estadual de Campinas", "priority": 1844, "external_id": null}, {"id": 10847667003, "name": "King Fahd University of Petroleum & Minerals", "priority": 1845, "external_id": null}, {"id": 10847668003, "name": "University Complutense Madrid", "priority": 1846, "external_id": null}, {"id": 10847669003, "name": "Universit\u00e9 Paris-Sorbonne (Paris IV)", "priority": 1847, "external_id": null}, {"id": 10847670003, "name": "University of Dundee", "priority": 1848, "external_id": null}, {"id": 10847671003, "name": "Universit\u00e9 Joseph Fourier - Grenoble 1", "priority": 1849, "external_id": null}, {"id": 10847672003, "name": "Waseda University", "priority": 1850, "external_id": null}, {"id": 10847673003, "name": "Indian Institute of Technology Delhi (IITD)", "priority": 1851, "external_id": null}, {"id": 10847674003, "name": "Universidad de Chile", "priority": 1852, "external_id": null}, {"id": 10847675003, "name": "Universit\u00e9 Paris 1 Panth\u00e9on-Sorbonne", "priority": 1853, "external_id": null}, {"id": 10847676003, "name": "Universit\u00e9 de Strasbourg", "priority": 1854, "external_id": null}, {"id": 10847677003, "name": "University of Twente", "priority": 1855, "external_id": null}, {"id": 10847678003, "name": "University of East Anglia (UEA)", "priority": 1856, "external_id": null}, {"id": 10847679003, "name": "National Chiao Tung University", "priority": 1857, "external_id": null}, {"id": 10847680003, "name": "Politecnico di Milano", "priority": 1858, "external_id": null}, {"id": 10847681003, "name": "Charles University", "priority": 1859, "external_id": null}, {"id": 10847682003, "name": "Indian Institute of Technology Bombay (IITB)", "priority": 1860, "external_id": null}, {"id": 10847683003, "name": "University of Milano", "priority": 1861, "external_id": null}, {"id": 10847684003, "name": "Westf\u00e4lische Wilhelms-Universit\u00e4t M\u00fcnster", "priority": 1862, "external_id": null}, {"id": 10847685003, "name": "University of Canterbury", "priority": 1863, "external_id": null}, {"id": 10847686003, "name": "Chulalongkorn University", "priority": 1864, "external_id": null}, {"id": 10847687003, "name": "Saint-Petersburg State University", "priority": 1865, "external_id": null}, {"id": 10847688003, "name": "University of Liege", "priority": 1866, "external_id": null}, {"id": 10847689003, "name": "Universit\u00e4t zu K\u00f6ln", "priority": 1867, "external_id": null}, {"id": 10847690003, "name": "Loughborough University", "priority": 1868, "external_id": null}, {"id": 10847691003, "name": "National Cheng Kung University", "priority": 1869, "external_id": null}, {"id": 10847692003, "name": "Universit\u00e4t Stuttgart", "priority": 1870, "external_id": null}, {"id": 10847693003, "name": "Hanyang University", "priority": 1871, "external_id": null}, {"id": 10847694003, "name": "American University of Beirut (AUB)", "priority": 1872, "external_id": null}, {"id": 10847695003, "name": "Norwegian University of Science And Technology", "priority": 1873, "external_id": null}, {"id": 10847696003, "name": "Beijing Normal University", "priority": 1874, "external_id": null}, {"id": 10847697003, "name": "King Saud University", "priority": 1875, "external_id": null}, {"id": 10847698003, "name": "University of Oulu", "priority": 1876, "external_id": null}, {"id": 10847699003, "name": "Kyung Hee University", "priority": 1877, "external_id": null}, {"id": 10847700003, "name": "University of Strathclyde", "priority": 1878, "external_id": null}, {"id": 10847701003, "name": "Universit\u00e4t Ulm", "priority": 1879, "external_id": null}, {"id": 10847702003, "name": "University of Pisa", "priority": 1880, "external_id": null}, {"id": 10847703003, "name": "Technische Universit\u00e4t Darmstadt", "priority": 1881, "external_id": null}, {"id": 10847704003, "name": "Technische Universit\u00e4t Dresden", "priority": 1882, "external_id": null}, {"id": 10847705003, "name": "Macquarie University", "priority": 1883, "external_id": null}, {"id": 10847706003, "name": "Vienna University of Technology", "priority": 1884, "external_id": null}, {"id": 10847707003, "name": "Royal Holloway University of London", "priority": 1885, "external_id": null}, {"id": 10847708003, "name": "Victoria University of Wellington", "priority": 1886, "external_id": null}, {"id": 10847709003, "name": "University of Padua", "priority": 1887, "external_id": null}, {"id": 10847710003, "name": "Universiti Kebangsaan Malaysia (UKM)", "priority": 1888, "external_id": null}, {"id": 10847711003, "name": "University of Technology, Sydney", "priority": 1889, "external_id": null}, {"id": 10847712003, "name": "Universit\u00e4t Konstanz", "priority": 1890, "external_id": null}, {"id": 10847713003, "name": "Universidad de Los Andes Colombia", "priority": 1891, "external_id": null}, {"id": 10847714003, "name": "Universit\u00e9 Paris Descartes", "priority": 1892, "external_id": null}, {"id": 10847715003, "name": "Tokyo Medical and Dental University", "priority": 1893, "external_id": null}, {"id": 10847716003, "name": "University of Wollongong", "priority": 1894, "external_id": null}, {"id": 10847717003, "name": "Universit\u00e4t Erlangen-N\u00fcrnberg", "priority": 1895, "external_id": null}, {"id": 10847718003, "name": "Queensland University of Technology", "priority": 1896, "external_id": null}, {"id": 10847719003, "name": "Tecnol\u00f3gico de Monterrey (ITESM)", "priority": 1897, "external_id": null}, {"id": 10847720003, "name": "Universit\u00e4t Mannheim", "priority": 1898, "external_id": null}, {"id": 10847721003, "name": "Universitat Pompeu Fabra", "priority": 1899, "external_id": null}, {"id": 10847722003, "name": "Mahidol University", "priority": 1900, "external_id": null}, {"id": 10847723003, "name": "Curtin University", "priority": 1901, "external_id": null}, {"id": 10847724003, "name": "National University of Ireland, Galway", "priority": 1902, "external_id": null}, {"id": 10847725003, "name": "Universidade Federal do Rio de Janeiro", "priority": 1903, "external_id": null}, {"id": 10847726003, "name": "University of Surrey", "priority": 1904, "external_id": null}, {"id": 10847727003, "name": "Hong Kong Baptist University", "priority": 1905, "external_id": null}, {"id": 10847728003, "name": "Ume\u00e5 University", "priority": 1906, "external_id": null}, {"id": 10847729003, "name": "Universit\u00e4t Innsbruck", "priority": 1907, "external_id": null}, {"id": 10847730003, "name": "RMIT University", "priority": 1908, "external_id": null}, {"id": 10847731003, "name": "University of Eastern Finland", "priority": 1909, "external_id": null}, {"id": 10847732003, "name": "Christian-Albrechts-Universit\u00e4t zu Kiel", "priority": 1910, "external_id": null}, {"id": 10847733003, "name": "Indian Institute of Technology Kanpur (IITK)", "priority": 1911, "external_id": null}, {"id": 10847734003, "name": "National Yang Ming University", "priority": 1912, "external_id": null}, {"id": 10847735003, "name": "Johannes Gutenberg Universit\u00e4t Mainz", "priority": 1913, "external_id": null}, {"id": 10847736003, "name": "The University of Newcastle", "priority": 1914, "external_id": null}, {"id": 10847737003, "name": "Al-Farabi Kazakh National University", "priority": 1915, "external_id": null}, {"id": 10847738003, "name": "\u00c9cole des Ponts ParisTech", "priority": 1916, "external_id": null}, {"id": 10847739003, "name": "University of Jyv\u00e4skyl\u00e4", "priority": 1917, "external_id": null}, {"id": 10847740003, "name": "L.N. Gumilyov Eurasian National University", "priority": 1918, "external_id": null}, {"id": 10847741003, "name": "Kobe University", "priority": 1919, "external_id": null}, {"id": 10847742003, "name": "University of Tromso", "priority": 1920, "external_id": null}, {"id": 10847743003, "name": "Hiroshima University", "priority": 1921, "external_id": null}, {"id": 10847744003, "name": "Universit\u00e9 Bordeaux 1, Sciences Technologies", "priority": 1922, "external_id": null}, {"id": 10847745003, "name": "University of Indonesia", "priority": 1923, "external_id": null}, {"id": 10847746003, "name": "Universit\u00e4t Leipzig", "priority": 1924, "external_id": null}, {"id": 10847747003, "name": "University of Southern Denmark", "priority": 1925, "external_id": null}, {"id": 10847748003, "name": "Indian Institute of Technology Madras (IITM)", "priority": 1926, "external_id": null}, {"id": 10847749003, "name": "University of The Witwatersrand", "priority": 1927, "external_id": null}, {"id": 10847750003, "name": "University of Navarra", "priority": 1928, "external_id": null}, {"id": 10847751003, "name": "Universidad Austral - Argentina", "priority": 1929, "external_id": null}, {"id": 10847752003, "name": "Universidad Carlos III de Madrid", "priority": 1930, "external_id": null}, {"id": 10847753003, "name": "Universit\u00e0\u00a1 degli Studi di Roma - Tor Vergata", "priority": 1931, "external_id": null}, {"id": 10847754003, "name": "Pontificia Universidad Cat\u00f3lica Argentina Santa Mar\u00eda de los Buenos Aires", "priority": 1932, "external_id": null}, {"id": 10847755003, "name": "UCA", "priority": 1933, "external_id": null}, {"id": 10847756003, "name": "Julius-Maximilians-Universit\u00e4t W\u00fcrzburg", "priority": 1934, "external_id": null}, {"id": 10847757003, "name": "Universidad Nacional de Colombia", "priority": 1935, "external_id": null}, {"id": 10847758003, "name": "Laval University", "priority": 1936, "external_id": null}, {"id": 10847759003, "name": "Ben Gurion University of The Negev", "priority": 1937, "external_id": null}, {"id": 10847760003, "name": "Link\u00f6ping University", "priority": 1938, "external_id": null}, {"id": 10847761003, "name": "Aalborg University", "priority": 1939, "external_id": null}, {"id": 10847762003, "name": "Bauman Moscow State Technical University", "priority": 1940, "external_id": null}, {"id": 10847763003, "name": "Ecole Normale Sup\u00e9rieure de Cachan", "priority": 1941, "external_id": null}, {"id": 10847764003, "name": "SOAS - School of Oriental and African Studies, University of London", "priority": 1942, "external_id": null}, {"id": 10847765003, "name": "University of Essex", "priority": 1943, "external_id": null}, {"id": 10847766003, "name": "University of Warsaw", "priority": 1944, "external_id": null}, {"id": 10847767003, "name": "Griffith University", "priority": 1945, "external_id": null}, {"id": 10847768003, "name": "University of South Australia", "priority": 1946, "external_id": null}, {"id": 10847769003, "name": "Massey University", "priority": 1947, "external_id": null}, {"id": 10847770003, "name": "University of Porto", "priority": 1948, "external_id": null}, {"id": 10847771003, "name": "Universitat Polit\u00e8cnica de Catalunya", "priority": 1949, "external_id": null}, {"id": 10847772003, "name": "Indian Institute of Technology Kharagpur (IITKGP)", "priority": 1950, "external_id": null}, {"id": 10847773003, "name": "City University London", "priority": 1951, "external_id": null}, {"id": 10847774003, "name": "Dublin City University", "priority": 1952, "external_id": null}, {"id": 10847775003, "name": "Pontificia Universidad Javeriana", "priority": 1953, "external_id": null}, {"id": 10847776003, "name": "James Cook University", "priority": 1954, "external_id": null}, {"id": 10847777003, "name": "Novosibirsk State University", "priority": 1955, "external_id": null}, {"id": 10847778003, "name": "Universidade Nova de Lisboa", "priority": 1956, "external_id": null}, {"id": 10847779003, "name": "Universit\u00e9 Aix-Marseille", "priority": 1957, "external_id": null}, {"id": 10847780003, "name": "Universiti Sains Malaysia (USM)", "priority": 1958, "external_id": null}, {"id": 10847781003, "name": "Universiti Teknologi Malaysia (UTM)", "priority": 1959, "external_id": null}, {"id": 10847782003, "name": "Universit\u00e9 Paris Dauphine", "priority": 1960, "external_id": null}, {"id": 10847783003, "name": "University of Coimbra", "priority": 1961, "external_id": null}, {"id": 10847784003, "name": "Brunel University", "priority": 1962, "external_id": null}, {"id": 10847785003, "name": "King Abdul Aziz University (KAU)", "priority": 1963, "external_id": null}, {"id": 10847786003, "name": "Ewha Womans University", "priority": 1964, "external_id": null}, {"id": 10847787003, "name": "Nankai University", "priority": 1965, "external_id": null}, {"id": 10847788003, "name": "Taipei Medical University", "priority": 1966, "external_id": null}, {"id": 10847789003, "name": "Universit\u00e4t Jena", "priority": 1967, "external_id": null}, {"id": 10847790003, "name": "Ruhr-Universit\u00e4t Bochum", "priority": 1968, "external_id": null}, {"id": 10847791003, "name": "Heriot-Watt University", "priority": 1969, "external_id": null}, {"id": 10847792003, "name": "Politecnico di Torino", "priority": 1970, "external_id": null}, {"id": 10847793003, "name": "Universit\u00e4t Bremen", "priority": 1971, "external_id": null}, {"id": 10847794003, "name": "Xi'an Jiaotong University", "priority": 1972, "external_id": null}, {"id": 10847795003, "name": "Birkbeck College, University of London", "priority": 1973, "external_id": null}, {"id": 10847796003, "name": "Oxford Brookes University", "priority": 1974, "external_id": null}, {"id": 10847797003, "name": "Jagiellonian University", "priority": 1975, "external_id": null}, {"id": 10847798003, "name": "University of Tampere", "priority": 1976, "external_id": null}, {"id": 10847799003, "name": "University of Florence", "priority": 1977, "external_id": null}, {"id": 10847800003, "name": "Deakin University", "priority": 1978, "external_id": null}, {"id": 10847801003, "name": "University of the Philippines", "priority": 1979, "external_id": null}, {"id": 10847802003, "name": "Universitat Polit\u00e8cnica de Val\u00e8ncia", "priority": 1980, "external_id": null}, {"id": 10847803003, "name": "Sun Yat-sen University", "priority": 1981, "external_id": null}, {"id": 10847804003, "name": "Universit\u00e9 Montpellier 2, Sciences et Techniques du Languedoc", "priority": 1982, "external_id": null}, {"id": 10847805003, "name": "Moscow State Institute of International Relations (MGIMO-University)", "priority": 1983, "external_id": null}, {"id": 10847806003, "name": "Stellenbosch University", "priority": 1984, "external_id": null}, {"id": 10847807003, "name": "Polit\u00e9cnica de Madrid", "priority": 1985, "external_id": null}, {"id": 10847808003, "name": "Instituto Tecnol\u00f3gico de Buenos Aires (ITBA)", "priority": 1986, "external_id": null}, {"id": 10847809003, "name": "La Trobe University", "priority": 1987, "external_id": null}, {"id": 10847810003, "name": "Universit\u00e9 Paul Sabatier Toulouse III", "priority": 1988, "external_id": null}, {"id": 10847811003, "name": "Karl-Franzens-Universit\u00e4t Graz", "priority": 1989, "external_id": null}, {"id": 10847812003, "name": "Universit\u00e4t D\u00fcsseldorf", "priority": 1990, "external_id": null}, {"id": 10847813003, "name": "University of Naples - Federico Ii", "priority": 1991, "external_id": null}, {"id": 10847814003, "name": "Aston University", "priority": 1992, "external_id": null}, {"id": 10847815003, "name": "University of Turin", "priority": 1993, "external_id": null}, {"id": 10847816003, "name": "Beihang University (former BUAA)", "priority": 1994, "external_id": null}, {"id": 10847817003, "name": "Indian Institute of Technology Roorkee (IITR)", "priority": 1995, "external_id": null}, {"id": 10847818003, "name": "National Central University", "priority": 1996, "external_id": null}, {"id": 10847819003, "name": "Sogang University", "priority": 1997, "external_id": null}, {"id": 10847820003, "name": "Universit\u00e4t Regensburg", "priority": 1998, "external_id": null}, {"id": 10847821003, "name": "Universit\u00e9 Lille 1, Sciences et Technologie", "priority": 1999, "external_id": null}, {"id": 10847822003, "name": "University of Tasmania", "priority": 2000, "external_id": null}, {"id": 10847823003, "name": "University of Waikato", "priority": 2001, "external_id": null}, {"id": 10847824003, "name": "Wuhan University", "priority": 2002, "external_id": null}, {"id": 10847825003, "name": "National Taiwan University of Science And Technology", "priority": 2003, "external_id": null}, {"id": 10847826003, "name": "Universidade Federal de S\u00e3o Paulo (UNIFESP)", "priority": 2004, "external_id": null}, {"id": 10847827003, "name": "Universit\u00e0 degli Studi di Pavia", "priority": 2005, "external_id": null}, {"id": 10847828003, "name": "Universit\u00e4t Bayreuth", "priority": 2006, "external_id": null}, {"id": 10847829003, "name": "Universit\u00e9 Claude Bernard Lyon 1", "priority": 2007, "external_id": null}, {"id": 10847830003, "name": "Universit\u00e9 du Qu\u00e9bec", "priority": 2008, "external_id": null}, {"id": 10847831003, "name": "Universiti Putra Malaysia (UPM)", "priority": 2009, "external_id": null}, {"id": 10847832003, "name": "University of Kent", "priority": 2010, "external_id": null}, {"id": 10847833003, "name": "University of St Gallen (HSG)", "priority": 2011, "external_id": null}, {"id": 10847834003, "name": "Bond University", "priority": 2012, "external_id": null}, {"id": 10847835003, "name": "United Arab Emirates University", "priority": 2013, "external_id": null}, {"id": 10847836003, "name": "Universidad de San Andr\u00c3\u00a9s", "priority": 2014, "external_id": null}, {"id": 10847837003, "name": "Universidad Nacional de La Plata", "priority": 2015, "external_id": null}, {"id": 10847838003, "name": "Universit\u00e4t des Saarlandes", "priority": 2016, "external_id": null}, {"id": 10847839003, "name": "American University of Sharjah (AUS)", "priority": 2017, "external_id": null}, {"id": 10847840003, "name": "Bilkent University", "priority": 2018, "external_id": null}, {"id": 10847841003, "name": "Flinders University", "priority": 2019, "external_id": null}, {"id": 10847842003, "name": "Hankuk (Korea) University of Foreign Studies", "priority": 2020, "external_id": null}, {"id": 10847843003, "name": "Middle East Technical University", "priority": 2021, "external_id": null}, {"id": 10847844003, "name": "Philipps-Universit\u00e4t Marburg", "priority": 2022, "external_id": null}, {"id": 10847845003, "name": "Swansea University", "priority": 2023, "external_id": null}, {"id": 10847846003, "name": "Tampere University of Technology", "priority": 2024, "external_id": null}, {"id": 10847847003, "name": "Universit\u00e4t Bielefeld", "priority": 2025, "external_id": null}, {"id": 10847848003, "name": "University of Manitoba", "priority": 2026, "external_id": null}, {"id": 10847849003, "name": "Chiba University", "priority": 2027, "external_id": null}, {"id": 10847850003, "name": "Moscow Institute of Physics and Technology State University", "priority": 2028, "external_id": null}, {"id": 10847851003, "name": "Tallinn University of Technology", "priority": 2029, "external_id": null}, {"id": 10847852003, "name": "Taras Shevchenko National University of Kyiv", "priority": 2030, "external_id": null}, {"id": 10847853003, "name": "Tokyo University of Science", "priority": 2031, "external_id": null}, {"id": 10847854003, "name": "University of Salamanca", "priority": 2032, "external_id": null}, {"id": 10847855003, "name": "University of Trento", "priority": 2033, "external_id": null}, {"id": 10847856003, "name": "Universit\u00e9 de Sherbrooke", "priority": 2034, "external_id": null}, {"id": 10847857003, "name": "Universit\u00e9 Panth\u00e9on-Assas (Paris 2)", "priority": 2035, "external_id": null}, {"id": 10847858003, "name": "University of Delhi", "priority": 2036, "external_id": null}, {"id": 10847859003, "name": "Abo Akademi University", "priority": 2037, "external_id": null}, {"id": 10847860003, "name": "Czech Technical University In Prague", "priority": 2038, "external_id": null}, {"id": 10847861003, "name": "Leibniz Universit\u00e4t Hannover", "priority": 2039, "external_id": null}, {"id": 10847862003, "name": "Pusan National University", "priority": 2040, "external_id": null}, {"id": 10847863003, "name": "Shanghai University", "priority": 2041, "external_id": null}, {"id": 10847864003, "name": "St. Petersburg State Politechnical University", "priority": 2042, "external_id": null}, {"id": 10847865003, "name": "Universit\u00e0 Cattolica del Sacro Cuore", "priority": 2043, "external_id": null}, {"id": 10847866003, "name": "University of Genoa", "priority": 2044, "external_id": null}, {"id": 10847867003, "name": "Bandung Institute of Technology (ITB)", "priority": 2045, "external_id": null}, {"id": 10847868003, "name": "Bogazici University", "priority": 2046, "external_id": null}, {"id": 10847869003, "name": "Goldsmiths, University of London", "priority": 2047, "external_id": null}, {"id": 10847870003, "name": "National Sun Yat-sen University", "priority": 2048, "external_id": null}, {"id": 10847871003, "name": "Renmin (People\u2019s) University of China", "priority": 2049, "external_id": null}, {"id": 10847872003, "name": "Universidad de Costa Rica", "priority": 2050, "external_id": null}, {"id": 10847873003, "name": "Universidad de Santiago de Chile - USACH", "priority": 2051, "external_id": null}, {"id": 10847874003, "name": "University of Tartu", "priority": 2052, "external_id": null}, {"id": 10847875003, "name": "Aristotle University of Thessaloniki", "priority": 2053, "external_id": null}, {"id": 10847876003, "name": "Auckland University of Technology", "priority": 2054, "external_id": null}, {"id": 10847877003, "name": "Bangor University", "priority": 2055, "external_id": null}, {"id": 10847878003, "name": "Charles Darwin University", "priority": 2056, "external_id": null}, {"id": 10847879003, "name": "Kingston University, London", "priority": 2057, "external_id": null}, {"id": 10847880003, "name": "Universitat de Valencia", "priority": 2058, "external_id": null}, {"id": 10847881003, "name": "Universit\u00e9 Montpellier 1", "priority": 2059, "external_id": null}, {"id": 10847882003, "name": "University of Pretoria", "priority": 2060, "external_id": null}, {"id": 10847883003, "name": "Lincoln University", "priority": 2061, "external_id": null}, {"id": 10847884003, "name": "National Taiwan Normal University", "priority": 2062, "external_id": null}, {"id": 10847885003, "name": "National University of Sciences And Technology (NUST) Islamabad", "priority": 2063, "external_id": null}, {"id": 10847886003, "name": "Swinburne University of Technology", "priority": 2064, "external_id": null}, {"id": 10847887003, "name": "Tongji University", "priority": 2065, "external_id": null}, {"id": 10847888003, "name": "Universidad de Zaragoza", "priority": 2066, "external_id": null}, {"id": 10847889003, "name": "Universidade Federal de Minas Gerais", "priority": 2067, "external_id": null}, {"id": 10847890003, "name": "Universit\u00e4t Duisburg-Essen", "priority": 2068, "external_id": null}, {"id": 10847891003, "name": "Al-Imam Mohamed Ibn Saud Islamic University", "priority": 2069, "external_id": null}, {"id": 10847892003, "name": "Harbin Institute of Technology", "priority": 2070, "external_id": null}, {"id": 10847893003, "name": "People's Friendship University of Russia", "priority": 2071, "external_id": null}, {"id": 10847894003, "name": "Universidade Estadual PaulistaJ\u00falio de Mesquita Filho' (UNESP)", "priority": 2072, "external_id": null}, {"id": 10847895003, "name": "Universit\u00e9 Nice Sophia-Antipolis", "priority": 2073, "external_id": null}, {"id": 10847896003, "name": "University of Crete", "priority": 2074, "external_id": null}, {"id": 10847897003, "name": "University of Milano-Bicocca", "priority": 2075, "external_id": null}, {"id": 10847898003, "name": "Ateneo de Manila University", "priority": 2076, "external_id": null}, {"id": 10847899003, "name": "Beijing Institute of Technology", "priority": 2077, "external_id": null}, {"id": 10847900003, "name": "Chang Gung University", "priority": 2078, "external_id": null}, {"id": 10847901003, "name": "hung-Ang University", "priority": 2079, "external_id": null}, {"id": 10847902003, "name": "Dublin Institute of Technology", "priority": 2080, "external_id": null}, {"id": 10847903003, "name": "Huazhong University of Science and Technology", "priority": 2081, "external_id": null}, {"id": 10847904003, "name": "International Islamic University Malaysia (IIUM)", "priority": 2082, "external_id": null}, {"id": 10847905003, "name": "Johannes Kepler University Linz", "priority": 2083, "external_id": null}, {"id": 10847906003, "name": "Justus-Liebig-Universit\u00e4t Gie\u00dfen", "priority": 2084, "external_id": null}, {"id": 10847907003, "name": "Kanazawa University", "priority": 2085, "external_id": null}, {"id": 10847908003, "name": "Keele University", "priority": 2086, "external_id": null}, {"id": 10847909003, "name": "Koc University", "priority": 2087, "external_id": null}, {"id": 10847910003, "name": "National and Kapodistrian University of Athens", "priority": 2088, "external_id": null}, {"id": 10847911003, "name": "National Research University \u2013 Higher School of Economics (HSE)", "priority": 2089, "external_id": null}, {"id": 10847912003, "name": "National Technical University of Athens", "priority": 2090, "external_id": null}, {"id": 10847913003, "name": "Okayama University", "priority": 2091, "external_id": null}, {"id": 10847914003, "name": "Sabanci University", "priority": 2092, "external_id": null}, {"id": 10847915003, "name": "Southeast University", "priority": 2093, "external_id": null}, {"id": 10847916003, "name": "Sultan Qaboos University", "priority": 2094, "external_id": null}, {"id": 10847917003, "name": "Technische Universit\u00e4t Braunschweig", "priority": 2095, "external_id": null}, {"id": 10847918003, "name": "Technische Universit\u00e4t Dortmund", "priority": 2096, "external_id": null}, {"id": 10847919003, "name": "The Catholic University of Korea", "priority": 2097, "external_id": null}, {"id": 10847920003, "name": "Tianjin University", "priority": 2098, "external_id": null}, {"id": 10847921003, "name": "Tokyo Metropolitan University", "priority": 2099, "external_id": null}, {"id": 10847922003, "name": "Universidad de Antioquia", "priority": 2100, "external_id": null}, {"id": 10847923003, "name": "University of Granada", "priority": 2101, "external_id": null}, {"id": 10847924003, "name": "Universidad de Palermo", "priority": 2102, "external_id": null}, {"id": 10847925003, "name": "Universidad Nacional de C\u00f3rdoba", "priority": 2103, "external_id": null}, {"id": 10847926003, "name": "Universidade de Santiago de Compostela", "priority": 2104, "external_id": null}, {"id": 10847927003, "name": "Universidade Federal do Rio Grande Do Sul", "priority": 2105, "external_id": null}, {"id": 10847928003, "name": "University of Siena", "priority": 2106, "external_id": null}, {"id": 10847929003, "name": "University of Trieste", "priority": 2107, "external_id": null}, {"id": 10847930003, "name": "Universitas Gadjah Mada", "priority": 2108, "external_id": null}, {"id": 10847931003, "name": "Universit\u00e9 de Lorraine", "priority": 2109, "external_id": null}, {"id": 10847932003, "name": "Universit\u00e9 de Rennes 1", "priority": 2110, "external_id": null}, {"id": 10847933003, "name": "University of Bradford", "priority": 2111, "external_id": null}, {"id": 10847934003, "name": "University of Hull", "priority": 2112, "external_id": null}, {"id": 10847935003, "name": "University of Kwazulu-Natal", "priority": 2113, "external_id": null}, {"id": 10847936003, "name": "University of Limerick", "priority": 2114, "external_id": null}, {"id": 10847937003, "name": "University of Stirling", "priority": 2115, "external_id": null}, {"id": 10847938003, "name": "University of Szeged", "priority": 2116, "external_id": null}, {"id": 10847939003, "name": "Ural Federal University", "priority": 2117, "external_id": null}, {"id": 10847940003, "name": "Xiamen University", "priority": 2118, "external_id": null}, {"id": 10847941003, "name": "Yokohama City University", "priority": 2119, "external_id": null}, {"id": 10847942003, "name": "Aberystwyth University", "priority": 2120, "external_id": null}, {"id": 10847943003, "name": "Belarus State University", "priority": 2121, "external_id": null}, {"id": 10847944003, "name": "Cairo University", "priority": 2122, "external_id": null}, {"id": 10847945003, "name": "Chiang Mai University", "priority": 2123, "external_id": null}, {"id": 10847946003, "name": "Chonbuk National University", "priority": 2124, "external_id": null}, {"id": 10847947003, "name": "E\u00f6tv\u00f6s Lor\u00e1nd University", "priority": 2125, "external_id": null}, {"id": 10847948003, "name": "Inha University", "priority": 2126, "external_id": null}, {"id": 10847949003, "name": "Instituto Polit\u00e9cnico Nacional (IPN)", "priority": 2127, "external_id": null}, {"id": 10847950003, "name": "Istanbul Technical University", "priority": 2128, "external_id": null}, {"id": 10847951003, "name": "Kumamoto University", "priority": 2129, "external_id": null}, {"id": 10847952003, "name": "Kyungpook National University", "priority": 2130, "external_id": null}, {"id": 10847953003, "name": "Lingnan University (Hong Kong)", "priority": 2131, "external_id": null}, {"id": 10847954003, "name": "Masaryk University", "priority": 2132, "external_id": null}, {"id": 10847955003, "name": "Murdoch University", "priority": 2133, "external_id": null}, {"id": 10847956003, "name": "Nagasaki University", "priority": 2134, "external_id": null}, {"id": 10847957003, "name": "National Chung Hsing University", "priority": 2135, "external_id": null}, {"id": 10847958003, "name": "National Taipei University of Technology", "priority": 2136, "external_id": null}, {"id": 10847959003, "name": "National University of Ireland Maynooth", "priority": 2137, "external_id": null}, {"id": 10847960003, "name": "Osaka City University", "priority": 2138, "external_id": null}, {"id": 10847961003, "name": "Pontificia Universidad Cat\u00f3lica del Per\u00fa", "priority": 2139, "external_id": null}, {"id": 10847962003, "name": "Pontificia Universidade Cat\u00f3lica de S\u00e3o Paulo (PUC -SP)", "priority": 2140, "external_id": null}, {"id": 10847963003, "name": "Pontificia Universidade Cat\u00f3lica do Rio de Janeiro (PUC - Rio)", "priority": 2141, "external_id": null}, {"id": 10847964003, "name": "Qatar University", "priority": 2142, "external_id": null}, {"id": 10847965003, "name": "Rhodes University", "priority": 2143, "external_id": null}, {"id": 10847966003, "name": "Tokyo University of Agriculture and Technology", "priority": 2144, "external_id": null}, {"id": 10847967003, "name": "Tomsk Polytechnic University", "priority": 2145, "external_id": null}, {"id": 10847968003, "name": "Tomsk State University", "priority": 2146, "external_id": null}, {"id": 10847969003, "name": "Umm Al-Qura University", "priority": 2147, "external_id": null}, {"id": 10847970003, "name": "Universidad Cat\u00f3lica Andr\u00e9s Bello - UCAB", "priority": 2148, "external_id": null}, {"id": 10847971003, "name": "Universidad Central de Venezuela - UCV", "priority": 2149, "external_id": null}, {"id": 10847972003, "name": "Universidad de Belgrano", "priority": 2150, "external_id": null}, {"id": 10847973003, "name": "Universidad de Concepci\u00f3n", "priority": 2151, "external_id": null}, {"id": 10847974003, "name": "Universidad de Sevilla", "priority": 2152, "external_id": null}, {"id": 10847975003, "name": "Universidade Catolica Portuguesa, Lisboa", "priority": 2153, "external_id": null}, {"id": 10847976003, "name": "Universidade de Brasilia (UnB)", "priority": 2154, "external_id": null}, {"id": 10847977003, "name": "University of Lisbon", "priority": 2155, "external_id": null}, {"id": 10847978003, "name": "University of Ljubljana", "priority": 2156, "external_id": null}, {"id": 10847979003, "name": "University of Seoul", "priority": 2157, "external_id": null}, {"id": 10847980003, "name": "Abu Dhabi University", "priority": 2158, "external_id": null}, {"id": 10847981003, "name": "Ain Shams University", "priority": 2159, "external_id": null}, {"id": 10847982003, "name": "Ajou University", "priority": 2160, "external_id": null}, {"id": 10847983003, "name": "De La Salle University", "priority": 2161, "external_id": null}, {"id": 10847984003, "name": "Dongguk University", "priority": 2162, "external_id": null}, {"id": 10847985003, "name": "Gifu University", "priority": 2163, "external_id": null}, {"id": 10847986003, "name": "Hacettepe University", "priority": 2164, "external_id": null}, {"id": 10847987003, "name": "Indian Institute of Technology Guwahati (IITG)", "priority": 2165, "external_id": null}, {"id": 10847988003, "name": "Jilin University", "priority": 2166, "external_id": null}, {"id": 10847989003, "name": "Kazan Federal University", "priority": 2167, "external_id": null}, {"id": 10847990003, "name": "King Khalid University", "priority": 2168, "external_id": null}, {"id": 10847991003, "name": "Martin-Luther-Universit\u00e4t Halle-Wittenberg", "priority": 2169, "external_id": null}, {"id": 10847992003, "name": "National Chengchi University", "priority": 2170, "external_id": null}, {"id": 10847993003, "name": "National Technical University of UkraineKyiv Polytechnic Institute'", "priority": 2171, "external_id": null}, {"id": 10847994003, "name": "Niigata University", "priority": 2172, "external_id": null}, {"id": 10847995003, "name": "Osaka Prefecture University", "priority": 2173, "external_id": null}, {"id": 10847996003, "name": "Paris Lodron University of Salzburg", "priority": 2174, "external_id": null}, {"id": 10847997003, "name": "Sharif University of Technology", "priority": 2175, "external_id": null}, {"id": 10847998003, "name": "Southern Federal University", "priority": 2176, "external_id": null}, {"id": 10847999003, "name": "Thammasat University", "priority": 2177, "external_id": null}, {"id": 10848000003, "name": "Universidad de Guadalajara (UDG)", "priority": 2178, "external_id": null}, {"id": 10848001003, "name": "Universidad de la Rep\u00fablica (UdelaR)", "priority": 2179, "external_id": null}, {"id": 10848002003, "name": "Universidad Iberoamericana (UIA)", "priority": 2180, "external_id": null}, {"id": 10848003003, "name": "Universidad Torcuato Di Tella", "priority": 2181, "external_id": null}, {"id": 10848004003, "name": "Universidade Federal da Bahia", "priority": 2182, "external_id": null}, {"id": 10848005003, "name": "Universidade Federal de S\u00e3o Carlos", "priority": 2183, "external_id": null}, {"id": 10848006003, "name": "Universidade Federal de Vi\u00e7osa", "priority": 2184, "external_id": null}, {"id": 10848007003, "name": "Perugia University", "priority": 2185, "external_id": null}, {"id": 10848008003, "name": "Universit\u00e9 de Nantes", "priority": 2186, "external_id": null}, {"id": 10848009003, "name": "Universit\u00e9 Saint-Joseph de Beyrouth", "priority": 2187, "external_id": null}, {"id": 10848010003, "name": "University of Canberra", "priority": 2188, "external_id": null}, {"id": 10848011003, "name": "University of Debrecen", "priority": 2189, "external_id": null}, {"id": 10848012003, "name": "University of Johannesburg", "priority": 2190, "external_id": null}, {"id": 10848013003, "name": "University of Mumbai", "priority": 2191, "external_id": null}, {"id": 10848014003, "name": "University of Patras", "priority": 2192, "external_id": null}, {"id": 10848015003, "name": "University of Tehran", "priority": 2193, "external_id": null}, {"id": 10848016003, "name": "University of Ulsan", "priority": 2194, "external_id": null}, {"id": 10848017003, "name": "University of Ulster", "priority": 2195, "external_id": null}, {"id": 10848018003, "name": "University of Zagreb", "priority": 2196, "external_id": null}, {"id": 10848019003, "name": "Vilnius University", "priority": 2197, "external_id": null}, {"id": 10848020003, "name": "Warsaw University of Technology", "priority": 2198, "external_id": null}, {"id": 10848021003, "name": "Al Azhar University", "priority": 2199, "external_id": null}, {"id": 10848022003, "name": "Bar-Ilan University", "priority": 2200, "external_id": null}, {"id": 10848023003, "name": "Brno University of Technology", "priority": 2201, "external_id": null}, {"id": 10848024003, "name": "Chonnam National University", "priority": 2202, "external_id": null}, {"id": 10848025003, "name": "Chungnam National University", "priority": 2203, "external_id": null}, {"id": 10848026003, "name": "Corvinus University of Budapest", "priority": 2204, "external_id": null}, {"id": 10848027003, "name": "Gunma University", "priority": 2205, "external_id": null}, {"id": 10848028003, "name": "Hallym University", "priority": 2206, "external_id": null}, {"id": 10848029003, "name": "Instituto Tecnol\u00f3gico Autonomo de M\u00e9xico (ITAM)", "priority": 2207, "external_id": null}, {"id": 10848030003, "name": "Istanbul University", "priority": 2208, "external_id": null}, {"id": 10848031003, "name": "Jordan University of Science & Technology", "priority": 2209, "external_id": null}, {"id": 10848032003, "name": "Kasetsart University", "priority": 2210, "external_id": null}, {"id": 10848033003, "name": "Kazakh-British Technical University", "priority": 2211, "external_id": null}, {"id": 10848034003, "name": "Khazar University", "priority": 2212, "external_id": null}, {"id": 10848035003, "name": "London Metropolitan University", "priority": 2213, "external_id": null}, {"id": 10848036003, "name": "Middlesex University", "priority": 2214, "external_id": null}, {"id": 10848037003, "name": "Universidad Industrial de Santander", "priority": 2215, "external_id": null}, {"id": 10848038003, "name": "Pontificia Universidad Cat\u00f3lica de Valpara\u00edso", "priority": 2216, "external_id": null}, {"id": 10848039003, "name": "Pontificia Universidade Cat\u00f3lica do Rio Grande do Sul", "priority": 2217, "external_id": null}, {"id": 10848040003, "name": "Qafqaz University", "priority": 2218, "external_id": null}, {"id": 10848041003, "name": "Ritsumeikan University", "priority": 2219, "external_id": null}, {"id": 10848042003, "name": "Shandong University", "priority": 2220, "external_id": null}, {"id": 10848043003, "name": "University of St. Kliment Ohridski", "priority": 2221, "external_id": null}, {"id": 10848044003, "name": "South Kazakhstan State University (SKSU)", "priority": 2222, "external_id": null}, {"id": 10848045003, "name": "Universidad Adolfo Ib\u00e1\u00f1ez", "priority": 2223, "external_id": null}, {"id": 10848046003, "name": "Universidad Aut\u00f3noma del Estado de M\u00e9xico", "priority": 2224, "external_id": null}, {"id": 10848047003, "name": "Universidad Aut\u00f3noma Metropolitana (UAM)", "priority": 2225, "external_id": null}, {"id": 10848048003, "name": "Universidad de Alcal\u00e1", "priority": 2226, "external_id": null}, {"id": 10848049003, "name": "Universidad Nacional Costa Rica", "priority": 2227, "external_id": null}, {"id": 10848050003, "name": "Universidad Nacional de Mar del Plata", "priority": 2228, "external_id": null}, {"id": 10848051003, "name": "Universidad Peruana Cayetano Heredia", "priority": 2229, "external_id": null}, {"id": 10848052003, "name": "Universidad Sim\u00f3n Bol\u00edvar Venezuela", "priority": 2230, "external_id": null}, {"id": 10848053003, "name": "Universidade Federal de Santa Catarina", "priority": 2231, "external_id": null}, {"id": 10848054003, "name": "Universidade Federal do Paran\u00e1 (UFPR)", "priority": 2232, "external_id": null}, {"id": 10848055003, "name": "Universidade Federal Fluminense", "priority": 2233, "external_id": null}, {"id": 10848056003, "name": "University of Modena", "priority": 2234, "external_id": null}, {"id": 10848057003, "name": "Universit\u00e9 Lumi\u00e8re Lyon 2", "priority": 2235, "external_id": null}, {"id": 10848058003, "name": "Universit\u00e9 Toulouse 1, Capitole", "priority": 2236, "external_id": null}, {"id": 10848059003, "name": "University of Economics Prague", "priority": 2237, "external_id": null}, {"id": 10848060003, "name": "University of Hertfordshire", "priority": 2238, "external_id": null}, {"id": 10848061003, "name": "University of Plymouth", "priority": 2239, "external_id": null}, {"id": 10848062003, "name": "University of Salford", "priority": 2240, "external_id": null}, {"id": 10848063003, "name": "University of Science and Technology Beijing", "priority": 2241, "external_id": null}, {"id": 10848064003, "name": "University of Western Sydney", "priority": 2242, "external_id": null}, {"id": 10848065003, "name": "Yamaguchi University", "priority": 2243, "external_id": null}, {"id": 10848066003, "name": "Yokohama National University", "priority": 2244, "external_id": null}, {"id": 10848067003, "name": "Airlangga University", "priority": 2245, "external_id": null}, {"id": 10848068003, "name": "Alexandria University", "priority": 2246, "external_id": null}, {"id": 10848069003, "name": "Alexandru Ioan Cuza University", "priority": 2247, "external_id": null}, {"id": 10848070003, "name": "Alpen-Adria-Universit\u00e4t Klagenfurt", "priority": 2248, "external_id": null}, {"id": 10848071003, "name": "Aoyama Gakuin University", "priority": 2249, "external_id": null}, {"id": 10848072003, "name": "Athens University of Economy And Business", "priority": 2250, "external_id": null}, {"id": 10848073003, "name": "Babes-Bolyai University", "priority": 2251, "external_id": null}, {"id": 10848074003, "name": "Baku State University", "priority": 2252, "external_id": null}, {"id": 10848075003, "name": "Belarusian National Technical University", "priority": 2253, "external_id": null}, {"id": 10848076003, "name": "Benem\u00e9rita Universidad Aut\u00f3noma de Puebla", "priority": 2254, "external_id": null}, {"id": 10848077003, "name": "Bogor Agricultural University", "priority": 2255, "external_id": null}, {"id": 10848078003, "name": "Coventry University", "priority": 2256, "external_id": null}, {"id": 10848079003, "name": "Cukurova University", "priority": 2257, "external_id": null}, {"id": 10848080003, "name": "Diponegoro University", "priority": 2258, "external_id": null}, {"id": 10848081003, "name": "Donetsk National University", "priority": 2259, "external_id": null}, {"id": 10848082003, "name": "Doshisha University", "priority": 2260, "external_id": null}, {"id": 10848083003, "name": "E.A.Buketov Karaganda State University", "priority": 2261, "external_id": null}, {"id": 10848084003, "name": "Far Eastern Federal University", "priority": 2262, "external_id": null}, {"id": 10848085003, "name": "Fu Jen Catholic University", "priority": 2263, "external_id": null}, {"id": 10848086003, "name": "Kagoshima University", "priority": 2264, "external_id": null}, {"id": 10848087003, "name": "Kaunas University of Technology", "priority": 2265, "external_id": null}, {"id": 10848088003, "name": "Kazakh Ablai khan University of International Relations and World Languages", "priority": 2266, "external_id": null}, {"id": 10848089003, "name": "Kazakh National Pedagogical University Abai", "priority": 2267, "external_id": null}, {"id": 10848090003, "name": "Kazakh National Technical University", "priority": 2268, "external_id": null}, {"id": 10848091003, "name": "Khon Kaen University", "priority": 2269, "external_id": null}, {"id": 10848092003, "name": "King Faisal University", "priority": 2270, "external_id": null}, {"id": 10848093003, "name": "King Mongkut''s University of Technology Thonburi", "priority": 2271, "external_id": null}, {"id": 10848094003, "name": "Kuwait University", "priority": 2272, "external_id": null}, {"id": 10848095003, "name": "Lodz University", "priority": 2273, "external_id": null}, {"id": 10848096003, "name": "Manchester Metropolitan University", "priority": 2274, "external_id": null}, {"id": 10848097003, "name": "Lobachevsky State University of Nizhni Novgorod", "priority": 2275, "external_id": null}, {"id": 10848098003, "name": "National Technical UniversityKharkiv Polytechnic Institute'", "priority": 2276, "external_id": null}, {"id": 10848099003, "name": "Nicolaus Copernicus University", "priority": 2277, "external_id": null}, {"id": 10848100003, "name": "Northumbria University at Newcastle", "priority": 2278, "external_id": null}, {"id": 10848101003, "name": "Nottingham Trent University", "priority": 2279, "external_id": null}, {"id": 10848102003, "name": "Ochanomizu University", "priority": 2280, "external_id": null}, {"id": 10848103003, "name": "Plekhanov Russian University of Economics", "priority": 2281, "external_id": null}, {"id": 10848104003, "name": "Pontificia Universidad Catolica del Ecuador", "priority": 2282, "external_id": null}, {"id": 10848105003, "name": "Prince of Songkla University", "priority": 2283, "external_id": null}, {"id": 10848106003, "name": "S.Seifullin Kazakh Agro Technical University", "priority": 2284, "external_id": null}, {"id": 10848107003, "name": "Saitama University", "priority": 2285, "external_id": null}, {"id": 10848108003, "name": "Sepuluh Nopember Institute of Technology", "priority": 2286, "external_id": null}, {"id": 10848109003, "name": "Shinshu University", "priority": 2287, "external_id": null}, {"id": 10848110003, "name": "The Robert Gordon University", "priority": 2288, "external_id": null}, {"id": 10848111003, "name": "Tokai University", "priority": 2289, "external_id": null}, {"id": 10848112003, "name": "Universidad ANAHUAC", "priority": 2290, "external_id": null}, {"id": 10848113003, "name": "Universidad Austral de Chile", "priority": 2291, "external_id": null}, {"id": 10848114003, "name": "University Aut\u00f3noma de Nuevo Le\u00f3n (UANL)", "priority": 2292, "external_id": null}, {"id": 10848115003, "name": "Universidad de la Habana", "priority": 2293, "external_id": null}, {"id": 10848116003, "name": "Universidad de La Sabana", "priority": 2294, "external_id": null}, {"id": 10848117003, "name": "Universidad de las Am\u00e9ricas Puebla (UDLAP)", "priority": 2295, "external_id": null}, {"id": 10848118003, "name": "Universidad de los Andes M\u00e9rida", "priority": 2296, "external_id": null}, {"id": 10848119003, "name": "University of Murcia", "priority": 2297, "external_id": null}, {"id": 10848120003, "name": "Universidad de Puerto Rico", "priority": 2298, "external_id": null}, {"id": 10848121003, "name": "Universidad de San Francisco de Quito", "priority": 2299, "external_id": null}, {"id": 10848122003, "name": "Universidad de Talca", "priority": 2300, "external_id": null}, {"id": 10848123003, "name": "Universidad del Norte", "priority": 2301, "external_id": null}, {"id": 10848124003, "name": "Universidad del Rosario", "priority": 2302, "external_id": null}, {"id": 10848125003, "name": "Universidad del Valle", "priority": 2303, "external_id": null}, {"id": 10848126003, "name": "Universidad Nacional de Cuyo", "priority": 2304, "external_id": null}, {"id": 10848127003, "name": "Universidad Nacional de Rosario", "priority": 2305, "external_id": null}, {"id": 10848128003, "name": "Universidad Nacional de Tucum\u00e1n", "priority": 2306, "external_id": null}, {"id": 10848129003, "name": "Universidad Nacional del Sur", "priority": 2307, "external_id": null}, {"id": 10848130003, "name": "Universidad Nacional Mayor de San Marcos", "priority": 2308, "external_id": null}, {"id": 10848131003, "name": "Universidad T\u00e9cnica Federico Santa Mar\u00eda", "priority": 2309, "external_id": null}, {"id": 10848132003, "name": "Universidad Tecnol\u00f3gica Nacional (UTN)", "priority": 2310, "external_id": null}, {"id": 10848133003, "name": "Universidade do Estado do Rio de Janeiro (UERJ)", "priority": 2311, "external_id": null}, {"id": 10848134003, "name": "Universidade Estadual de Londrina (UEL)", "priority": 2312, "external_id": null}, {"id": 10848135003, "name": "Universidade Federal de Santa Maria", "priority": 2313, "external_id": null}, {"id": 10848136003, "name": "Universidade Federal do Cear\u00e1 (UFC)", "priority": 2314, "external_id": null}, {"id": 10848137003, "name": "Universidade Federal do Pernambuco", "priority": 2315, "external_id": null}, {"id": 10848138003, "name": "Universit\u00e0 Ca'' Foscari Venezia", "priority": 2316, "external_id": null}, {"id": 10848139003, "name": "Catania University", "priority": 2317, "external_id": null}, {"id": 10848140003, "name": "Universit\u00e0 degli Studi Roma Tre", "priority": 2318, "external_id": null}, {"id": 10848141003, "name": "Universit\u00e9 Charles-de-Gaulle Lille 3", "priority": 2319, "external_id": null}, {"id": 10848142003, "name": "Universit\u00e9 de Caen Basse-Normandie", "priority": 2320, "external_id": null}, {"id": 10848143003, "name": "Universit\u00e9 de Cergy-Pontoise", "priority": 2321, "external_id": null}, {"id": 10848144003, "name": "Universit\u00e9 de Poitiers", "priority": 2322, "external_id": null}, {"id": 10848145003, "name": "Universit\u00e9 Jean Moulin Lyon 3", "priority": 2323, "external_id": null}, {"id": 10848146003, "name": "Universit\u00e9 Lille 2 Droit et Sant\u00e9", "priority": 2324, "external_id": null}, {"id": 10848147003, "name": "Universit\u00e9 Paris Ouest Nanterre La D\u00e9fense", "priority": 2325, "external_id": null}, {"id": 10848148003, "name": "Universit\u00e9 Paul-Val\u00e9ry Montpellier 3", "priority": 2326, "external_id": null}, {"id": 10848149003, "name": "Universit\u00e9 Pierre Mend\u00e8s France - Grenoble 2", "priority": 2327, "external_id": null}, {"id": 10848150003, "name": "Universit\u00e9 Stendhal Grenoble 3", "priority": 2328, "external_id": null}, {"id": 10848151003, "name": "Universit\u00e9 Toulouse II, Le Mirail", "priority": 2329, "external_id": null}, {"id": 10848152003, "name": "Universiti Teknologi MARA - UiTM", "priority": 2330, "external_id": null}, {"id": 10848153003, "name": "University of Baghdad", "priority": 2331, "external_id": null}, {"id": 10848154003, "name": "University of Bahrain", "priority": 2332, "external_id": null}, {"id": 10848155003, "name": "University of Bari", "priority": 2333, "external_id": null}, {"id": 10848156003, "name": "University of Belgrade", "priority": 2334, "external_id": null}, {"id": 10848157003, "name": "University of Brawijaya", "priority": 2335, "external_id": null}, {"id": 10848158003, "name": "University of Brescia", "priority": 2336, "external_id": null}, {"id": 10848159003, "name": "University of Bucharest", "priority": 2337, "external_id": null}, {"id": 10848160003, "name": "University of Calcutta", "priority": 2338, "external_id": null}, {"id": 10848161003, "name": "University of Central Lancashire", "priority": 2339, "external_id": null}, {"id": 10848162003, "name": "University of Colombo", "priority": 2340, "external_id": null}, {"id": 10848163003, "name": "University of Dhaka", "priority": 2341, "external_id": null}, {"id": 10848164003, "name": "University of East London", "priority": 2342, "external_id": null}, {"id": 10848165003, "name": "University of Engineering & Technology (UET) Lahore", "priority": 2343, "external_id": null}, {"id": 10848166003, "name": "University of Greenwich", "priority": 2344, "external_id": null}, {"id": 10848167003, "name": "University of Jordan", "priority": 2345, "external_id": null}, {"id": 10848168003, "name": "University of Karachi", "priority": 2346, "external_id": null}, {"id": 10848169003, "name": "University of Lahore", "priority": 2347, "external_id": null}, {"id": 10848170003, "name": "University of Latvia", "priority": 2348, "external_id": null}, {"id": 10848171003, "name": "University of New England", "priority": 2349, "external_id": null}, {"id": 10848172003, "name": "University of Pune", "priority": 2350, "external_id": null}, {"id": 10848173003, "name": "University of Santo Tomas", "priority": 2351, "external_id": null}, {"id": 10848174003, "name": "University of Southern Queensland", "priority": 2352, "external_id": null}, {"id": 10848175003, "name": "University of Wroclaw", "priority": 2353, "external_id": null}, {"id": 10848176003, "name": "Verona University", "priority": 2354, "external_id": null}, {"id": 10848177003, "name": "Victoria University", "priority": 2355, "external_id": null}, {"id": 10848178003, "name": "Vilnius Gediminas Technical University", "priority": 2356, "external_id": null}, {"id": 10848179003, "name": "Voronezh State University", "priority": 2357, "external_id": null}, {"id": 10848180003, "name": "Vytautas Magnus University", "priority": 2358, "external_id": null}, {"id": 10848181003, "name": "West University of Timisoara", "priority": 2359, "external_id": null}, {"id": 10848182003, "name": "University of South Alabama", "priority": 2360, "external_id": null}, {"id": 10848183003, "name": "University of Arkansas", "priority": 2361, "external_id": null}, {"id": 10848184003, "name": "University of California - Berkeley", "priority": 2362, "external_id": null}, {"id": 10848185003, "name": "University of Connecticut", "priority": 2363, "external_id": null}, {"id": 10848186003, "name": "University of South Florida", "priority": 2364, "external_id": null}, {"id": 10848187003, "name": "University of Georgia", "priority": 2365, "external_id": null}, {"id": 10848188003, "name": "University of Hawaii - Manoa", "priority": 2366, "external_id": null}, {"id": 10848189003, "name": "Iowa State University", "priority": 2367, "external_id": null}, {"id": 10848190003, "name": "Murray State University", "priority": 2368, "external_id": null}, {"id": 10848191003, "name": "University of Louisville", "priority": 2369, "external_id": null}, {"id": 10848192003, "name": "Western Kentucky University", "priority": 2370, "external_id": null}, {"id": 10848193003, "name": "Louisiana State University - Baton Rouge", "priority": 2371, "external_id": null}, {"id": 10848194003, "name": "University of Maryland - College Park", "priority": 2372, "external_id": null}, {"id": 10848195003, "name": "University of Minnesota - Twin Cities", "priority": 2373, "external_id": null}, {"id": 10848196003, "name": "University of Montana", "priority": 2374, "external_id": null}, {"id": 10848197003, "name": "East Carolina University", "priority": 2375, "external_id": null}, {"id": 10848198003, "name": "University of North Carolina - Chapel Hill", "priority": 2376, "external_id": null}, {"id": 10848199003, "name": "Wake Forest University", "priority": 2377, "external_id": null}, {"id": 10848200003, "name": "University of Nebraska - Lincoln", "priority": 2378, "external_id": null}, {"id": 10848201003, "name": "New Mexico State University", "priority": 2379, "external_id": null}, {"id": 10848202003, "name": "Ohio State University - Columbus", "priority": 2380, "external_id": null}, {"id": 10848203003, "name": "University of Oklahoma", "priority": 2381, "external_id": null}, {"id": 10848204003, "name": "Pennsylvania State University - University Park", "priority": 2382, "external_id": null}, {"id": 10848205003, "name": "University of Pittsburgh", "priority": 2383, "external_id": null}, {"id": 10848206003, "name": "University of Tennessee - Chattanooga", "priority": 2384, "external_id": null}, {"id": 10848207003, "name": "Vanderbilt University", "priority": 2385, "external_id": null}, {"id": 10848208003, "name": "Rice University", "priority": 2386, "external_id": null}, {"id": 10848209003, "name": "University of Utah", "priority": 2387, "external_id": null}, {"id": 10848210003, "name": "University of Richmond", "priority": 2388, "external_id": null}, {"id": 10848211003, "name": "University of Arkansas - Pine Bluff", "priority": 2389, "external_id": null}, {"id": 10848212003, "name": "University of Central Florida", "priority": 2390, "external_id": null}, {"id": 10848213003, "name": "Florida Atlantic University", "priority": 2391, "external_id": null}, {"id": 10848214003, "name": "Hampton University", "priority": 2392, "external_id": null}, {"id": 10848215003, "name": "Liberty University", "priority": 2393, "external_id": null}, {"id": 10848216003, "name": "Mercer University", "priority": 2394, "external_id": null}, {"id": 10848217003, "name": "Middle Tennessee State University", "priority": 2395, "external_id": null}, {"id": 10848218003, "name": "University of Nevada - Las Vegas", "priority": 2396, "external_id": null}, {"id": 10848219003, "name": "South Carolina State University", "priority": 2397, "external_id": null}, {"id": 10848220003, "name": "University of Tennessee - Martin", "priority": 2398, "external_id": null}, {"id": 10848221003, "name": "Weber State University", "priority": 2399, "external_id": null}, {"id": 10848222003, "name": "Youngstown State University", "priority": 2400, "external_id": null}, {"id": 10848223003, "name": "University of the Incarnate Word", "priority": 2401, "external_id": null}, {"id": 10848224003, "name": "University of Washington", "priority": 2402, "external_id": null}, {"id": 10848225003, "name": "University of Louisiana - Lafayette", "priority": 2403, "external_id": null}, {"id": 10848226003, "name": "Coastal Carolina University", "priority": 2404, "external_id": null}, {"id": 10848227003, "name": "Utah State University", "priority": 2405, "external_id": null}, {"id": 10848228003, "name": "University of Alabama", "priority": 2406, "external_id": null}, {"id": 10848229003, "name": "University of Illinois - Urbana-Champaign", "priority": 2407, "external_id": null}, {"id": 10848230003, "name": "United States Air Force Academy", "priority": 2408, "external_id": null}, {"id": 10848231003, "name": "University of Akron", "priority": 2409, "external_id": null}, {"id": 10848232003, "name": "University of Central Arkansas", "priority": 2410, "external_id": null}, {"id": 10848233003, "name": "University of Kansas", "priority": 2411, "external_id": null}, {"id": 10848234003, "name": "University of Northern Colorado", "priority": 2412, "external_id": null}, {"id": 10848235003, "name": "University of Northern Iowa", "priority": 2413, "external_id": null}, {"id": 10848236003, "name": "University of South Carolina", "priority": 2414, "external_id": null}, {"id": 10848237003, "name": "Tennessee Technological University", "priority": 2415, "external_id": null}, {"id": 10848238003, "name": "University of Texas - El Paso", "priority": 2416, "external_id": null}, {"id": 10848239003, "name": "Texas Tech University", "priority": 2417, "external_id": null}, {"id": 10848240003, "name": "Tulane University", "priority": 2418, "external_id": null}, {"id": 10848241003, "name": "Virginia Military Institute", "priority": 2419, "external_id": null}, {"id": 10848242003, "name": "Western Michigan University", "priority": 2420, "external_id": null}, {"id": 10848243003, "name": "Wilfrid Laurier University", "priority": 2421, "external_id": null}, {"id": 10848244003, "name": "University of San Diego", "priority": 2422, "external_id": null}, {"id": 10848245003, "name": "University of California - San Diego", "priority": 2423, "external_id": null}, {"id": 10848246003, "name": "Brooks Institute of Photography", "priority": 2424, "external_id": null}, {"id": 10848247003, "name": "Acupuncture and Integrative Medicine College - Berkeley", "priority": 2425, "external_id": null}, {"id": 10848248003, "name": "Southern Alberta Institute of Technology", "priority": 2426, "external_id": null}, {"id": 10848249003, "name": "Susquehanna University", "priority": 2427, "external_id": null}, {"id": 10848250003, "name": "University of Texas - Dallas", "priority": 2428, "external_id": null}, {"id": 10848251003, "name": "Thunderbird School of Global Management", "priority": 2429, "external_id": null}, {"id": 10848252003, "name": "Presidio Graduate School", "priority": 2430, "external_id": null}, {"id": 10848253003, "name": "\u00c9cole sup\u00e9rieure de commerce de Dijon", "priority": 2431, "external_id": null}, {"id": 10848254003, "name": "University of California - San Francisco", "priority": 2432, "external_id": null}, {"id": 10848255003, "name": "Hack Reactor", "priority": 2433, "external_id": null}, {"id": 10848256003, "name": "St. Mary''s College of California", "priority": 2434, "external_id": null}, {"id": 10848257003, "name": "New England Law", "priority": 2435, "external_id": null}, {"id": 10848258003, "name": "University of California, Merced", "priority": 2436, "external_id": null}, {"id": 10848259003, "name": "University of California, Hastings College of the Law", "priority": 2437, "external_id": null}, {"id": 10848260003, "name": "V.N. Karazin Kharkiv National University", "priority": 2438, "external_id": null}, {"id": 10848261003, "name": "SIM University (UniSIM)", "priority": 2439, "external_id": null}, {"id": 10848262003, "name": "Singapore Management University (SMU)", "priority": 2440, "external_id": null}, {"id": 10848263003, "name": "Singapore University of Technology and Design (SUTD)", "priority": 2441, "external_id": null}, {"id": 10848264003, "name": "Singapore Institute of Technology (SIT)", "priority": 2442, "external_id": null}, {"id": 10848265003, "name": "Nanyang Polytechnic (NYP)", "priority": 2443, "external_id": null}, {"id": 10848266003, "name": "Ngee Ann Polytechnic (NP)", "priority": 2444, "external_id": null}, {"id": 10848267003, "name": "Republic Polytechnic (RP)", "priority": 2445, "external_id": null}, {"id": 10848268003, "name": "Singapore Polytechnic (SP)", "priority": 2446, "external_id": null}, {"id": 10848269003, "name": "Temasek Polytechnic (TP)", "priority": 2447, "external_id": null}, {"id": 10848270003, "name": "INSEAD", "priority": 2448, "external_id": null}, {"id": 10848271003, "name": "Funda\u00e7\u00e3o Get\u00falio Vargas", "priority": 2449, "external_id": null}, {"id": 10848272003, "name": "Acharya Nagarjuna University", "priority": 2450, "external_id": null}, {"id": 10848273003, "name": "University of California - Santa Barbara", "priority": 2451, "external_id": null}, {"id": 10848274003, "name": "University of California - Irvine", "priority": 2452, "external_id": null}, {"id": 10848275003, "name": "California State University - Long Beach", "priority": 2453, "external_id": null}, {"id": 10848276003, "name": "Robert Morris University Illinois", "priority": 2454, "external_id": null}, {"id": 10848277003, "name": "Harold Washington College - City Colleges of Chicago", "priority": 2455, "external_id": null}, {"id": 10848278003, "name": "Harry S Truman College - City Colleges of Chicago", "priority": 2456, "external_id": null}, {"id": 10848279003, "name": "Kennedy-King College - City Colleges of Chicago", "priority": 2457, "external_id": null}, {"id": 10848280003, "name": "Malcolm X College - City Colleges of Chicago", "priority": 2458, "external_id": null}, {"id": 10848281003, "name": "Olive-Harvey College - City Colleges of Chicago", "priority": 2459, "external_id": null}, {"id": 10848282003, "name": "Richard J Daley College - City Colleges of Chicago", "priority": 2460, "external_id": null}, {"id": 10848283003, "name": "Wilbur Wright College - City Colleges of Chicago", "priority": 2461, "external_id": null}, {"id": 10848284003, "name": "Abertay University", "priority": 2462, "external_id": null}, {"id": 10848285003, "name": "Pontif\u00edcia Universidade Cat\u00f3lica de Minas Gerais", "priority": 2463, "external_id": null}, {"id": 10848286003, "name": "Other", "priority": 2464, "external_id": null}, {"id": 19126655003, "name": "Atlanta College of Arts", "priority": 2465, "external_id": null}]}, "emitted_at": 1664285620787} {"stream": "custom_fields", "data": {"id": 4680899003, "name": "Degree", "active": true, "field_type": "candidate", "priority": 1, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "degree", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10848287003, "name": "High School", "priority": 0, "external_id": null}, {"id": 10848288003, "name": "Associate's Degree", "priority": 1, "external_id": null}, {"id": 10848289003, "name": "Bachelor's Degree", "priority": 2, "external_id": null}, {"id": 10848290003, "name": "Master's Degree", "priority": 3, "external_id": null}, {"id": 10848291003, "name": "Master of Business Administration (M.B.A.)", "priority": 4, "external_id": null}, {"id": 10848292003, "name": "Juris Doctor (J.D.)", "priority": 5, "external_id": null}, {"id": 10848293003, "name": "Doctor of Medicine (M.D.)", "priority": 6, "external_id": null}, {"id": 10848294003, "name": "Doctor of Philosophy (Ph.D.)", "priority": 7, "external_id": null}, {"id": 10848295003, "name": "Engineer's Degree", "priority": 8, "external_id": null}, {"id": 10848296003, "name": "Other", "priority": 9, "external_id": null}]}, "emitted_at": 1664285620804} {"stream": "custom_fields", "data": {"id": 4680900003, "name": "Discipline", "active": true, "field_type": "candidate", "priority": 2, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "discipline", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10848297003, "name": "Accounting", "priority": 0, "external_id": null}, {"id": 10848298003, "name": "African Studies", "priority": 1, "external_id": null}, {"id": 10848299003, "name": "Agriculture", "priority": 2, "external_id": null}, {"id": 10848300003, "name": "Anthropology", "priority": 3, "external_id": null}, {"id": 10848301003, "name": "Applied Health Services", "priority": 4, "external_id": null}, {"id": 10848302003, "name": "Architecture", "priority": 5, "external_id": null}, {"id": 10848303003, "name": "Art", "priority": 6, "external_id": null}, {"id": 10848304003, "name": "Asian Studies", "priority": 7, "external_id": null}, {"id": 10848305003, "name": "Biology", "priority": 8, "external_id": null}, {"id": 10848306003, "name": "Business", "priority": 9, "external_id": null}, {"id": 10848307003, "name": "Business Administration", "priority": 10, "external_id": null}, {"id": 10848308003, "name": "Chemistry", "priority": 11, "external_id": null}, {"id": 10848309003, "name": "Classical Languages", "priority": 12, "external_id": null}, {"id": 10848310003, "name": "Communications & Film", "priority": 13, "external_id": null}, {"id": 10848311003, "name": "Computer Science", "priority": 14, "external_id": null}, {"id": 10848312003, "name": "Dentistry", "priority": 15, "external_id": null}, {"id": 10848313003, "name": "Developing Nations", "priority": 16, "external_id": null}, {"id": 10848314003, "name": "Discipline Unknown", "priority": 17, "external_id": null}, {"id": 10848315003, "name": "Earth Sciences", "priority": 18, "external_id": null}, {"id": 10848316003, "name": "Economics", "priority": 19, "external_id": null}, {"id": 10848317003, "name": "Education", "priority": 20, "external_id": null}, {"id": 10848318003, "name": "Electronics", "priority": 21, "external_id": null}, {"id": 10848319003, "name": "Engineering", "priority": 22, "external_id": null}, {"id": 10848320003, "name": "English Studies", "priority": 23, "external_id": null}, {"id": 10848321003, "name": "Environmental Studies", "priority": 24, "external_id": null}, {"id": 10848322003, "name": "European Studies", "priority": 25, "external_id": null}, {"id": 10848323003, "name": "Fashion", "priority": 26, "external_id": null}, {"id": 10848324003, "name": "Finance", "priority": 27, "external_id": null}, {"id": 10848325003, "name": "Fine Arts", "priority": 28, "external_id": null}, {"id": 10848326003, "name": "General Studies", "priority": 29, "external_id": null}, {"id": 10848327003, "name": "Health Services", "priority": 30, "external_id": null}, {"id": 10848328003, "name": "History", "priority": 31, "external_id": null}, {"id": 10848329003, "name": "Human Resources Management", "priority": 32, "external_id": null}, {"id": 10848330003, "name": "Humanities", "priority": 33, "external_id": null}, {"id": 10848331003, "name": "Industrial Arts & Carpentry", "priority": 34, "external_id": null}, {"id": 10848332003, "name": "Information Systems", "priority": 35, "external_id": null}, {"id": 10848333003, "name": "International Relations", "priority": 36, "external_id": null}, {"id": 10848334003, "name": "Journalism", "priority": 37, "external_id": null}, {"id": 10848335003, "name": "Languages", "priority": 38, "external_id": null}, {"id": 10848336003, "name": "Latin American Studies", "priority": 39, "external_id": null}, {"id": 10848337003, "name": "Law", "priority": 40, "external_id": null}, {"id": 10848338003, "name": "Linguistics", "priority": 41, "external_id": null}, {"id": 10848339003, "name": "Manufacturing & Mechanics", "priority": 42, "external_id": null}, {"id": 10848340003, "name": "Mathematics", "priority": 43, "external_id": null}, {"id": 10848341003, "name": "Medicine", "priority": 44, "external_id": null}, {"id": 10848342003, "name": "Middle Eastern Studies", "priority": 45, "external_id": null}, {"id": 10848343003, "name": "Naval Science", "priority": 46, "external_id": null}, {"id": 10848344003, "name": "North American Studies", "priority": 47, "external_id": null}, {"id": 10848345003, "name": "Nuclear Technics", "priority": 48, "external_id": null}, {"id": 10848346003, "name": "Operations Research & Strategy", "priority": 49, "external_id": null}, {"id": 10848347003, "name": "Organizational Theory", "priority": 50, "external_id": null}, {"id": 10848348003, "name": "Philosophy", "priority": 51, "external_id": null}, {"id": 10848349003, "name": "Physical Education", "priority": 52, "external_id": null}, {"id": 10848350003, "name": "Physical Sciences", "priority": 53, "external_id": null}, {"id": 10848351003, "name": "Physics", "priority": 54, "external_id": null}, {"id": 10848352003, "name": "Political Science", "priority": 55, "external_id": null}, {"id": 10848353003, "name": "Psychology", "priority": 56, "external_id": null}, {"id": 10848354003, "name": "Public Policy", "priority": 57, "external_id": null}, {"id": 10848355003, "name": "Public Service", "priority": 58, "external_id": null}, {"id": 10848356003, "name": "Religious Studies", "priority": 59, "external_id": null}, {"id": 10848357003, "name": "Russian & Soviet Studies", "priority": 60, "external_id": null}, {"id": 10848358003, "name": "Scandinavian Studies", "priority": 61, "external_id": null}, {"id": 10848359003, "name": "Science", "priority": 62, "external_id": null}, {"id": 10848360003, "name": "Slavic Studies", "priority": 63, "external_id": null}, {"id": 10848361003, "name": "Social Science", "priority": 64, "external_id": null}, {"id": 10848362003, "name": "Social Sciences", "priority": 65, "external_id": null}, {"id": 10848363003, "name": "Sociology", "priority": 66, "external_id": null}, {"id": 10848364003, "name": "Speech", "priority": 67, "external_id": null}, {"id": 10848365003, "name": "Statistics & Decision Theory", "priority": 68, "external_id": null}, {"id": 10848366003, "name": "Urban Studies", "priority": 69, "external_id": null}, {"id": 10848367003, "name": "Veterinary Medicine", "priority": 70, "external_id": null}, {"id": 10848368003, "name": "Other", "priority": 71, "external_id": null}]}, "emitted_at": 1664285620804} diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/users.json b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/users.json index 6e0e7a4a788a..57aa80f97ead 100644 --- a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/users.json +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/users.json @@ -40,6 +40,75 @@ }, "linked_candidate_ids": { "type": ["null", "array"] + }, + "departments": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "parent_id": { + "type": ["null", "integer"] + }, + "parent_department_external_id": { + "type": ["null", "string"] + }, + "child_ids": { + "type": "array" + }, + "child_department_external_ids": { + "type": "array" + }, + "external_id": { + "type": ["null", "string"] + } + } + } + }, + "offices": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "location": { + "type": "object", + "properties": { + "name": { + "type": ["null", "string"] + } + } + }, + "primary_contact_user_id": { + "type": "integer" + }, + "parent_id": { + "type": ["null", "integer"] + }, + "parent_office_external_id": { + "type": ["null", "string"] + }, + "child_ids": { + "type": "array" + }, + "child_office_external_ids": { + "type": "array" + }, + "external_id": { + "type": ["null", "string"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml index ac1d4738c9e2..e9a1a02ef6d8 100644 --- a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml @@ -12,7 +12,7 @@ tests: - config_path: "integration_tests/invalid_config_oauth.json" status: "failed" - config_path: "integration_tests/invalid_config_wrong_title.json" - status: "exception" + status: "failed" discovery: - config_path: "secrets/config.json" backward_compatibility_tests_config: diff --git a/airbyte-integrations/connectors/source-hubspot/setup.py b/airbyte-integrations/connectors/source-hubspot/setup.py index 0ebfed398b9c..a4c05e9a795b 100644 --- a/airbyte-integrations/connectors/source-hubspot/setup.py +++ b/airbyte-integrations/connectors/source-hubspot/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk~=0.2", "backoff==1.11.1", "pendulum==2.1.2", "requests==2.26.0", diff --git a/airbyte-integrations/connectors/source-iterable/Dockerfile b/airbyte-integrations/connectors/source-iterable/Dockerfile index 15e1b2a8a277..b757f1c06cc5 100644 --- a/airbyte-integrations/connectors/source-iterable/Dockerfile +++ b/airbyte-integrations/connectors/source-iterable/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.20 +LABEL io.airbyte.version=0.1.21 LABEL io.airbyte.name=airbyte/source-iterable diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/source.py b/airbyte-integrations/connectors/source-iterable/source_iterable/source.py index 7ac4de3f9b87..d63158d256e2 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/source.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/source.py @@ -4,12 +4,14 @@ from typing import Any, List, Mapping, Tuple +import requests.exceptions from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from .streams import ( + AccessCheck, Campaigns, CampaignsMetrics, Channels, @@ -75,6 +77,18 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: return False, f"Unable to connect to Iterable API with the provided credentials - {e}" def streams(self, config: Mapping[str, Any]) -> List[Stream]: + def all_streams_accessible(): + access_check_stream = AccessCheck(authenticator=authenticator) + slice_ = next(iter(access_check_stream.stream_slices(sync_mode=SyncMode.full_refresh))) + try: + list(access_check_stream.read_records(stream_slice=slice_, sync_mode=SyncMode.full_refresh)) + except requests.exceptions.RequestException as e: + if e.response.status_code == requests.codes.UNAUTHORIZED: + return False + raise + else: + return True + authenticator = TokenAuthenticator(token=config["api_key"], auth_header="Api-Key", auth_method="") # end date is provided for integration tests only start_date, end_date = config["start_date"], config.get("end_date") @@ -95,13 +109,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: # A simple check is done - a read operation on a stream that can be accessed only via a Server side API key. # If read is successful - other streams should be supported as well. # More on this - https://support.iterable.com/hc/en-us/articles/360043464871-API-Keys- - users_stream = ListUsers(authenticator=authenticator) - for slice_ in users_stream.stream_slices(sync_mode=SyncMode.full_refresh): - users = users_stream.read_records(stream_slice=slice_, sync_mode=SyncMode.full_refresh) - # first slice is enough - break - - if next(users, None): + if all_streams_accessible(): streams.extend( [ Users(authenticator=authenticator, **date_range), diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py index a759d11b5c0e..9df4df672cdc 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py @@ -668,3 +668,11 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class Users(IterableExportStreamRanged): data_field = "user" cursor_field = "profileUpdatedAt" + + +class AccessCheck(ListUsers): + # since 401 error is failed silently in all the streams, + # we need another class to distinguish an empty stream from 401 response + def check_unauthorized_key(self, response: requests.Response) -> bool: + # this allows not retrying 401 and raising the error upstream + return response.status_code != codes.UNAUTHORIZED diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_source.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_source.py index c48ea066f285..3b32ee28e128 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_source.py @@ -11,9 +11,9 @@ @responses.activate -@pytest.mark.parametrize("body, status, expected_streams", (({}, 401, 7), ({"lists": [{"id": 1}]}, 200, 44))) +@pytest.mark.parametrize("body, status, expected_streams", ((b"", 401, 7), (b"", 200, 44), (b"alpha@gmail.com\nbeta@gmail.com", 200, 44))) def test_source_streams(mock_lists_resp, config, body, status, expected_streams): - responses.add(responses.GET, "https://api.iterable.com/api/lists/getUsers?listId=1", json=body, status=status) + responses.add(responses.GET, "https://api.iterable.com/api/lists/getUsers?listId=1", body=body, status=status) streams = SourceIterable().streams(config=config) assert len(streams) == expected_streams diff --git a/airbyte-integrations/connectors/source-lever-hiring/Dockerfile b/airbyte-integrations/connectors/source-lever-hiring/Dockerfile index f76384f30350..55095a551839 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/Dockerfile +++ b/airbyte-integrations/connectors/source-lever-hiring/Dockerfile @@ -34,5 +34,5 @@ COPY source_lever_hiring ./source_lever_hiring ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-lever-hiring diff --git a/airbyte-integrations/connectors/source-lever-hiring/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/sample_config.json index 2730c7b0ee81..e5ac0a8d0b19 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/sample_config.json @@ -1,7 +1,7 @@ -{ - "client_id": "client_id", - "client_secret": "client_secret", - "refresh_token": "refresh_token", - "environment": "Sandbox", - "start_date": "2021-07-12T00:00:00Z" -} +{ "credentials" : { + "api_key": "c1adBeHUJKqyPM0X01tshKl4Pwh1LyPdmlorXjfmoDE0lVxZl", + "auth_type": "Api Key" + }, + "start_date": "2021-07-12T00:00:00Z", + "environment": "Sandbox" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-lever-hiring/integration_tests/spec.json b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/spec.json index 751b39c470f9..59b326343ee4 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/spec.json @@ -22,8 +22,6 @@ "auth_type": { "type": "string", "const": "Client", - "enum": ["Client"], - "default": "Client", "order": 0 }, "client_id": { @@ -37,12 +35,6 @@ "description": "The Client Secret of your Lever Hiring developer application.", "airbyte_secret": true }, - "option_title": { - "type": "string", - "title": "Credentials Title", - "description": "OAuth Credentials", - "const": "OAuth Credentials" - }, "refresh_token": { "type": "string", "title": "Refresh Token", @@ -50,6 +42,25 @@ "airbyte_secret": true } } + }, + { + "type": "object", + "title": "Authenticate via Lever (Api Key)", + "required": ["api_key"], + "properties": { + "auth_type": { + "type": "string", + "const": "Api Key", + "order": 0 + }, + "api_key": { + "title": "Api key", + "type": "string", + "description": "The Api Key of your Lever Hiring account.", + "airbyte_secret": true, + "order":1 + } + } } ] }, diff --git a/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/source.py b/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/source.py index d70ef1c2abef..0ec31c366fd8 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/source.py +++ b/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/source.py @@ -6,18 +6,28 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator +from airbyte_cdk.sources.streams.http.auth import BasicHttpAuthenticator, Oauth2Authenticator from .streams import Applications, Interviews, Notes, Offers, Opportunities, Referrals, Users def _auth_from_config(config): - return Oauth2Authenticator( - client_id=config["credentials"]["client_id"], - client_secret=config["credentials"]["client_secret"], - refresh_token=config["credentials"]["refresh_token"], - token_refresh_endpoint=f"{SourceLeverHiring.URL_MAP_ACCORDING_ENVIRONMENT[config['environment']]['login']}oauth/token", - ) + try: + if config["credentials"]["auth_type"] == "Api Key": + return BasicHttpAuthenticator(username=config["credentials"]["api_key"], password=None, auth_method="Basic") + elif config["credentials"]["auth_type"] == "Client": + return Oauth2Authenticator( + client_id=config["credentials"]["client_id"], + client_secret=config["credentials"]["client_secret"], + refresh_token=config["credentials"]["refresh_token"], + token_refresh_endpoint=f"{SourceLeverHiring.URL_MAP_ACCORDING_ENVIRONMENT[config['environment']]['login']}oauth/token", + ) + else: + print("Auth type was not configured properly") + return None + except Exception as e: + print(f"{e.__class__} occurred, there's an issue with credentials in your config") + raise e class SourceLeverHiring(AbstractSource): diff --git a/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/spec.json b/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/spec.json index 751b39c470f9..59b326343ee4 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/spec.json +++ b/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/spec.json @@ -22,8 +22,6 @@ "auth_type": { "type": "string", "const": "Client", - "enum": ["Client"], - "default": "Client", "order": 0 }, "client_id": { @@ -37,12 +35,6 @@ "description": "The Client Secret of your Lever Hiring developer application.", "airbyte_secret": true }, - "option_title": { - "type": "string", - "title": "Credentials Title", - "description": "OAuth Credentials", - "const": "OAuth Credentials" - }, "refresh_token": { "type": "string", "title": "Refresh Token", @@ -50,6 +42,25 @@ "airbyte_secret": true } } + }, + { + "type": "object", + "title": "Authenticate via Lever (Api Key)", + "required": ["api_key"], + "properties": { + "auth_type": { + "type": "string", + "const": "Api Key", + "order": 0 + }, + "api_key": { + "title": "Api key", + "type": "string", + "description": "The Api Key of your Lever Hiring account.", + "airbyte_secret": true, + "order":1 + } + } } ] }, diff --git a/airbyte-integrations/connectors/source-lever-hiring/unit_tests/conftest.py b/airbyte-integrations/connectors/source-lever-hiring/unit_tests/conftest.py index 75250b480fbf..a02ce7a2939f 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-lever-hiring/unit_tests/conftest.py @@ -5,21 +5,6 @@ from pytest import fixture -@fixture -def test_config(): - return { - "credentials": { - "client_id": "test_client_id", - "client_secret": "test_client_secret", - "refresh_token": "test_refresh_token", - "access_token": "test_access_token", - "expires_in": 3600, - }, - "environment": "Sandbox", - "start_date": "2021-05-07T00:00:00Z", - } - - @fixture def test_full_refresh_config(): return {"base_url": "test_base_url"} diff --git a/airbyte-integrations/connectors/source-lever-hiring/unit_tests/test_source.py b/airbyte-integrations/connectors/source-lever-hiring/unit_tests/test_source.py index 942c2ac8e6c3..edbe8d50634e 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-lever-hiring/unit_tests/test_source.py @@ -4,28 +4,51 @@ from unittest.mock import MagicMock +import pytest import responses from source_lever_hiring.source import SourceLeverHiring -def setup_responses(): - responses.add( - responses.POST, "https://sandbox-lever.auth0.com/oauth/token", json={"access_token": "fake_access_token", "expires_in": 3600} - ) - - +@pytest.mark.parametrize( + ("response", "url", "payload", "test_config"), + [ + ( + responses.POST, + "https://sandbox-lever.auth0.com/oauth/token", + {"access_token": "test_access_token", "expires_in": 3600}, + { + "credentials": { + "auth_type": "Client", + "client_id": "test_client_id", + "client_secret": "test_client_secret", + "refresh_token": "test_refresh_token", + "access_token": "test_access_token", + "expires_in": 3600, + }, + "environment": "Sandbox", + "start_date": "2021-05-07T00:00:00Z", + }, + ), + ( + None, + None, + None, + { + "credentials": { + "auth_type": "Api Key", + "api_key": "test_api_key", + }, + "environment": "Sandbox", + "start_date": "2021-05-07T00:00:00Z", + }, + ), + ], +) @responses.activate -def test_check_connection(test_config): - setup_responses() +def test_source(response, url, payload, test_config): + if response: + responses.add(response, url, json=payload) source = SourceLeverHiring() logger_mock = MagicMock() assert source.check_connection(logger_mock, test_config) == (True, None) - - -@responses.activate -def test_streams(test_config): - setup_responses() - source = SourceLeverHiring() - streams = source.streams(test_config) - expected_streams_number = 7 - assert len(streams) == expected_streams_number + assert len(source.streams(test_config)) == 7 diff --git a/airbyte-integrations/connectors/source-lokalise/.dockerignore b/airbyte-integrations/connectors/source-lokalise/.dockerignore new file mode 100644 index 000000000000..f277dc6b20d6 --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_lokalise +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-lokalise/Dockerfile b/airbyte-integrations/connectors/source-lokalise/Dockerfile new file mode 100644 index 000000000000..33887dbc0d6e --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_lokalise ./source_lokalise + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-lokalise diff --git a/airbyte-integrations/connectors/source-lokalise/README.md b/airbyte-integrations/connectors/source-lokalise/README.md new file mode 100644 index 000000000000..b9b209ecb381 --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/README.md @@ -0,0 +1,79 @@ +# Lokalise Source + +This is the repository for the Lokalise configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/lokalise). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-lokalise:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/lokalise) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_lokalise/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source lokalise test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-lokalise:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-lokalise:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-lokalise:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-lokalise:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-lokalise:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-lokalise:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-lokalise:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-lokalise:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-lokalise/__init__.py b/airbyte-integrations/connectors/source-lokalise/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-lokalise/acceptance-test-config.yml b/airbyte-integrations/connectors/source-lokalise/acceptance-test-config.yml new file mode 100644 index 000000000000..05f1e698b9d9 --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/acceptance-test-config.yml @@ -0,0 +1,27 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-lokalise:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_lokalise/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + incremental: + bypass_reason: "This connector does not implement incremental sync" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-lokalise/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-lokalise/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-lokalise/build.gradle b/airbyte-integrations/connectors/source-lokalise/build.gradle new file mode 100644 index 000000000000..be52bb5ad5ca --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_lokalise' +} diff --git a/airbyte-integrations/connectors/source-lokalise/integration_tests/__init__.py b/airbyte-integrations/connectors/source-lokalise/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-lokalise/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-lokalise/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..aaad076bfedf --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "keys": { + "key_id": "1234x" + } +} diff --git a/airbyte-integrations/connectors/source-lokalise/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-lokalise/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-lokalise/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-lokalise/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..e6f628fe658e --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/integration_tests/configured_catalog.json @@ -0,0 +1,49 @@ +{ + "streams": [ + { + "stream": { + "name": "keys", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "languages", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "comments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "contributors", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "translations", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-lokalise/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-lokalise/integration_tests/invalid_config.json new file mode 100644 index 000000000000..17d902f551d3 --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "project_id": "invalid project ID", + "api_key": "invalid API key" +} diff --git a/airbyte-integrations/connectors/source-lokalise/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-lokalise/integration_tests/sample_config.json new file mode 100644 index 000000000000..b155168d8c5a --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/integration_tests/sample_config.json @@ -0,0 +1,4 @@ +{ + "project_id": "", + "api_key": "" +} diff --git a/airbyte-integrations/connectors/source-lokalise/main.py b/airbyte-integrations/connectors/source-lokalise/main.py new file mode 100644 index 000000000000..0cc2ac41da14 --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_lokalise import SourceLokalise + +if __name__ == "__main__": + source = SourceLokalise() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-lokalise/requirements.txt b/airbyte-integrations/connectors/source-lokalise/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-lokalise/setup.py b/airbyte-integrations/connectors/source-lokalise/setup.py new file mode 100644 index 000000000000..def00ba65970 --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_lokalise", + description="Source implementation for Lokalise.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-lokalise/source_lokalise/__init__.py b/airbyte-integrations/connectors/source-lokalise/source_lokalise/__init__.py new file mode 100644 index 000000000000..05e2035af504 --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/source_lokalise/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceLokalise + +__all__ = ["SourceLokalise"] diff --git a/airbyte-integrations/connectors/source-lokalise/source_lokalise/lokalise.yaml b/airbyte-integrations/connectors/source-lokalise/source_lokalise/lokalise.yaml new file mode 100644 index 000000000000..56ef4a46d18c --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/source_lokalise/lokalise.yaml @@ -0,0 +1,131 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: + - "{{ options['name'] }}" + + requester: + url_base: "https://api.lokalise.com" + http_method: "GET" + authenticator: + type: ApiKeyAuthenticator + header: "X-Api-Token" + api_token: "{{ config['api_key'] }}" + + increment_paginator: + type: "DefaultPaginator" + url_base: "*ref(definitions.requester.url_base)" + pagination_strategy: + type: "PageIncrement" + page_size: 1000 + page_token_option: + inject_into: "request_parameter" + field_name: "page" + page_size_option: + inject_into: "request_parameter" + field_name: "limit" + + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + $ref: "*ref(definitions.increment_paginator)" + requester: + $ref: "*ref(definitions.requester)" + + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + + keys_stream: + # https://developers.lokalise.com/reference/list-all-keys + $ref: "*ref(definitions.base_stream)" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + $ref: "*ref(definitions.increment_paginator)" + requester: + $ref: "*ref(definitions.requester)" + $options: + name: "keys" + primary_key: "key_id" + path: "/api2/projects/{{ config['project_id'] }}/keys" + + languages_stream: + # https://developers.lokalise.com/reference/list-all-keys + $ref: "*ref(definitions.base_stream)" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + $ref: "*ref(definitions.increment_paginator)" + requester: + $ref: "*ref(definitions.requester)" + $options: + name: "languages" + primary_key: "lang_id" + path: "/api2/projects/{{ config['project_id'] }}/languages" + + comments_stream: + # https://developers.lokalise.com/reference/list-project-comments + $ref: "*ref(definitions.base_stream)" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + $ref: "*ref(definitions.increment_paginator)" + requester: + $ref: "*ref(definitions.requester)" + $options: + name: "comments" + primary_key: "comment_id" + path: "/api2/projects/{{ config['project_id'] }}/comments" + + contributors_stream: + # https://developers.lokalise.com/reference/list-all-contributors + $ref: "*ref(definitions.base_stream)" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + $ref: "*ref(definitions.increment_paginator)" + requester: + $ref: "*ref(definitions.requester)" + $options: + name: "contributors" + primary_key: "user_id" + path: "/api2/projects/{{ config['project_id'] }}/contributors" + + translations_stream: + # https://developers.lokalise.com/reference/list-all-translations + $ref: "*ref(definitions.base_stream)" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + $ref: "*ref(definitions.increment_paginator)" + requester: + $ref: "*ref(definitions.requester)" + $options: + name: "translations" + primary_key: "translation_id" + path: "/api2/projects/{{ config['project_id'] }}/translations" + + +streams: + - "*ref(definitions.keys_stream)" + - "*ref(definitions.languages_stream)" + - "*ref(definitions.comments_stream)" + - "*ref(definitions.contributors_stream)" + - "*ref(definitions.translations_stream)" + +check: + stream_names: + - "keys" + - "languages" + - "comments" + - "contributors" + - "translations" diff --git a/airbyte-integrations/connectors/source-lokalise/source_lokalise/schemas/comments.json b/airbyte-integrations/connectors/source-lokalise/source_lokalise/schemas/comments.json new file mode 100644 index 000000000000..d9f30c0f7a62 --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/source_lokalise/schemas/comments.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "added_at_timestamp": { + "type": "integer" + }, + "added_at": { + "type": "string" + }, + "added_by_email": { + "type": "string" + }, + "key_id": { + "type": "integer" + }, + "added_by": { + "type": "integer" + }, + "comment": { + "type": "string" + }, + "comment_id": { + "type": "integer" + } + } +} diff --git a/airbyte-integrations/connectors/source-lokalise/source_lokalise/schemas/contributors.json b/airbyte-integrations/connectors/source-lokalise/source_lokalise/schemas/contributors.json new file mode 100644 index 000000000000..0e0dc48fff42 --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/source_lokalise/schemas/contributors.json @@ -0,0 +1,53 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "is_admin": { + "type": "boolean" + }, + "is_reviewer": { + "type": "boolean" + }, + "languages": { + "type": "array", + "items": { + "type": "object", + "properties": { + "lang_name": { + "type": "string" + }, + "lang_id": { + "type": "integer" + }, + "is_writable": { + "type": "boolean" + }, + "lang_iso": { + "type": "string" + } + } + } + }, + "user_id": { + "type": "integer" + }, + "created_at_timestamp": { + "type": "integer" + }, + "created_at": { + "type": "string" + }, + "admin_rights": { + "type": "array", + "items": { + "type": "string" + } + }, + "fullname": { + "type": "string" + }, + "email": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-lokalise/source_lokalise/schemas/keys.json b/airbyte-integrations/connectors/source-lokalise/source_lokalise/schemas/keys.json new file mode 100644 index 000000000000..82cd8b070e2f --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/source_lokalise/schemas/keys.json @@ -0,0 +1,220 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "key_id": { + "type": ["null", "number"] + }, + "created_at": { + "type": ["null", "string"] + }, + "created_at_timestamp": { + "type": ["null", "number"] + }, + "key_name": { + "type": "object", + "properties": { + "ios": { + "type": ["null", "string"] + }, + "android": { + "type": ["null", "string"] + }, + "web": { + "type": ["null", "string"] + }, + "other": { + "type": ["null", "string"] + } + } + }, + "filenames": { + "type": "object", + "properties": { + "ios": { + "type": ["null", "string"] + }, + "android": { + "type": ["null", "string"] + }, + "web": { + "type": ["null", "string"] + }, + "other": { + "type": ["null", "string"] + } + } + }, + "description": { + "type": ["null", "string"] + }, + "platforms": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "tags": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "comments": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "comment_id": { + "type": "number" + }, + "comment": { + "type": "string" + }, + "added_by": { + "type": "number" + }, + "added_by_email": { + "type": "string" + }, + "added_at": { + "type": "string" + }, + "added_at_timestamp": { + "type": "number" + } + } + } + }, + "screenshots": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "screenshot_id": { + "type": "number" + }, + "key_ids": { + "type": ["null", "array"], + "items": { + "type": "number" + } + }, + "url": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "screenshot_tags": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "width": { + "type": ["null", "number"] + }, + "height": { + "type": ["null", "number"] + }, + "created_at": { + "type": ["null", "string"] + }, + "created_at_timestamp": { + "type": ["null", "number"] + } + } + } + }, + "translations": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "translation_id": { + "type": "number" + }, + "key_id": { + "type": "number" + }, + "language_iso": { + "type": "string" + }, + "translation": { + "type": "string" + }, + "modified_by": { + "type": "number" + }, + "modified_by_email": { + "type": "string" + }, + "modified_at": { + "type": "string" + }, + "modified_at_timestamp": { + "type": "number" + }, + "is_reviewed": { + "type": "boolean" + }, + "is_unverified": { + "type": "boolean" + }, + "verified_by": { + "type": "number" + }, + "words": { + "type": "number" + }, + "custom_translation_statuses": { + "type": "string" + }, + "task_id": { + "type": "number" + } + } + } + }, + "is_plural": { + "type": "boolean" + }, + "plural_name": { + "type": "string" + }, + "is_hidden": { + "type": "boolean" + }, + "is_archived": { + "type": "boolean" + }, + "context": { + "type": "string" + }, + "base_words": { + "type": "number" + }, + "char_limit": { + "type": "number" + }, + "custom_attributes": { + "type": "string" + }, + "modified_at": { + "type": "string" + }, + "modified_at_timestamp": { + "type": "number" + }, + "translations_modified_at": { + "type": "string" + }, + "translations_modified_at_timestamp": { + "type": "number" + } + } +} diff --git a/airbyte-integrations/connectors/source-lokalise/source_lokalise/schemas/languages.json b/airbyte-integrations/connectors/source-lokalise/source_lokalise/schemas/languages.json new file mode 100644 index 000000000000..bbcdca8c5a1c --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/source_lokalise/schemas/languages.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "lang_id": { + "type": ["null", "number"] + }, + "lang_iso": { + "type": ["null", "string"] + }, + "lang_name": { + "type": ["null", "string"] + }, + "is_rtl": { + "type": ["null", "boolean"] + }, + "plural_forms": { + "type": ["null", "array"], + "items": { + "type": "string" + } + } + } +} diff --git a/airbyte-integrations/connectors/source-lokalise/source_lokalise/schemas/translations.json b/airbyte-integrations/connectors/source-lokalise/source_lokalise/schemas/translations.json new file mode 100644 index 000000000000..4b44512ae487 --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/source_lokalise/schemas/translations.json @@ -0,0 +1,54 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "modified_at_timestamp": { + "type": "integer" + }, + "translation_id": { + "type": "integer" + }, + "reviewed_by": { + "type": "integer" + }, + "key_id": { + "type": "integer" + }, + "words": { + "type": "integer" + }, + "custom_translation_statuses": { + "type": "array" + }, + "segment_number": { + "type": "integer" + }, + "task_id": { + "type": "null" + }, + "language_iso": { + "type": "string" + }, + "is_fuzzy": { + "type": "boolean" + }, + "modified_by_email": { + "type": "string" + }, + "is_reviewed": { + "type": "boolean" + }, + "is_unverified": { + "type": "boolean" + }, + "translation": { + "type": "string" + }, + "modified_by": { + "type": "integer" + }, + "modified_at": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-lokalise/source_lokalise/source.py b/airbyte-integrations/connectors/source-lokalise/source_lokalise/source.py new file mode 100644 index 000000000000..704d63423d9e --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/source_lokalise/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceLokalise(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "lokalise.yaml"}) diff --git a/airbyte-integrations/connectors/source-lokalise/source_lokalise/spec.yaml b/airbyte-integrations/connectors/source-lokalise/source_lokalise/spec.yaml new file mode 100644 index 000000000000..404ce5119476 --- /dev/null +++ b/airbyte-integrations/connectors/source-lokalise/source_lokalise/spec.yaml @@ -0,0 +1,19 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/lokalise +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Lokalise Spec + type: object + required: + - api_key + - project_id + additionalProperties: true + properties: + api_key: + title: API Key + type: string + description: Lokalise API Key with read-access. Available at Profile settings > API tokens. See here. + airbyte_secret: true + project_id: + title: Project Id + type: string + description: Lokalise project ID. Available at Project Settings > General. diff --git a/airbyte-integrations/connectors/source-mailerlite/.dockerignore b/airbyte-integrations/connectors/source-mailerlite/.dockerignore new file mode 100644 index 000000000000..50fb533d4521 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_mailerlite +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-mailerlite/Dockerfile b/airbyte-integrations/connectors/source-mailerlite/Dockerfile new file mode 100644 index 000000000000..07a11b787733 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_mailerlite ./source_mailerlite + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-mailerlite diff --git a/airbyte-integrations/connectors/source-mailerlite/README.md b/airbyte-integrations/connectors/source-mailerlite/README.md new file mode 100644 index 000000000000..eae17dff484c --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/README.md @@ -0,0 +1,79 @@ +# Mailerlite Source + +This is the repository for the Mailerlite configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/mailerlite). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-mailerlite:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/mailerlite) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_mailerlite/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source mailerlite test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-mailerlite:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-mailerlite:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-mailerlite:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailerlite:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailerlite:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-mailerlite:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-mailerlite:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-mailerlite:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-mailerlite/__init__.py b/airbyte-integrations/connectors/source-mailerlite/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-mailerlite/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mailerlite/acceptance-test-config.yml new file mode 100644 index 000000000000..2ceeef4ebece --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-mailerlite:dev +tests: + spec: + - spec_path: "source_mailerlite/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.txt" + # extra_fields: no + # exact_order: no + # extra_records: yes + # incremental: # TODO if your connector does not implement incremental sync, remove this block + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-mailerlite/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-mailerlite/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-mailerlite/build.gradle b/airbyte-integrations/connectors/source-mailerlite/build.gradle new file mode 100644 index 000000000000..eb5cc9daee14 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_mailerlite' +} diff --git a/airbyte-integrations/connectors/source-mailerlite/integration_tests/__init__.py b/airbyte-integrations/connectors/source-mailerlite/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-mailerlite/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-mailerlite/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-mailerlite/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mailerlite/integration_tests/acceptance.py new file mode 100644 index 000000000000..1302b2f57e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-mailerlite/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-mailerlite/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..84ad8862b8e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/integration_tests/configured_catalog.json @@ -0,0 +1,76 @@ +{ + "streams": [ + { + "stream": { + "name": "subscribers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "segments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "campaigns", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "automations", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "timezones", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "forms_popup", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "forms_embedded", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "forms_promotion", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-mailerlite/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-mailerlite/integration_tests/invalid_config.json new file mode 100644 index 000000000000..871a8395c5c5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "api_token": "" +} diff --git a/airbyte-integrations/connectors/source-mailerlite/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-mailerlite/integration_tests/sample_config.json new file mode 100644 index 000000000000..5d547f94643e --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "api_token": "" +} diff --git a/airbyte-integrations/connectors/source-mailerlite/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-mailerlite/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-mailerlite/main.py b/airbyte-integrations/connectors/source-mailerlite/main.py new file mode 100644 index 000000000000..58a60c98140b --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_mailerlite import SourceMailerlite + +if __name__ == "__main__": + source = SourceMailerlite() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mailerlite/requirements.txt b/airbyte-integrations/connectors/source-mailerlite/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-mailerlite/setup.py b/airbyte-integrations/connectors/source-mailerlite/setup.py new file mode 100644 index 000000000000..705801cb0329 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_mailerlite", + description="Source implementation for Mailerlite.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/__init__.py b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/__init__.py new file mode 100644 index 000000000000..2176a697b86a --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceMailerlite + +__all__ = ["SourceMailerlite"] diff --git a/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/mailerlite.yaml b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/mailerlite.yaml new file mode 100644 index 000000000000..a82887180375 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/mailerlite.yaml @@ -0,0 +1,150 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: ["data"] + + requester: + url_base: "https://connect.mailerlite.com/api" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['api_token'] }}" + + increment_paginator: + type: "DefaultPaginator" + url_base: "*ref(definitions.requester.url_base)" + page_size_option: + inject_into: "request_parameter" + field_name: "limit" + pagination_strategy: + type: "PageIncrement" + page_size: 25 + page_token_option: + inject_into: "request_parameter" + field_name: "page" + + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + requester: + $ref: "*ref(definitions.requester)" + + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + + # STREAMS + # API Docs: https://developers.mailerlite.com/docs/subscribers.html + subscribers_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "subscribers" + primary_key: "id" + path: "/subscribers" + retriever: + $ref: "*ref(definitions.retriever)" + paginator: + $ref: "*ref(definitions.increment_paginator)" + + # API Docs: https://developers.mailerlite.com/docs/segments.html + segments_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "segments" + primary_key: "id" + path: "/segments" + retriever: + $ref: "*ref(definitions.retriever)" + paginator: + $ref: "*ref(definitions.increment_paginator)" + pagination_strategy: + type: "PageIncrement" + page_size: 50 + + # API Docs: https://developers.mailerlite.com/docs/automations.html + automations_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "automations" + primary_key: "id" + path: "/automations" + retriever: + $ref: "*ref(definitions.retriever)" + paginator: + $ref: "*ref(definitions.increment_paginator)" + pagination_strategy: + type: "PageIncrement" + page_size: 10 + + # API Docs: https://developers.mailerlite.com/docs/campaigns.html + campaigns_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "campaigns" + primary_key: "id" + path: "/campaigns" + retriever: + $ref: "*ref(definitions.retriever)" + paginator: + $ref: "*ref(definitions.increment_paginator)" + + # API Docs: https://developers.mailerlite.com/docs/timezones.html + timezones_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "timezones" + primary_key: "id" + path: "/timezones" + + # API Docs: https://developers.mailerlite.com/docs/forms.html + forms_popup_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "forms_popup" + primary_key: "id" + path: "/forms/popup" + retriever: + $ref: "*ref(definitions.retriever)" + paginator: + $ref: "*ref(definitions.increment_paginator)" + + # API Docs: https://developers.mailerlite.com/docs/forms.html + forms_embedded_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "forms_embedded" + primary_key: "id" + path: "/forms/embedded" + retriever: + $ref: "*ref(definitions.retriever)" + paginator: + $ref: "*ref(definitions.increment_paginator)" + + # API Docs: https://developers.mailerlite.com/docs/forms.html + forms_promotion_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "forms_promotion" + primary_key: "id" + path: "/forms/promotion" + retriever: + $ref: "*ref(definitions.retriever)" + paginator: + $ref: "*ref(definitions.increment_paginator)" + +streams: + - "*ref(definitions.subscribers_stream)" + - "*ref(definitions.segments_stream)" + - "*ref(definitions.automations_stream)" + - "*ref(definitions.campaigns_stream)" + - "*ref(definitions.timezones_stream)" + - "*ref(definitions.forms_popup_stream)" + - "*ref(definitions.forms_embedded_stream)" + - "*ref(definitions.forms_promotion_stream)" + +check: + stream_names: ["timezones"] diff --git a/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/automations.json b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/automations.json new file mode 100644 index 000000000000..c7cf008c2d0b --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/automations.json @@ -0,0 +1,559 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "trigger_data": { + "type": "object", + "properties": { + "track_ecommerce": { + "type": "boolean" + }, + "repeatable": { + "type": "boolean" + }, + "group_id": { + "type": "integer" + }, + "exclude_group_ids": { + "type": "array", + "items": { + "type": "integer" + } + }, + "valid": { + "type": "boolean" + } + }, + "required": ["valid"] + }, + "steps": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "parent_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "subject": { + "type": "string" + }, + "from": { + "type": "string" + }, + "from_name": { + "type": "string" + }, + "email_id": { + "type": "string" + }, + "email": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "account_id": { + "type": "string" + }, + "emailable_id": { + "type": "string" + }, + "emailable_type": { + "type": "string" + }, + "type": { + "type": "string" + }, + "from": { + "type": "string" + }, + "from_name": { + "type": "string" + }, + "name": { + "type": "string" + }, + "subject": { + "type": "string" + }, + "plain_text": { + "type": ["string", "null"] + }, + "screenshot_url": { + "type": ["string", "null"] + }, + "preview_url": { + "type": ["string", "null"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + }, + "is_designed": { + "type": "boolean" + }, + "language_id": { + "type": "integer" + }, + "is_winner": { + "type": "boolean" + }, + "stats": { + "type": "object", + "properties": { + "sent": { + "type": "integer" + }, + "opens_count": { + "type": "integer" + }, + "unique_opens_count": { + "type": "integer" + }, + "open_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "clicks_count": { + "type": "integer" + }, + "unique_clicks_count": { + "type": "integer" + }, + "click_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "unsubscribes_count": { + "type": "integer" + }, + "unsubscribe_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "spam_count": { + "type": "integer" + }, + "spam_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "hard_bounces_count": { + "type": "integer" + }, + "hard_bounce_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "soft_bounces_count": { + "type": "integer" + }, + "soft_bounce_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "forwards_count": { + "type": "integer" + } + }, + "required": [ + "sent", + "opens_count", + "unique_opens_count", + "open_rate", + "clicks_count", + "unique_clicks_count", + "click_rate", + "unsubscribes_count", + "unsubscribe_rate", + "spam_count", + "spam_rate", + "hard_bounces_count", + "hard_bounce_rate", + "soft_bounces_count", + "soft_bounce_rate", + "forwards_count" + ] + }, + "send_after": { + "type": ["string", "null"] + }, + "track_opens": { + "type": "boolean" + } + }, + "required": [ + "id", + "account_id", + "emailable_id", + "emailable_type", + "type", + "from", + "from_name", + "name", + "subject", + "plain_text", + "screenshot_url", + "preview_url", + "created_at", + "updated_at", + "is_designed", + "language_id", + "is_winner", + "stats", + "send_after", + "track_opens" + ] + }, + "language_id": { + "type": "integer" + }, + "complete": { + "type": "boolean" + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + }, + "track_opens": { + "type": "boolean" + }, + "google_analytics": { + "type": ["string", "null"] + }, + "tracking_was_disabled": { + "type": "boolean" + }, + "description": { + "type": "string" + } + }, + "required": ["id"] + } + }, + "triggers": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "group_id": { + "type": "string" + }, + "group": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "url": { + "type": ["string", "null"] + } + }, + "required": ["id", "name", "url"] + }, + "exclude_group_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "excluded_groups": { + "type": "array", + "items": { + "type": "string" + } + }, + "broken": { + "type": "boolean" + } + }, + "required": [ + "id", + "type", + "group_id", + "group", + "exclude_group_ids", + "excluded_groups", + "broken" + ] + } + }, + "complete": { + "type": "boolean" + }, + "broken": { + "type": "boolean" + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } + }, + "emails_count": { + "type": "integer" + }, + "first_email_screenshot_url": { + "type": ["string", "null"] + }, + "stats": { + "type": "object", + "properties": { + "completed_subscribers_count": { + "type": "integer" + }, + "subscribers_in_queue_count": { + "type": "integer" + }, + "bounce_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "click_to_open_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "sent": { + "type": "integer" + }, + "opens_count": { + "type": "integer" + }, + "unique_opens_count": { + "type": ["integer", "null"] + }, + "open_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "clicks_count": { + "type": "integer" + }, + "unique_clicks_count": { + "type": ["integer", "null"] + }, + "click_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "unsubscribes_count": { + "type": "integer" + }, + "unsubscribe_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "spam_count": { + "type": "integer" + }, + "spam_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "hard_bounces_count": { + "type": "integer" + }, + "hard_bounce_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "soft_bounces_count": { + "type": "integer" + }, + "soft_bounce_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + } + }, + "required": [ + "completed_subscribers_count", + "subscribers_in_queue_count", + "bounce_rate", + "click_to_open_rate", + "sent", + "opens_count", + "unique_opens_count", + "open_rate", + "clicks_count", + "unique_clicks_count", + "click_rate", + "unsubscribes_count", + "unsubscribe_rate", + "spam_count", + "spam_rate", + "hard_bounces_count", + "hard_bounce_rate", + "soft_bounces_count", + "soft_bounce_rate" + ] + }, + "created_at": { + "type": "string" + }, + "has_banned_content": { + "type": "boolean" + }, + "qualified_subscribers_count": { + "type": "integer" + } + }, + "required": [ + "id", + "name", + "enabled", + "trigger_data", + "steps", + "triggers", + "complete", + "broken", + "warnings", + "emails_count", + "first_email_screenshot_url", + "stats", + "created_at", + "has_banned_content", + "qualified_subscribers_count" + ] +} diff --git a/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/campaigns.json b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/campaigns.json new file mode 100644 index 000000000000..b6ad9a125a0c --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/campaigns.json @@ -0,0 +1,567 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "integer" + }, + "account_id": { + "type": "string", + "format": "integer" + }, + "name": { + "type": "string" + }, + "type": { + "type": "string" + }, + "status": { + "type": "string" + }, + "missing_data": { + "type": "array", + "items": { + "type": "string" + } + }, + "settings": { + "type": "object", + "properties": { + "track_opens": { + "type": "string", + "format": "integer" + }, + "use_google_analytics": { + "type": "string", + "format": "integer" + }, + "ecommerce_tracking": { + "type": "string", + "format": "integer" + } + }, + "required": ["ecommerce_tracking", "track_opens", "use_google_analytics"] + }, + "filter": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "operator": { + "type": "string" + }, + "args": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string", + "format": "integer" + } + }, + { + "type": "string" + } + ] + } + } + }, + "required": ["args", "operator"] + } + } + }, + "filter_for_humans": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "delivery_schedule": { + "type": "string" + }, + "language_id": { + "type": "string", + "format": "integer" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "updated_at": { + "type": "string", + "format": "date-time" + }, + "scheduled_for": { + "type": "string", + "format": "date-time" + }, + "queued_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "started_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "finished_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "stopped_at": { + "type": ["string", "null"] + }, + "default_email_id": { + "type": "string" + }, + "emails": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "integer" + }, + "account_id": { + "type": "string", + "format": "integer" + }, + "emailable_id": { + "type": "string", + "format": "integer" + }, + "emailable_type": { + "type": "string" + }, + "type": { + "type": "string" + }, + "from": { + "type": "string" + }, + "from_name": { + "type": "string" + }, + "name": { + "type": ["string", "null"] + }, + "subject": { + "type": "string" + }, + "plain_text": { + "type": "string" + }, + "screenshot_url": { + "type": ["string", "null"] + }, + "preview_url": { + "type": ["string", "null"] + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "updated_at": { + "type": "string", + "format": "date-time" + }, + "is_designed": { + "type": "boolean" + }, + "language_id": { + "type": ["integer", "null"] + }, + "is_winner": { + "type": "boolean" + }, + "stats": { + "type": "object", + "properties": { + "sent": { + "type": "integer" + }, + "opens_count": { + "type": "integer" + }, + "unique_opens_count": { + "type": "integer" + }, + "open_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "clicks_count": { + "type": "integer" + }, + "unique_clicks_count": { + "type": "integer" + }, + "click_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "unsubscribes_count": { + "type": "integer" + }, + "unsubscribe_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "spam_count": { + "type": "integer" + }, + "spam_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "hard_bounces_count": { + "type": "integer" + }, + "hard_bounce_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "soft_bounces_count": { + "type": "integer" + }, + "soft_bounce_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "forwards_count": { + "type": "integer" + }, + "click_to_open_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + } + }, + "required": [ + "click_rate", + "clicks_count", + "forwards_count", + "hard_bounce_rate", + "hard_bounces_count", + "open_rate", + "opens_count", + "sent", + "soft_bounce_rate", + "soft_bounces_count", + "spam_count", + "spam_rate", + "unique_clicks_count", + "unique_opens_count", + "unsubscribe_rate", + "unsubscribes_count" + ] + }, + "send_after": { + "type": ["string", "null"] + }, + "track_opens": { + "type": "boolean" + } + }, + "required": [ + "account_id", + "created_at", + "emailable_id", + "emailable_type", + "from", + "from_name", + "id", + "is_designed", + "is_winner", + "language_id", + "name", + "plain_text", + "preview_url", + "screenshot_url", + "send_after", + "stats", + "subject", + "track_opens", + "type", + "updated_at" + ] + } + }, + "used_in_automations": { + "type": "boolean" + }, + "type_for_humans": { + "type": "string" + }, + "stats": { + "type": "object", + "properties": { + "sent": { + "type": "integer" + }, + "opens_count": { + "type": "integer" + }, + "unique_opens_count": { + "type": "integer" + }, + "open_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "clicks_count": { + "type": "integer" + }, + "unique_clicks_count": { + "type": "integer" + }, + "click_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "unsubscribes_count": { + "type": "integer" + }, + "unsubscribe_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "spam_count": { + "type": "integer" + }, + "spam_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "hard_bounces_count": { + "type": "integer" + }, + "hard_bounce_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "soft_bounces_count": { + "type": "integer" + }, + "soft_bounce_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "forwards_count": { + "type": "integer" + }, + "click_to_open_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + } + }, + "required": [ + "click_rate", + "clicks_count", + "forwards_count", + "hard_bounce_rate", + "hard_bounces_count", + "open_rate", + "opens_count", + "sent", + "soft_bounce_rate", + "soft_bounces_count", + "spam_count", + "spam_rate", + "unique_clicks_count", + "unique_opens_count", + "unsubscribe_rate", + "unsubscribes_count" + ] + }, + "is_stopped": { + "type": "boolean" + }, + "has_winner": { + "type": ["string", "null"] + }, + "winner_version_for_human": { + "type": ["string", "null"] + }, + "winner_sending_time_for_humans": { + "type": ["string", "null"] + }, + "winner_selected_manually_at": { + "type": ["string", "null"] + }, + "uses_ecommerce": { + "type": "boolean" + }, + "uses_survey": { + "type": "boolean" + }, + "can_be_scheduled": { + "type": "boolean" + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } + }, + "initial_created_at": { + "type": ["string", "null"] + }, + "is_currently_sending_out": { + "type": "boolean" + } + }, + "required": [ + "account_id", + "can_be_scheduled", + "created_at", + "default_email_id", + "delivery_schedule", + "emails", + "filter", + "filter_for_humans", + "finished_at", + "has_winner", + "id", + "initial_created_at", + "is_currently_sending_out", + "is_stopped", + "language_id", + "missing_data", + "name", + "queued_at", + "scheduled_for", + "settings", + "started_at", + "status", + "stopped_at", + "type", + "type_for_humans", + "updated_at", + "used_in_automations", + "uses_ecommerce", + "uses_survey", + "warnings", + "winner_selected_manually_at", + "winner_sending_time_for_humans", + "winner_version_for_human" + ] +} diff --git a/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/forms_embedded.json b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/forms_embedded.json new file mode 100644 index 000000000000..0a9a1749a10e --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/forms_embedded.json @@ -0,0 +1,191 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "slug": { + "type": "string" + }, + "name": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "conversions_count": { + "type": "integer" + }, + "opens_count": { + "type": "integer" + }, + "conversion_rate": { + "type": "object", + "properties": { + "float": { + "type": "integer" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "settings": { + "type": "object", + "properties": { + "double_optin": { + "type": "boolean" + }, + "groot_id": { + "type": "integer" + } + }, + "required": ["double_optin", "groot_id"] + }, + "last_registration_at": { + "type": ["null", "string"] + }, + "active": { + "type": "boolean" + }, + "is_broken": { + "type": "boolean" + }, + "has_content": { + "type": "boolean" + }, + "can": { + "type": "object", + "properties": { + "update": { + "type": "boolean" + } + }, + "required": ["update"] + }, + "used_in_automations": { + "type": "boolean" + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } + }, + "double_optin": { + "type": "boolean" + }, + "screenshot_url": { + "type": "string" + }, + "has_missing_groups": { + "type": "boolean" + }, + "groups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "active_count": { + "type": "integer" + }, + "sent_count": { + "type": "integer" + }, + "opens_count": { + "type": "integer" + }, + "open_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "clicks_count": { + "type": "integer" + }, + "click_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "unsubscribed_count": { + "type": "integer" + }, + "unconfirmed_count": { + "type": "integer" + }, + "bounced_count": { + "type": "integer" + }, + "junk_count": { + "type": "integer" + }, + "created_at": { + "type": "string" + } + }, + "required": [ + "id", + "name", + "active_count", + "sent_count", + "opens_count", + "open_rate", + "clicks_count", + "click_rate", + "unsubscribed_count", + "unconfirmed_count", + "bounced_count", + "junk_count", + "created_at" + ] + } + } + }, + "required": [ + "id", + "type", + "slug", + "name", + "created_at", + "conversions_count", + "opens_count", + "conversion_rate", + "settings", + "last_registration_at", + "active", + "is_broken", + "has_content", + "can", + "used_in_automations", + "warnings", + "double_optin", + "screenshot_url", + "has_missing_groups", + "groups" + ] +} diff --git a/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/forms_popup.json b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/forms_popup.json new file mode 100644 index 000000000000..fa12ba28badc --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/forms_popup.json @@ -0,0 +1,245 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "slug": { + "type": "string" + }, + "name": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "conversions_count": { + "type": "integer" + }, + "opens_count": { + "type": "integer" + }, + "conversion_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "settings": { + "type": "object", + "properties": { + "double_optin": { + "type": "boolean" + }, + "groot_id": { + "type": "integer" + }, + "form_type": { + "type": "string" + }, + "triggers": { + "type": "array", + "items": { + "type": "string" + } + }, + "timeout_seconds": { + "type": "integer" + }, + "scroll_percentage": { + "type": "integer" + }, + "frequency": { + "type": "integer" + }, + "frequency_unit": { + "type": "string" + }, + "visibility": { + "type": "string" + }, + "url_list": { + "type": "array", + "items": { + "type": "string" + } + }, + "url_list_strict": { + "type": ["null", "string"] + }, + "hide_on": { + "type": "array", + "items": { + "type": "string" + } + }, + "schedule": { + "type": "string" + } + }, + "required": [ + "double_optin", + "groot_id", + "form_type", + "triggers", + "timeout_seconds", + "scroll_percentage", + "frequency", + "frequency_unit", + "visibility", + "url_list", + "url_list_strict", + "hide_on", + "schedule" + ] + }, + "last_registration_at": { + "type": ["string", "null"] + }, + "active": { + "type": "boolean" + }, + "is_broken": { + "type": "boolean" + }, + "has_content": { + "type": "boolean" + }, + "can": { + "type": "object", + "properties": { + "update": { + "type": "boolean" + } + }, + "required": ["update"] + }, + "used_in_automations": { + "type": "boolean" + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } + }, + "double_optin": { + "type": ["boolean", "null"] + }, + "screenshot_url": { + "type": ["string", "null"] + }, + "has_missing_groups": { + "type": "boolean" + }, + "groups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "active_count": { + "type": "integer" + }, + "sent_count": { + "type": "integer" + }, + "opens_count": { + "type": "integer" + }, + "open_rate": { + "type": "object", + "properties": { + "float": { + "type": "integer" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "clicks_count": { + "type": "integer" + }, + "click_rate": { + "type": "object", + "properties": { + "float": { + "type": "integer" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "unsubscribed_count": { + "type": "integer" + }, + "unconfirmed_count": { + "type": "integer" + }, + "bounced_count": { + "type": "integer" + }, + "junk_count": { + "type": "integer" + }, + "created_at": { + "type": "string" + } + }, + "required": [ + "id", + "name", + "active_count", + "sent_count", + "opens_count", + "open_rate", + "clicks_count", + "click_rate", + "unsubscribed_count", + "unconfirmed_count", + "bounced_count", + "junk_count", + "created_at" + ] + } + } + }, + "required": [ + "id", + "type", + "slug", + "name", + "created_at", + "conversions_count", + "opens_count", + "conversion_rate", + "settings", + "last_registration_at", + "active", + "is_broken", + "has_content", + "can", + "used_in_automations", + "warnings", + "double_optin", + "screenshot_url" + ] +} diff --git a/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/forms_promotion.json b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/forms_promotion.json new file mode 100644 index 000000000000..501d8d238114 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/forms_promotion.json @@ -0,0 +1,185 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "slug": { + "type": "string" + }, + "name": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "conversions_count": { + "type": "integer" + }, + "opens_count": { + "type": "integer" + }, + "conversion_rate": { + "type": "object", + "properties": { + "float": { + "type": "integer" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "settings": { + "type": "object", + "properties": { + "groot_id": { + "type": "integer" + }, + "form_type": { + "type": "string" + }, + "triggers": { + "type": "array", + "items": { + "type": "string" + } + }, + "timeout_seconds": { + "type": "integer" + }, + "scroll_percentage": { + "type": "integer" + }, + "frequency": { + "type": "integer" + }, + "frequency_unit": { + "type": "string" + }, + "visibility": { + "type": "string" + }, + "url_list": { + "type": "array", + "items": { + "type": "string" + } + }, + "url_list_strict": { + "type": ["array", "null"], + "items": { + "type": "string" + } + }, + "hide_on": { + "type": "array", + "items": { + "type": "string" + } + }, + "schedule": { + "type": "string" + }, + "schedule_from": { + "type": "string" + }, + "schedule_until": { + "type": "string" + }, + "schedule_timezone_id": { + "type": "string" + } + }, + "required": [ + "groot_id", + "form_type", + "triggers", + "timeout_seconds", + "scroll_percentage", + "frequency", + "frequency_unit", + "visibility", + "url_list", + "url_list_strict", + "hide_on", + "schedule", + "schedule_from", + "schedule_until", + "schedule_timezone_id" + ] + }, + "last_registration_at": { + "type": ["string", "null"] + }, + "active": { + "type": "boolean" + }, + "is_broken": { + "type": "boolean" + }, + "has_content": { + "type": "boolean" + }, + "can": { + "type": "object", + "properties": { + "update": { + "type": "boolean" + } + }, + "required": ["update"] + }, + "used_in_automations": { + "type": "boolean" + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } + }, + "double_optin": { + "type": ["string", "null"] + }, + "screenshot_url": { + "type": "string" + }, + "has_missing_groups": { + "type": "boolean" + }, + "groups": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "id", + "type", + "slug", + "name", + "created_at", + "conversions_count", + "opens_count", + "conversion_rate", + "settings", + "last_registration_at", + "active", + "is_broken", + "has_content", + "can", + "used_in_automations", + "warnings", + "double_optin", + "screenshot_url", + "has_missing_groups", + "groups" + ] +} diff --git a/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/segments.json b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/segments.json new file mode 100644 index 000000000000..c743d2b7d085 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/segments.json @@ -0,0 +1,42 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "total": { + "type": "integer" + }, + "open_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "click_rate": { + "type": "object", + "properties": { + "float": { + "type": "number" + }, + "string": { + "type": "string" + } + }, + "required": ["float", "string"] + }, + "created_at": { + "type": "string" + } + }, + "required": ["id", "name", "total", "open_rate", "click_rate", "created_at"] +} diff --git a/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/subscribers.json b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/subscribers.json new file mode 100644 index 000000000000..160b5ae38e96 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/subscribers.json @@ -0,0 +1,111 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "email": { + "type": "string" + }, + "status": { + "type": "string" + }, + "source": { + "type": "string" + }, + "sent": { + "type": "integer" + }, + "opens_count": { + "type": "integer" + }, + "clicks_count": { + "type": "integer" + }, + "open_rate": { + "type": "integer" + }, + "click_rate": { + "type": "integer" + }, + "ip_address": { + "type": ["string", "null"] + }, + "subscribed_at": { + "type": "string" + }, + "unsubscribed_at": { + "type": ["string", "null"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + }, + "fields": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "last_name": { + "type": ["string", "null"] + }, + "company": { + "type": ["string", "null"] + }, + "country": { + "type": ["string", "null"] + }, + "city": { + "type": ["string", "null"] + }, + "phone": { + "type": ["string", "null"] + }, + "state": { + "type": ["string", "null"] + }, + "z_i_p": { + "type": ["string", "null"] + } + }, + "required": [ + "name", + "last_name", + "company", + "country", + "city", + "phone", + "state", + "z_i_p" + ] + }, + "opted_in_at": { + "type": ["string", "null"] + }, + "optin_ip": { + "type": ["string", "null"] + } + }, + "required": [ + "id", + "email", + "status", + "source", + "sent", + "opens_count", + "clicks_count", + "open_rate", + "click_rate", + "ip_address", + "subscribed_at", + "unsubscribed_at", + "created_at", + "updated_at", + "fields", + "opted_in_at", + "optin_ip" + ] +} diff --git a/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/timezones.json b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/timezones.json new file mode 100644 index 000000000000..5d05b42913c3 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/schemas/timezones.json @@ -0,0 +1,21 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "name_for_humans": { + "type": "string" + }, + "offset_name": { + "type": "string" + }, + "offset": { + "type": "integer" + } + }, + "required": ["id", "name", "name_for_humans", "offset_name", "offset"] +} diff --git a/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/source.py b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/source.py new file mode 100644 index 000000000000..fc42b3678c9b --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceMailerlite(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "mailerlite.yaml"}) diff --git a/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/spec.yaml b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/spec.yaml new file mode 100644 index 000000000000..0e54f5173f2f --- /dev/null +++ b/airbyte-integrations/connectors/source-mailerlite/source_mailerlite/spec.yaml @@ -0,0 +1,13 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/mailerlite +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Mailerlite Spec + type: object + required: + - api_token + additionalProperties: true + properties: + api_token: + type: string + description: Your API Token. See here. + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-mailjet-mail/.dockerignore b/airbyte-integrations/connectors/source-mailjet-mail/.dockerignore new file mode 100644 index 000000000000..ebac9a93e9f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_mailjet_mail +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-mailjet-mail/Dockerfile b/airbyte-integrations/connectors/source-mailjet-mail/Dockerfile new file mode 100644 index 000000000000..c65f2c6a7eaf --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_mailjet_mail ./source_mailjet_mail + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-mailjet-mail diff --git a/airbyte-integrations/connectors/source-mailjet-mail/README.md b/airbyte-integrations/connectors/source-mailjet-mail/README.md new file mode 100644 index 000000000000..e2814ba1bb13 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/README.md @@ -0,0 +1,79 @@ +# Mailjet Mail Source + +This is the repository for the Mailjet Mail configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/mailjet-mail). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-mailjet-mail:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/mailjet-mail) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_mailjet_mail/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source mailjet-mail test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-mailjet-mail:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-mailjet-mail:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-mailjet-mail:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailjet-mail:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailjet-mail:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-mailjet-mail:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-mailjet-mail:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-mailjet-mail:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-mailjet-mail/__init__.py b/airbyte-integrations/connectors/source-mailjet-mail/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml new file mode 100644 index 000000000000..b599202d3b9a --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml @@ -0,0 +1,25 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-mailjet-mail:dev +tests: + spec: + - spec_path: "source_mailjet_mail/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.txt" + # extra_fields: no + # exact_order: no + # extra_records: yes + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-mailjet-mail/build.gradle b/airbyte-integrations/connectors/source-mailjet-mail/build.gradle new file mode 100644 index 000000000000..2078df5f5d93 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_mailjet_mail' +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/__init__.py b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/acceptance.py new file mode 100644 index 000000000000..1302b2f57e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..78d2e30219ac --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json @@ -0,0 +1,49 @@ +{ + "streams": [ + { + "stream": { + "name": "contactslist", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "stats_api_lifetime_message", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "contacts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "campaign", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "message", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/invalid_config.json new file mode 100644 index 000000000000..1d6826198820 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "ooo2e2eec3frrrrrrf8", + "api_key_secret": "ppp0d6be348arrrre0ffb" +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json new file mode 100644 index 000000000000..b8356ca5774f --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "2e2eec3e6ab140cf8fb42181773d75da", + "api_key_secret": "0d6be348aaefb42181773d75da3c7608b" +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/main.py b/airbyte-integrations/connectors/source-mailjet-mail/main.py new file mode 100644 index 000000000000..58793767365f --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_mailjet_mail import SourceMailjetMail + +if __name__ == "__main__": + source = SourceMailjetMail() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mailjet-mail/requirements.txt b/airbyte-integrations/connectors/source-mailjet-mail/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-mailjet-mail/setup.py b/airbyte-integrations/connectors/source-mailjet-mail/setup.py new file mode 100644 index 000000000000..9fc525ecd862 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_mailjet_mail", + description="Source implementation for Mailjet Mail.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/__init__.py b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/__init__.py new file mode 100644 index 000000000000..a02451809fe7 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceMailjetMail + +__all__ = ["SourceMailjetMail"] diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/mailjet_mail.yaml b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/mailjet_mail.yaml new file mode 100644 index 000000000000..68833d35f41a --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/mailjet_mail.yaml @@ -0,0 +1,98 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: ["Data"] + requester: + url_base: "https://api.mailjet.com/v3/REST" + http_method: "GET" + authenticator: + type: BasicHttpAuthenticator + username: "{{ config['api_key'] }}" + password: "{{ config['api_key_secret'] }}" + offset_paginator: + type: DefaultPaginator + $options: + url_base: "*ref(definitions.requester.url_base)" + pagination_strategy: + type: "OffsetIncrement" + page_size: 100 + page_token_option: + field_name: "offset" + inject_into: "request_parameter" + page_size_option: + inject_into: "request_parameter" + field_name: "limit" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + requester: + $ref: "*ref(definitions.requester)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + contactslist_stream: + $ref: "*ref(definitions.base_stream)" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + $ref: "*ref(definitions.offset_paginator)" + requester: + $ref: "*ref(definitions.requester)" + $options: + name: "contactslist" + primary_key: "ID" + path: "/contactslist" + contacts_stream: + $ref: "*ref(definitions.base_stream)" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + $ref: "*ref(definitions.offset_paginator)" + requester: + $ref: "*ref(definitions.requester)" + $options: + name: "contacts" + primary_key: "ID" + path: "/contact" + stats_api_lifetime_message_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "stats_api_lifetime_message" + primary_key: "APIKeyID" + path: "/statcounters?CounterSource=APIKey&CounterResolution=Lifetime&CounterTiming=Message" + campaign_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "campaign" + primary_key: "ID" + path: "/campaign" + message_stream: + $ref: "*ref(definitions.base_stream)" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + $ref: "*ref(definitions.offset_paginator)" + requester: + $ref: "*ref(definitions.requester)" + $options: + name: "message" + primary_key: "ID" + path: "/message" + +streams: + - "*ref(definitions.contactslist_stream)" + - "*ref(definitions.contacts_stream)" + - "*ref(definitions.stats_api_lifetime_message_stream)" + - "*ref(definitions.campaign_stream)" + - "*ref(definitions.message_stream)" + +check: + stream_names: + - "contactslist" diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/campaign.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/campaign.json new file mode 100644 index 000000000000..f722bb731bb8 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/campaign.json @@ -0,0 +1,72 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "CampaignType": { + "type": "integer" + }, + "ClickTracked": { + "type": "integer" + }, + "CreatedAt": { + "type": "string" + }, + "CustomValue": { + "type": "string" + }, + "FirstMessageID": { + "type": "integer" + }, + "FromEmail": { + "type": "string" + }, + "FromID": { + "type": "number" + }, + "FromName": { + "type": "string" + }, + "HasHtmlCount": { + "type": "number" + }, + "HasTxtCount": { + "type": "number" + }, + "ID": { + "type": "integer" + }, + "IsDeleted": { + "type": "boolean" + }, + "IsStarred": { + "type": "boolean" + }, + "ListID": { + "type": "integer" + }, + "NewsLetterID": { + "type": "integer" + }, + "OpenTracked": { + "type": "number" + }, + "SendEndAt": { + "type": "string" + }, + "SendStartAt": { + "type": "string" + }, + "SpamassScore": { + "type": "integer" + }, + "Status": { + "type": "integer" + }, + "Subject": { + "type": "string" + }, + "UnsubscribeTrackedCount": { + "type": "integer" + } + } +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contacts.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contacts.json new file mode 100644 index 000000000000..aff93f074c24 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contacts.json @@ -0,0 +1,42 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "CreatedAt": { + "type": "string" + }, + "DeliveredCount": { + "type": "integer" + }, + "Email": { + "type": "string" + }, + "ID": { + "type": "integer" + }, + "IsOptInPending": { + "type": "boolean" + }, + "IsSpamComplaining": { + "type": "boolean" + }, + "LastActivityAt": { + "type": "string" + }, + "LastUpdateAt": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "UnsubscribedAt": { + "type": "string" + }, + "UnsubscribedBy": { + "type": "string" + }, + "IsExcludedFromCampaigns": { + "type": "boolean" + } + } +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contactslist.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contactslist.json new file mode 100644 index 000000000000..8e373f00d21f --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contactslist.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "Address": { + "type": "string" + }, + "CreatedAt": { + "type": "string" + }, + "ID": { + "type": "integer" + }, + "IsDeleted": { + "type": "boolean" + }, + "Name": { + "type": "string" + }, + "SubscriberCount": { + "type": "integer" + } + } +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/message.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/message.json new file mode 100644 index 000000000000..4f70b7c38d55 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/message.json @@ -0,0 +1,72 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ArrivedAt": { + "type": "string" + }, + "AttachmentCount": { + "type": "integer" + }, + "AttemptCount": { + "type": "integer" + }, + "ContactAlt": { + "type": "string" + }, + "ContactID": { + "type": "integer" + }, + "Delay": { + "type": "integer" + }, + "DestinationID": { + "type": "integer" + }, + "FilterTime": { + "type": "integer" + }, + "ID": { + "type": "integer" + }, + "IsClickTracked": { + "type": "boolean" + }, + "IsHTMLPartIncluded": { + "type": "boolean" + }, + "IsOpenTracked": { + "type": "boolean" + }, + "IsTextPartIncluded": { + "type": "boolean" + }, + "IsUnsubTracked": { + "type": "boolean" + }, + "MessageSize": { + "type": "integer" + }, + "SenderID": { + "type": "integer" + }, + "SpamassassinScore": { + "type": "integer" + }, + "SpamassRules": { + "type": "string" + }, + "StatePermanent": { + "type": "boolean" + }, + "Status": { + "type": "string" + }, + "Subject": { + "type": "string" + }, + "UUID": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats_api_lifetime_message.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats_api_lifetime_message.json new file mode 100644 index 000000000000..fdb92709ba89 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats_api_lifetime_message.json @@ -0,0 +1,69 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "APIKeyID": { + "type": "integer" + }, + "EventClickDelay": { + "type": "integer" + }, + "EventClickedCount": { + "type": "integer" + }, + "EventOpenDelay": { + "type": "integer" + }, + "EventOpenedCount": { + "type": "integer" + }, + "EventUnsubscribedCount": { + "type": "integer" + }, + "EventWorkflowExitedCount": { + "type": "integer" + }, + "MessageBlockedCount": { + "type": "integer" + }, + "MessageClickedCount": { + "type": "integer" + }, + "MessageDeferredCount": { + "type": "integer" + }, + "MessageHardBouncedCount": { + "type": "integer" + }, + "MessageOpenedCount": { + "type": "integer" + }, + "MessageQueuedCount": { + "type": "integer" + }, + "MessageSentCount": { + "type": "integer" + }, + "MessageSoftBouncedCount": { + "type": "integer" + }, + "MessageSpamCount": { + "type": "integer" + }, + "MessageUnsubscribedCount": { + "type": "integer" + }, + "MessageWorkFlowExitedCount": { + "type": "integer" + }, + "SourceID": { + "type": "integer" + }, + "Timeslice": { + "type": "string" + }, + "Total": { + "type": "integer" + } + } +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/source.py b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/source.py new file mode 100644 index 000000000000..2f3fa93c6441 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceMailjetMail(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "mailjet_mail.yaml"}) diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml new file mode 100644 index 000000000000..585b8c30c43a --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml @@ -0,0 +1,23 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/mailjet-mail +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Mailjet Mail Spec + type: object + required: + - api_key + - api_key_secret + additionalProperties: true + properties: + api_key: + title: API Key + type: string + description: >- + Your API Key. See here. + api_key_secret: + title: API Secret Key + type: string + description: >- + Your API Secret Key. See here. + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-mailjet-sms/.dockerignore b/airbyte-integrations/connectors/source-mailjet-sms/.dockerignore new file mode 100644 index 000000000000..d78ee20e73f8 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_mailjet_sms +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-mailjet-sms/Dockerfile b/airbyte-integrations/connectors/source-mailjet-sms/Dockerfile new file mode 100644 index 000000000000..3edf019983f5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_mailjet_sms ./source_mailjet_sms + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-mailjet-sms diff --git a/airbyte-integrations/connectors/source-mailjet-sms/README.md b/airbyte-integrations/connectors/source-mailjet-sms/README.md new file mode 100644 index 000000000000..c93c50623f14 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/README.md @@ -0,0 +1,79 @@ +# Mailjet Sms Source + +This is the repository for the Mailjet Sms configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/mailjet-sms). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-mailjet-sms:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/mailjet-sms) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_mailjet_sms/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source mailjet-sms test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-mailjet-sms:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-mailjet-sms:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-mailjet-sms:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailjet-sms:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailjet-sms:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-mailjet-sms:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-mailjet-sms:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-mailjet-sms:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-mailjet-sms/__init__.py b/airbyte-integrations/connectors/source-mailjet-sms/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-mailjet-sms/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mailjet-sms/acceptance-test-config.yml new file mode 100644 index 000000000000..8a87141b3344 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/acceptance-test-config.yml @@ -0,0 +1,27 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-mailjet-sms:dev +tests: + spec: + - spec_path: "source_mailjet_sms/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 30 + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.txt" + # extra_fields: no + # exact_order: no + # extra_records: yes + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 30 diff --git a/airbyte-integrations/connectors/source-mailjet-sms/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-mailjet-sms/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-mailjet-sms/build.gradle b/airbyte-integrations/connectors/source-mailjet-sms/build.gradle new file mode 100644 index 000000000000..6ec77e0faeec --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_mailjet_sms' +} diff --git a/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/__init__.py b/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/acceptance.py new file mode 100644 index 000000000000..1302b2f57e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..8d2564fa27de --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/configured_catalog.json @@ -0,0 +1,13 @@ +{ + "streams": [ + { + "stream": { + "name": "sms", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/invalid_config.json new file mode 100644 index 000000000000..87fa3ed48875 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/invalid_config.json @@ -0,0 +1,5 @@ +{ + "token": "xxxxxxxxxxx", + "start_date": 1666262531, + "end_date": 1666953731 +} diff --git a/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/sample_config.json new file mode 100644 index 000000000000..ed1748cb1b13 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "token": "d36cb74ebe137152e09e4e5ca08d1cfd", + "start_date": 1666262531, + "end_date": 1666953731 +} diff --git a/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-mailjet-sms/main.py b/airbyte-integrations/connectors/source-mailjet-sms/main.py new file mode 100644 index 000000000000..d3cad995e0da --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_mailjet_sms import SourceMailjetSms + +if __name__ == "__main__": + source = SourceMailjetSms() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mailjet-sms/requirements.txt b/airbyte-integrations/connectors/source-mailjet-sms/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-mailjet-sms/setup.py b/airbyte-integrations/connectors/source-mailjet-sms/setup.py new file mode 100644 index 000000000000..1fbf89120862 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_mailjet_sms", + description="Source implementation for Mailjet Sms.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/__init__.py b/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/__init__.py new file mode 100644 index 000000000000..a8199863bdbd --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceMailjetSms + +__all__ = ["SourceMailjetSms"] diff --git a/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/mailjet_sms.yaml b/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/mailjet_sms.yaml new file mode 100644 index 000000000000..b237cbda54a4 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/mailjet_sms.yaml @@ -0,0 +1,59 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: ["Data"] + requester: + url_base: "https://api.mailjet.com/v4" + http_method: "GET" + request_options_provider: + request_parameters: + fromTS: "{{ config['start_date'] }}" + ToTS: "{{ config['end_date'] }}" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['token'] }}" + offset_paginator: + type: DefaultPaginator + $options: + url_base: "*ref(definitions.requester.url_base)" + pagination_strategy: + type: "OffsetIncrement" + page_size: 100 + page_token_option: + field_name: "Offset" + inject_into: "request_parameter" + page_size_option: + inject_into: "request_parameter" + field_name: "Limit" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + requester: + $ref: "*ref(definitions.requester)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + sms_stream: + $ref: "*ref(definitions.base_stream)" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + $ref: "*ref(definitions.offset_paginator)" + requester: + $ref: "*ref(definitions.requester)" + $options: + name: "sms" + primary_key: "ID" + path: "/sms" + +streams: + - "*ref(definitions.sms_stream)" + +check: + stream_names: + - "sms" diff --git a/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/schemas/sms.json b/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/schemas/sms.json new file mode 100644 index 000000000000..055e2b553a56 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/schemas/sms.json @@ -0,0 +1,46 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ID": { + "type": "string" + }, + "From": { + "type": "string" + }, + "To": { + "type": "string" + }, + "Status": { + "type": "object", + "properties": { + "Code": { + "type": "number" + }, + "Name": { + "type": "string" + }, + "Description": { + "type": "string" + } + } + }, + "Cost": { + "type": "object", + "properties": { + "Value": { + "type": "number" + }, + "Currency": { + "type": "string" + } + } + }, + "CreationTS": { + "type": "integer" + }, + "SmsCount": { + "type": "integer" + } + } +} diff --git a/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/source.py b/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/source.py new file mode 100644 index 000000000000..c34b4bc06ba3 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceMailjetSms(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "mailjet_sms.yaml"}) diff --git a/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/spec.yaml b/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/spec.yaml new file mode 100644 index 000000000000..14d4d60f5edf --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-sms/source_mailjet_sms/spec.yaml @@ -0,0 +1,30 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/mailjet-sms +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Mailjet Sms Spec + type: object + required: + - token + additionalProperties: true + properties: + token: + title: Access Token + type: string + description: >- + Your access token. See here. + airbyte_secret: true + start_date: + title: Start date + type: integer + description: Retrieve SMS messages created after the specified timestamp. Required format - Unix timestamp. + pattern: ^[0-9]*$ + examples: + - 1666261656 + end_date: + title: End date + type: integer + description: Retrieve SMS messages created before the specified timestamp. Required format - Unix timestamp. + pattern: ^[0-9]*$ + examples: + - 1666281656 diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile index 8ba8453b09ed..29a82548717a 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.23 +LABEL io.airbyte.version=0.4.24 LABEL io.airbyte.name=airbyte/source-mssql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mssql/Dockerfile b/airbyte-integrations/connectors/source-mssql/Dockerfile index 44c38200c998..1b369f4b119d 100644 --- a/airbyte-integrations/connectors/source-mssql/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.23 +LABEL io.airbyte.version=0.4.24 LABEL io.airbyte.name=airbyte/source-mssql diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile index f0a549848bbb..820305f7665f 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile @@ -16,6 +16,6 @@ ENV APPLICATION source-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.7 +LABEL io.airbyte.version=1.0.8 LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java index ba22c3d1e924..62afb15893f0 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java @@ -303,7 +303,9 @@ void testStrictSSLSecuredWithTunnel() throws Exception { .putIfAbsent(JdbcUtils.SSL_MODE_KEY, Jsons.jsonNode(sslMode)); ((ObjectNode) config).putIfAbsent("tunnel_method", Jsons.jsonNode(tunnelMode)); - final Exception exception = assertThrows(NullPointerException.class, () -> source.check(config)); + final AirbyteConnectionStatus actual = source.check(config); + assertEquals(Status.FAILED, actual.getStatus()); + assertTrue(actual.getMessage().contains("Could not connect with provided SSH configuration.")); } @Test @@ -322,7 +324,9 @@ void testStrictSSLUnsecuredWithTunnel() throws Exception { .putIfAbsent(JdbcUtils.SSL_MODE_KEY, Jsons.jsonNode(sslMode)); ((ObjectNode) config).putIfAbsent("tunnel_method", Jsons.jsonNode(tunnelMode)); - final Exception exception = assertThrows(NullPointerException.class, () -> source.check(config)); + final AirbyteConnectionStatus actual = source.check(config); + assertEquals(Status.FAILED, actual.getStatus()); + assertTrue(actual.getMessage().contains("Could not connect with provided SSH configuration.")); } @Override diff --git a/airbyte-integrations/connectors/source-mysql/Dockerfile b/airbyte-integrations/connectors/source-mysql/Dockerfile index 74587aade9e7..547ac9477d5d 100644 --- a/airbyte-integrations/connectors/source-mysql/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql/Dockerfile @@ -16,6 +16,6 @@ ENV APPLICATION source-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.7 +LABEL io.airbyte.version=1.0.8 LABEL io.airbyte.name=airbyte/source-mysql diff --git a/airbyte-integrations/connectors/source-nasa/.dockerignore b/airbyte-integrations/connectors/source-nasa/.dockerignore new file mode 100644 index 000000000000..9e72055452fa --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_nasa +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-nasa/Dockerfile b/airbyte-integrations/connectors/source-nasa/Dockerfile new file mode 100644 index 000000000000..82be10b93751 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.13-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_nasa ./source_nasa + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-nasa diff --git a/airbyte-integrations/connectors/source-nasa/README.md b/airbyte-integrations/connectors/source-nasa/README.md new file mode 100644 index 000000000000..532154929ece --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/README.md @@ -0,0 +1,132 @@ +# Nasa Source + +This is the repository for the Nasa source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/nasa). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-nasa:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/nasa) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_nasa/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source nasa test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-nasa:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-nasa:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-nasa:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-nasa:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-nasa:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-nasa:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-nasa:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-nasa:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-nasa/acceptance-test-config.yml b/airbyte-integrations/connectors/source-nasa/acceptance-test-config.yml new file mode 100644 index 000000000000..e9fdfb3e37ed --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/acceptance-test-config.yml @@ -0,0 +1,29 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-nasa:dev +tests: + spec: + - spec_path: "source_nasa/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "integration_tests/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + expect_records: + path: "integration_tests/expected_records.txt" + extra_fields: no + exact_order: no + extra_records: yes + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-nasa/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-nasa/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-nasa/build.gradle b/airbyte-integrations/connectors/source-nasa/build.gradle new file mode 100644 index 000000000000..84acad7f37c4 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_nasa' +} diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/__init__.py b/airbyte-integrations/connectors/source-nasa/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-nasa/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..a73baeb404d6 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "nasa_apod": { + "date": "9999-12-31" + } +} diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-nasa/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/config.json b/airbyte-integrations/connectors/source-nasa/integration_tests/config.json new file mode 100644 index 000000000000..1a58cb54281b --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/config.json @@ -0,0 +1,5 @@ +{ + "api_key": "DEMO_KEY", + "start_date": "2022-09-10", + "end_date": "2022-09-15" +} diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-nasa/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..caa03877beac --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/configured_catalog.json @@ -0,0 +1,26 @@ +{ + "streams": [ + { + "stream": { + "name": "nasa_apod", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "date" + ], + "source_defined_primary_key": [ + [ + "date" + ] + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + } + ] +} + diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-nasa/integration_tests/expected_records.txt new file mode 100644 index 000000000000..93467678bace --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/expected_records.txt @@ -0,0 +1,6 @@ +{"stream": "nasa_apod", "data": {"copyright": "Gerardo Ferrarino", "date": "2022-09-10", "explanation": "This 180 degree panoramic night skyscape captures our Milky Way Galaxy as it arcs above the horizon on a winter's night in August. Near midnight, the galactic center is close to the zenith with the clear waters of Lake Traful, Neuquen, Argentina, South America, planet Earth below. Zodiacal light, dust reflected sunlight along the Solar System's ecliptic plane, is also visible in the region's very dark night sky. The faint band of light reaches up from the distant snowy peaks toward the galaxy's center. Follow the arc of the Milky Way to the left to find the southern hemisphere stellar beacons Alpha and Beta Centauri. Close to the horizon bright star Vega is reflected in the calm mountain lake.", "hdurl": "https://apod.nasa.gov/apod/image/2209/Traful-Lake.jpg", "media_type": "image", "service_version": "v1", "title": "Galaxy by the Lake", "url": "https://apod.nasa.gov/apod/image/2209/Traful-Lake1024.jpg"}, "emitted_at": 1666637798520} +{"stream": "nasa_apod", "data": {"date": "2022-09-11", "explanation": "How does your favorite planet spin? Does it spin rapidly around a nearly vertical axis, or horizontally, or backwards? The featured video animates NASA images of all eight planets in our Solar System to show them spinning side-by-side for an easy comparison. In the time-lapse video, a day on Earth -- one Earth rotation -- takes just a few seconds. Jupiter rotates the fastest, while Venus spins not only the slowest (can you see it?), but backwards. The inner rocky planets across the top underwent dramatic spin-altering collisions during the early days of the Solar System. Why planets spin and tilt as they do remains a topic of research with much insight gained from modern computer modeling and the recent discovery and analysis of hundreds of exoplanets: planets orbiting other stars.", "media_type": "video", "service_version": "v1", "title": "Planets of the Solar System: Tilts and Spins", "url": "https://www.youtube.com/embed/my1euFQHH-o?rel=0"}, "emitted_at": 1666637798523} +{"stream": "nasa_apod", "data": {"copyright": "Daniel \u0160\u010derba", "date": "2022-09-12", "explanation": "What are those red filaments in the sky? They are a rarely seen form of lightning confirmed only about 35 years ago: red sprites. Research has shown that following a powerful positive cloud-to-ground lightning strike, red sprites may start as 100-meter balls of ionized air that shoot down from about 80-km high at 10 percent the speed of light. They are quickly followed by a group of upward streaking ionized balls. The featured image was taken late last month from the Jeseniky Mountains in northern Moravia in the Czech Republic. The distance to the red sprites is about 200 kilometers. Red sprites take only a fraction of a second to occur and are best seen when powerful thunderstorms are visible from the side. APOD in world languages: Arabic, Bulgarian, Catalan, Chinese (Beijing), Chinese (Taiwan), Croatian, Czech, Dutch, Farsi, French, French (Canada), German, Hebrew, Indonesian, Japanese, Korean, Montenegrin, Polish, Russian, Serbian, Slovenian, Spanish, Taiwanese, Turkish, and Ukrainian", "hdurl": "https://apod.nasa.gov/apod/image/2209/sprites_scerba_4240.jpg", "media_type": "image", "service_version": "v1", "title": "Red Sprite Lightning over the Czech Republic", "url": "https://apod.nasa.gov/apod/image/2209/sprites_scerba_960.jpg"}, "emitted_at": 1666637798524} +{"stream": "nasa_apod", "data": {"copyright": "Alan FriedmanAverted Imagination", "date": "2022-09-13", "explanation": "rlier this month, the Sun exhibited one of the longer filaments on record. Visible as the bright curving streak around the image center, the snaking filament's full extent was estimated to be over half of the Sun's radius -- more than 350,000 kilometers long. A filament is composed of hot gas held aloft by the Sun's magnetic field, so that viewed from the side it would appear as a raised prominence. A different, smaller prominence is simultaneously visible at the Sun's edge. The featured image is in false-color and color-inverted to highlight not only the filament but the Sun's carpet chromosphere. The bright dot on the upper right is actually a dark sunspot about the size of the Earth. Solar filaments typically last from hours to days, eventually collapsing to return hot plasma back to the Sun. Sometimes, though, they explode and expel particles into the Solar System, some of which trigger auroras on Earth. The pictured filament appeared in early September and continued to hold steady for about a week.", "hdurl": "https://apod.nasa.gov/apod/image/2209/SnakingFilament_Friedman_960.jpg", "media_type": "image", "service_version": "v1", "title": "A Long Snaking Filament on the Sun", "url": "https://apod.nasa.gov/apod/image/2209/SnakingFilament_Friedman_960.jpg"}, "emitted_at": 1666637798524} +{"stream": "nasa_apod", "data": {"copyright": "Jarmo Ruuth Text: Ata SarajediniFlorida Atlantic U.Astronomy Minute", "date": "2022-09-14", "explanation": "It is one of the largest nebulas on the sky -- why isn't it better known? Roughly the same angular size as the Andromeda Galaxy, the Great Lacerta Nebula can be found toward the constellation of the Lizard (Lacerta). The emission nebula is difficult to see with wide-field binoculars because it is so faint, but also usually difficult to see with a large telescope because it is so great in angle -- spanning about three degrees. The depth, breadth, waves, and beauty of the nebula -- cataloged as Sharpless 126 (Sh2-126) -- can best be seen and appreciated with a long duration camera exposure. The featured image is one such combined exposure -- in this case 10 hours over five different colors and over six nights during this past June and July at the IC Astronomy Observatory in Spain. The hydrogen gas in the Great Lacerta Nebula glows red because it is excited by light from the bright star 10 Lacertae, one of the bright blue stars just above the red-glowing nebula's center. The stars and nebula are about 1,200 light years distant. Harvest Full Moon 2022: Notable Submissions to APOD", "hdurl": "https://apod.nasa.gov/apod/image/2209/GreatLacerta_Ruuth_3719.jpg", "media_type": "image", "service_version": "v1", "title": "Waves of the Great Lacerta Nebula", "url": "https://apod.nasa.gov/apod/image/2209/GreatLacerta_Ruuth_960.jpg"}, "emitted_at": 1666637798524} +{"stream": "nasa_apod", "data": {"copyright": "Dario Giannobile", "date": "2022-09-15", "explanation": "For northern hemisphere dwellers, September's Full Moon was the Harvest Moon. Reflecting warm hues at sunset it rises over the historic town of Castiglione di Sicilia in this telephoto view from September 9. Famed in festival, story, and song Harvest Moon is just the traditional name of the full moon nearest the autumnal equinox. According to lore the name is a fitting one. Despite the diminishing daylight hours as the growing season drew to a close, farmers could harvest crops by the light of a full moon shining on from dusk to dawn. Harvest Full Moon 2022: Notable Submissions to APOD", "hdurl": "https://apod.nasa.gov/apod/image/2209/HarvestMoonCastiglioneSicilyLD.jpg", "media_type": "image", "service_version": "v1", "title": "Harvest Moon over Sicily", "url": "https://apod.nasa.gov/apod/image/2209/HarvestMoonCastiglioneSicily1024.jpg"}, "emitted_at": 1666637798525} diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-nasa/integration_tests/invalid_config.json new file mode 100644 index 000000000000..20e5c97d5ff1 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "DEMO_KEY", + "date": "xxx" +} diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-nasa/integration_tests/sample_config.json new file mode 100644 index 000000000000..2f3a9a920871 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "api_key": "DEMO_KEY", + "date": "2022-10-20", + "concept_tags": true +} diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-nasa/integration_tests/sample_state.json new file mode 100644 index 000000000000..6f6300ecb66b --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "nasa-apod": { + "date": "2022-10-15" + } +} diff --git a/airbyte-integrations/connectors/source-nasa/main.py b/airbyte-integrations/connectors/source-nasa/main.py new file mode 100644 index 000000000000..9ed7e7381e18 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_nasa import SourceNasa + +if __name__ == "__main__": + source = SourceNasa() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-nasa/requirements.txt b/airbyte-integrations/connectors/source-nasa/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-nasa/setup.py b/airbyte-integrations/connectors/source-nasa/setup.py new file mode 100644 index 000000000000..bde7a24e5245 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.2", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_nasa", + description="Source implementation for Nasa.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-nasa/source_nasa/__init__.py b/airbyte-integrations/connectors/source-nasa/source_nasa/__init__.py new file mode 100644 index 000000000000..6f155993533b --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/source_nasa/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceNasa + +__all__ = ["SourceNasa"] diff --git a/airbyte-integrations/connectors/source-nasa/source_nasa/schemas/nasa_apod.json b/airbyte-integrations/connectors/source-nasa/source_nasa/schemas/nasa_apod.json new file mode 100644 index 000000000000..fc6ee91c2b8f --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/source_nasa/schemas/nasa_apod.json @@ -0,0 +1,61 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "resource": { + "type": ["null", "object"], + "properties": { + "image_set": { + "type": ["null", "string"] + }, + "planet": { + "type": ["null", "string"] + } + } + }, + "concept_tags": { + "type": ["null", "boolean"] + }, + "title": { + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"], + "format": "%Y-%m-%d" + }, + "url": { + "type": ["null", "string"], + "format": "uri" + }, + "hdurl": { + "type": ["null", "string"], + "format": "uri" + }, + "media_type": { + "type": ["null", "string"], + "enum": ["image", "video"] + }, + "explanation": { + "type": ["null", "string"] + }, + "concepts": { + "type": ["null", "object", "string"], + "patternProperties": { + "^[0-9]+$": { + "type": ["null", "string"] + } + } + }, + "thumbnail_url": { + "type": ["null", "string"], + "format": "uri" + }, + "copyright": { + "type": ["null", "string"] + }, + "service_version": { + "type": ["null", "string"], + "pattern": "^v[0-9]$" + } + } +} diff --git a/airbyte-integrations/connectors/source-nasa/source_nasa/source.py b/airbyte-integrations/connectors/source-nasa/source_nasa/source.py new file mode 100644 index 000000000000..3f47309ab7c3 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/source_nasa/source.py @@ -0,0 +1,209 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from abc import ABC +from datetime import datetime, time, timedelta +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union + +import requests +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import IncrementalMixin, Stream +from airbyte_cdk.sources.streams.http import HttpStream + +date_format = "%Y-%m-%d" + + +class NasaStream(HttpStream, ABC): + + api_key = "api_key" + url_base = "https://api.nasa.gov/" + + def __init__(self, config: Mapping[str, any], **kwargs): + super().__init__() + self.config = config + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {self.api_key: self.config[self.api_key]} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + r = response.json() + if type(r) is dict: + yield r + else: # We got a list + yield from r + + +class NasaApod(NasaStream, IncrementalMixin): + + cursor_field = "date" + primary_key = "date" + start_date_key = "start_date" + end_date_key = "end_date" + + def __init__(self, config: Mapping[str, any], **kwargs): + super().__init__(config) + self.start_date = ( + datetime.strptime(config.pop(self.start_date_key), date_format) if self.start_date_key in config else datetime.now() + ) + self.end_date = datetime.strptime(config.pop(self.end_date_key), date_format) if self.end_date_key in config else datetime.now() + self.sync_mode = SyncMode.full_refresh + self._cursor_value = self.start_date + + @property + def state(self) -> Mapping[str, Any]: + return {self.cursor_field: self._cursor_value.strftime(date_format)} + + @state.setter + def state(self, value: Mapping[str, Any]): + self._cursor_value = datetime.strptime(value[self.cursor_field], date_format) + + def _chunk_date_range(self, start_date: datetime) -> List[Mapping[str, Any]]: + """ + Returns a list of each day between the start date and end date. + The return value is a list of dicts {'date': date_string}. + """ + dates = [] + while start_date <= self.end_date: + dates.append({self.cursor_field: start_date.strftime(date_format)}) + start_date += timedelta(days=1) + return dates + + def stream_slices( + self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + if ( + stream_state + and self.cursor_field in stream_state + and datetime.strptime(stream_state[self.cursor_field], date_format) > self.end_date + ): + return [] + if sync_mode == SyncMode.full_refresh: + return [self.start_date] + + start_date = ( + datetime.strptime(stream_state[self.cursor_field], date_format) + if stream_state and self.cursor_field in stream_state + else self.start_date + ) + return self._chunk_date_range(start_date) + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "planetary/apod" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + request_dict = {**self.config, **super().request_params(stream_state, stream_slice, next_page_token)} + if self.sync_mode == SyncMode.full_refresh: + request_dict[self.start_date_key] = self.start_date.strftime(date_format) + request_dict[self.end_date_key] = self.end_date.strftime(date_format) + else: + request_dict[self.primary_key] = stream_slice[self.cursor_field] + return request_dict + + def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: + self.sync_mode = kwargs.get("sync_mode", SyncMode.full_refresh) + if self._cursor_value and self._cursor_value > self.end_date: + yield [] + else: + for record in super().read_records(*args, **kwargs): + if self._cursor_value: + latest_record_date = datetime.strptime(record[self.cursor_field], date_format) + self._cursor_value = max(self._cursor_value, latest_record_date) + yield record + + +# Source +class SourceNasa(AbstractSource): + + count_key = "count" + start_date_key = "start_date" + end_date_key = "end_date" + min_count_value, max_count_value = 1, 101 + min_date = datetime.strptime("1995-06-16", date_format) + max_date = datetime.combine(datetime.today(), time(0, 0)) + timedelta(days=1) + invalid_conbination_message_template = "Invalid parameter combination. Cannot use {} and {} together." + invalid_parameter_value_template = "Invalid {} value: {}. {}." + invalid_parameter_value_range_template = "The value should be in the range [{},{})" + + def _parse_date(self, date_str: str) -> Union[datetime, str]: + """ + Parses the date string into a datetime object. + + :param date_str: string containing the date according to DATE_FORMAT + :return Union[datetime, str]: str if not correctly formatted or if it does not satify the constraints [self.MIN_DATE, self.MAX_DATE), datetime otherwise. + """ + try: + date = datetime.strptime(date_str, date_format) + if date < self.min_date or date >= self.max_date: + return self.invalid_parameter_value_template.format( + self.date_key, date_str, self.invalid_parameter_value_range_template.format(self.min_date, self.max_date) + ) + else: + return date + except ValueError: + return self.invalid_parameter_value_template.format(self.date_key, date_str, f"It should be formatted as '{date_format}'") + + def check_connection(self, logger, config) -> Tuple[bool, any]: + """ + Verifies that the input configuration supplied by the user can be used to connect to the underlying data source. + + :param config: the user-input config object conforming to the connector's spec.yaml + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + if self.start_date_key in config: + start_date = self._parse_date(config[self.start_date_key]) + if type(start_date) is not datetime: + return False, start_date + + if self.count_key in config: + return False, self.invalid_conbination_message_template.format(self.start_date_key, self.count_key) + + if self.end_date_key in config: + end_date = self._parse_date(config[self.end_date_key]) + if type(end_date) is not datetime: + return False, end_date + + if self.count_key in config: + return False, self.invalid_conbination_message_template.format(self.end_date_key, self.count_key) + + if self.start_date_key not in config: + return False, f"Cannot use {self.end_date_key} without specifying {self.start_date_key}." + + if start_date > end_date: + return False, f"Invalid values. start_date ({start_date}) needs to be lower than or equal to end_date ({end_date})." + + if self.count_key in config: + count_value = config[self.count_key] + if count_value < self.min_count_value or count_value >= self.max_count_value: + return False, self.invalid_parameter_value_template.format( + self.count_key, + count_value, + self.invalid_parameter_value_range_template.format(self.min_count_value, self.max_count_value), + ) + + try: + stream = NasaApod(authenticator=None, config=config) + records = stream.read_records(sync_mode=SyncMode.full_refresh) + next(records) + return True, None + except requests.exceptions.RequestException as e: + return False, e + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + return [NasaApod(authenticator=None, config=config)] diff --git a/airbyte-integrations/connectors/source-nasa/source_nasa/spec.yaml b/airbyte-integrations/connectors/source-nasa/source_nasa/spec.yaml new file mode 100644 index 000000000000..ace88f9ff111 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/source_nasa/spec.yaml @@ -0,0 +1,51 @@ +documentationUrl: https://docs.airbyte.io/integrations/sources/nasa-apod +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: NASA spec + type: object + required: + - api_key + properties: + api_key: + type: string + description: API access key used to retrieve data from the NASA APOD API. + airbyte_secret: true + concept_tags: + type: boolean + default: false + description: >- + Indicates whether concept tags should be returned with the rest of the response. + The concept tags are not necessarily included in the explanation, but rather derived + from common search tags that are associated with the description text. (Better than + just pure text search.) Defaults to False. + count: + type: integer + minimum: 1 + maximum: 100 + description: >- + A positive integer, no greater than 100. If this is specified then `count` randomly + chosen images will be returned in a JSON array. Cannot be used in conjunction with + `date` or `start_date` and `end_date`. + start_date: + type: string + description: >- + Indicates the start of a date range. All images in the range from `start_date` to + `end_date` will be returned in a JSON array. Must be after 1995-06-16, the first day + an APOD picture was posted. There are no images for tomorrow available through this API. + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + examples: + - "2022-10-20" + end_date: + type: string + description: >- + Indicates that end of a date range. If `start_date` is specified without an `end_date` + then `end_date` defaults to the current date. + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + examples: + - "2022-10-20" + thumbs: + type: boolean + default: false + description: >- + Indicates whether the API should return a thumbnail image URL for video files. If set to True, + the API returns URL of video thumbnail. If an APOD is not a video, this parameter is ignored. diff --git a/airbyte-integrations/connectors/source-nasa/unit_tests/__init__.py b/airbyte-integrations/connectors/source-nasa/unit_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-nasa/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-nasa/unit_tests/test_incremental_streams.py new file mode 100644 index 000000000000..be0de14c84bb --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/unit_tests/test_incremental_streams.py @@ -0,0 +1,51 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from datetime import datetime, timedelta + +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_nasa.source import NasaApod + +config = {"api_key": "foobar"} + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(NasaApod, "path", "v0/example_endpoint") + mocker.patch.object(NasaApod, "primary_key", "test_primary_key") + mocker.patch.object(NasaApod, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + stream = NasaApod(config=config) + expected_cursor_field = "date" + assert stream.cursor_field == expected_cursor_field + + +def test_stream_slices(patch_incremental_base_class): + stream = NasaApod(config=config) + start_date = datetime.now() - timedelta(days=3) + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": ["date"], "stream_state": {"date": start_date.strftime("%Y-%m-%d")}} + expected_stream_slice = [{"date": (start_date + timedelta(days=x)).strftime("%Y-%m-%d")} for x in range(4)] + assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + mocker.patch.object(NasaApod, "cursor_field", "dummy_field") + stream = NasaApod(config=config) + assert stream.supports_incremental + + +def test_source_defined_cursor(patch_incremental_base_class): + stream = NasaApod(config=config) + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(patch_incremental_base_class): + stream = NasaApod(config=config) + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-nasa/unit_tests/test_source.py b/airbyte-integrations/connectors/source-nasa/unit_tests/test_source.py new file mode 100644 index 000000000000..49c4570bd888 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/unit_tests/test_source.py @@ -0,0 +1,58 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from datetime import datetime, timedelta +from unittest.mock import MagicMock, patch + +import pytest +from source_nasa.source import NasaApod, SourceNasa + +date_format = "%Y-%m-%d" +min_date = datetime.strptime("1995-06-16", date_format) +tomorrow = SourceNasa().max_date +after_tomorrow_str = (tomorrow + timedelta(days=1)).strftime(date_format) +valid_date_str = (min_date + timedelta(days=10)).strftime(date_format) + + +@pytest.mark.parametrize( + ("config", "expected_return"), + [ + ({"api_key": "foobar"}, (True, None)), + ({"api_key": "foobar", "start_date": valid_date_str}, (True, None)), + ( + {"api_key": "foobar", "start_date": valid_date_str, "count": 5}, + (False, "Invalid parameter combination. Cannot use start_date and count together."), + ), + ( + {"api_key": "foobar", "end_date": valid_date_str, "count": 5}, + (False, "Invalid parameter combination. Cannot use end_date and count together."), + ), + ({"api_key": "foobar", "end_date": valid_date_str}, (False, "Cannot use end_date without specifying start_date.")), + ( + {"api_key": "foobar", "start_date": valid_date_str, "end_date": min_date.strftime(date_format)}, + ( + False, + f"Invalid values. start_date ({datetime.strptime(valid_date_str, date_format)}) needs to be lower than or equal to end_date ({min_date}).", + ), + ), + ({"api_key": "foobar", "start_date": min_date.strftime(date_format), "end_date": valid_date_str}, (True, None)), + ({"api_key": "foobar", "count": 0}, (False, "Invalid count value: 0. The value should be in the range [1,101).")), + ({"api_key": "foobar", "count": 101}, (False, "Invalid count value: 101. The value should be in the range [1,101).")), + ({"api_key": "foobar", "count": 1}, (True, None)), + ], +) +def test_check_connection(mocker, config, expected_return): + with patch.object(NasaApod, "read_records") as mock_http_request: + mock_http_request.return_value = iter([None]) + source = SourceNasa() + logger_mock = MagicMock() + assert source.check_connection(logger_mock, config) == expected_return + + +def test_streams(mocker): + source = SourceNasa() + config_mock = MagicMock() + streams = source.streams(config_mock) + expected_streams_number = 1 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-nasa/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-nasa/unit_tests/test_streams.py new file mode 100644 index 000000000000..0a6ae568ef8c --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/unit_tests/test_streams.py @@ -0,0 +1,80 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from datetime import datetime +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_nasa.source import NasaApod + +api_key_value = "foobar" +config = {"api_key": api_key_value} + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(NasaApod, "path", "v0/example_endpoint") + mocker.patch.object(NasaApod, "primary_key", "test_primary_key") + mocker.patch.object(NasaApod, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = NasaApod(config={**config, "start_date": "2022-09-10"}) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {"api_key": api_key_value, "start_date": "2022-09-10", "end_date": datetime.now().strftime("%Y-%m-%d")} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = NasaApod(config=config) + inputs = {"response": MagicMock()} + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = NasaApod(config=config) + response_object = [{"foo": "bar", "baz": ["qux"]}] + response_mock = MagicMock() + response_mock.configure_mock(**{"json.return_value": response_object}) + inputs = {"response": response_mock} + assert next(stream.parse_response(**inputs)) == response_object[0] + + +def test_request_headers(patch_base_class): + stream = NasaApod(config=config) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = NasaApod(config=config) + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = NasaApod(config=config) + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = NasaApod(config=config) + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-news-api/.dockerignore b/airbyte-integrations/connectors/source-news-api/.dockerignore new file mode 100644 index 000000000000..6b2a7b25bbac --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_news_api +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-news-api/Dockerfile b/airbyte-integrations/connectors/source-news-api/Dockerfile new file mode 100644 index 000000000000..6f01c95de685 --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_news_api ./source_news_api + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-news-api diff --git a/airbyte-integrations/connectors/source-news-api/README.md b/airbyte-integrations/connectors/source-news-api/README.md new file mode 100644 index 000000000000..45ddb2499eeb --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/README.md @@ -0,0 +1,79 @@ +# News Api Source + +This is the repository for the News Api configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/news-api). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-news-api:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/news-api) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_news_api/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source news-api test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-news-api:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-news-api:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-news-api:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-news-api:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-news-api:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-news-api:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-news-api:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-news-api:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-news-api/__init__.py b/airbyte-integrations/connectors/source-news-api/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-news-api/acceptance-test-config.yml b/airbyte-integrations/connectors/source-news-api/acceptance-test-config.yml new file mode 100644 index 000000000000..707b28130dcb --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/acceptance-test-config.yml @@ -0,0 +1,20 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-news-api:dev +tests: + spec: + - spec_path: "source_news_api/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-news-api/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-news-api/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-news-api/build.gradle b/airbyte-integrations/connectors/source-news-api/build.gradle new file mode 100644 index 000000000000..748352a9064e --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_news_api' +} diff --git a/airbyte-integrations/connectors/source-news-api/integration_tests/__init__.py b/airbyte-integrations/connectors/source-news-api/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-news-api/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-news-api/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-news-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-news-api/integration_tests/acceptance.py new file mode 100644 index 000000000000..1302b2f57e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-news-api/integration_tests/catalog.json b/airbyte-integrations/connectors/source-news-api/integration_tests/catalog.json new file mode 100644 index 000000000000..861a9605319a --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/integration_tests/catalog.json @@ -0,0 +1,18 @@ +{ + "streams": [ + { + "name": "everything", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": "publishedAt", + "json_schema": {} + }, + { + "name": "top_headlines", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": "publishedAt", + "json_schema": {} + } + ] +} diff --git a/airbyte-integrations/connectors/source-news-api/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-news-api/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..83f37c070216 --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "everything", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "top_headlines", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-news-api/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-news-api/integration_tests/invalid_config.json new file mode 100644 index 000000000000..aa05f886ebcf --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/integration_tests/invalid_config.json @@ -0,0 +1,8 @@ +{ + "api_key": "YOUR_API_KEY", + "domains": ["bbc.co.uk", "techcrunch.com", "bloomberg.com"], + "category": "business", + "country": "gb", + "sort_by": "publishedAt", + "start_date": "2065-10-21T35:40:00" +} diff --git a/airbyte-integrations/connectors/source-news-api/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-news-api/integration_tests/sample_config.json new file mode 100644 index 000000000000..8620b1c1d87b --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/integration_tests/sample_config.json @@ -0,0 +1,9 @@ +{ + "api_key": "YOUR_API_KEY", + "domains": ["bbc.co.uk", "techcrunch.com", "bloomberg.com"], + "category": "business", + "country": "gb", + "sort_by": "publishedAt", + "start_date": "2022-10-21T14:40:00", + "search_query": "+bitcoin OR +crypto" +} diff --git a/airbyte-integrations/connectors/source-news-api/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-news-api/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-news-api/main.py b/airbyte-integrations/connectors/source-news-api/main.py new file mode 100644 index 000000000000..ff3276306206 --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_news_api import SourceNewsApi + +if __name__ == "__main__": + source = SourceNewsApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-news-api/requirements.txt b/airbyte-integrations/connectors/source-news-api/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-news-api/setup.py b/airbyte-integrations/connectors/source-news-api/setup.py new file mode 100644 index 000000000000..4e5daa9de7c0 --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_news_api", + description="Source implementation for News Api.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-news-api/source_news_api/__init__.py b/airbyte-integrations/connectors/source-news-api/source_news_api/__init__.py new file mode 100644 index 000000000000..8482828261eb --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/source_news_api/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceNewsApi + +__all__ = ["SourceNewsApi"] diff --git a/airbyte-integrations/connectors/source-news-api/source_news_api/news_api.yaml b/airbyte-integrations/connectors/source-news-api/source_news_api/news_api.yaml new file mode 100644 index 000000000000..5550c0d46823 --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/source_news_api/news_api.yaml @@ -0,0 +1,74 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: ["articles"] + requester: + url_base: "https://newsapi.org/v2" + http_method: "GET" + authenticator: + type: ApiKeyAuthenticator + header: "X-Api-Key" + api_token: "{{ config['api_key'] }}" + request_options_provider: + request_parameters: + q: "{{ config['search_query'] }}" + searchIn: "{{ ','.join(config.get('search_in', [])) }}" + sources: "{{ ','.join(config.get('sources', [])) }}" + domains: "{{ ','.join(config.get('domains', [])) }}" + excludeDomains: "{{ ','.join(config.get('exclude_domains', [])) }}" + from: "{{ config['start_date'] }}" + to: "{{ config['end_date'] }}" + language: "{{ config['language'] }}" + sortBy: "{{ config['sort_by'] }}" + + # The following parameters are only added if the use_... option is + # present on the stream. This is because News API does not allow + # these parameters if they're not required for the endpoint. Also, + # these parameters cannot be mixed with the 'sources' parameter. + country: "{{ config['country'] if options['use_country'] is defined and not config.get('sources') else None }}" + category: "{{ config['category'] if options['use_category'] is defined and not config.get('sources') else None }}" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: DefaultPaginator + url_base: "*ref(definitions.requester.url_base)" + page_size_option: + inject_into: "request_parameter" + field_name: "pageSize" + pagination_strategy: + type: PageIncrement + page_size: 100 + page_token_option: + inject_into: "request_parameter" + field_name: "page" + requester: + $ref: "*ref(definitions.requester)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + everything_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "everything" + primary_key: "publishedAt" + path: "/everything" + top_headlines_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "top_headlines" + primary_key: "publishedAt" + path: "/top-headlines" + use_country: true + use_category: true + +streams: + - "*ref(definitions.everything_stream)" + - "*ref(definitions.top_headlines_stream)" + +check: + stream_names: + - "everything" + - "top_headlines" diff --git a/airbyte-integrations/connectors/source-news-api/source_news_api/schemas/everything.json b/airbyte-integrations/connectors/source-news-api/source_news_api/schemas/everything.json new file mode 100644 index 000000000000..46afd67c6412 --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/source_news_api/schemas/everything.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "title": { + "type": ["null", "string"] + }, + "author": { + "type": ["null", "string"] + }, + "publishedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": "object", + "properties": { + "Name": { + "type": ["null", "string"] + }, + "Id": { + "type": ["null", "string"] + } + } + }, + "url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-news-api/source_news_api/schemas/top_headlines.json b/airbyte-integrations/connectors/source-news-api/source_news_api/schemas/top_headlines.json new file mode 100644 index 000000000000..46afd67c6412 --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/source_news_api/schemas/top_headlines.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "title": { + "type": ["null", "string"] + }, + "author": { + "type": ["null", "string"] + }, + "publishedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": "object", + "properties": { + "Name": { + "type": ["null", "string"] + }, + "Id": { + "type": ["null", "string"] + } + } + }, + "url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-news-api/source_news_api/source.py b/airbyte-integrations/connectors/source-news-api/source_news_api/source.py new file mode 100644 index 000000000000..064c0a21fbe6 --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/source_news_api/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceNewsApi(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "news_api.yaml"}) diff --git a/airbyte-integrations/connectors/source-news-api/source_news_api/spec.yaml b/airbyte-integrations/connectors/source-news-api/source_news_api/spec.yaml new file mode 100644 index 000000000000..d6f37ba7c049 --- /dev/null +++ b/airbyte-integrations/connectors/source-news-api/source_news_api/spec.yaml @@ -0,0 +1,185 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/news-api +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: News Api Spec + type: object + required: + - api_key + - country + - category + - sort_by + additionalProperties: true + properties: + api_key: + type: string + description: API Key + airbyte_secret: true + order: 0 + search_query: + type: string + description: | + Search query. See https://newsapi.org/docs/endpoints/everything for + information. + examples: + - "+bitcoin OR +crypto" + - "sunak AND (truss OR johnson)" + order: 1 + search_in: + type: array + description: | + Where to apply search query. Possible values are: title, description, + content. + items: + type: string + enum: + - title + - description + - content + order: 2 + sources: + type: array + description: | + Identifiers (maximum 20) for the news sources or blogs you want + headlines from. Use the `/sources` endpoint to locate these + programmatically or look at the sources index: + https://newsapi.com/sources. Will override both country and category. + items: + type: string + order: 3 + domains: + type: array + description: | + A comma-seperated string of domains (eg bbc.co.uk, techcrunch.com, + engadget.com) to restrict the search to. + items: + type: string + order: 4 + exclude_domains: + type: array + description: | + A comma-seperated string of domains (eg bbc.co.uk, techcrunch.com, + engadget.com) to remove from the results. + items: + type: string + order: 5 + start_date: + type: string + description: | + A date and optional time for the oldest article allowed. This should + be in ISO 8601 format (e.g. 2021-01-01 or 2021-01-01T12:00:00). + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2})?$" + order: 6 + end_date: + type: string + description: | + A date and optional time for the newest article allowed. This should + be in ISO 8601 format (e.g. 2021-01-01 or 2021-01-01T12:00:00). + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2})?$" + order: 7 + language: + type: string + description: | + The 2-letter ISO-639-1 code of the language you want to get headlines + for. Possible options: ar de en es fr he it nl no pt ru se ud zh. + enum: + - ar + - de + - en + - es + - fr + - he + - it + - nl + - no + - pt + - ru + - se + - ud + - zh + order: 8 + country: + type: string + description: | + The 2-letter ISO 3166-1 code of the country you want to get headlines + for. You can't mix this with the sources parameter. + enum: + - ae + - ar + - at + - au + - be + - bg + - br + - ca + - ch + - cn + - co + - cu + - cz + - de + - eg + - fr + - gb + - gr + - hk + - hu + - id + - ie + - il + - in + - it + - jp + - kr + - lt + - lv + - ma + - mx + - my + - ng + - nl + - no + - nz + - ph + - pl + - pt + - ro + - rs + - ru + - sa + - se + - sg + - si + - sk + - th + - tr + - tw + - ua + - us + - ve + - za + default: us + order: 9 + category: + type: string + description: The category you want to get top headlines for. + enum: + - business + - entertainment + - general + - health + - science + - sports + - technology + default: business + order: 10 + sort_by: + type: string + description: | + The order to sort the articles in. Possible options: relevancy, + popularity, publishedAt. + enum: + - relevancy + - popularity + - publishedAt + default: publishedAt + order: 11 diff --git a/airbyte-integrations/connectors/source-oura/.dockerignore b/airbyte-integrations/connectors/source-oura/.dockerignore new file mode 100644 index 000000000000..938727ea015c --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_oura +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-oura/Dockerfile b/airbyte-integrations/connectors/source-oura/Dockerfile new file mode 100644 index 000000000000..7ab5a66e72d4 --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_oura ./source_oura + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-oura diff --git a/airbyte-integrations/connectors/source-oura/README.md b/airbyte-integrations/connectors/source-oura/README.md new file mode 100644 index 000000000000..7029f82d9cae --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/README.md @@ -0,0 +1,79 @@ +# Oura Source + +This is the repository for the Oura configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/oura). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-oura:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/oura) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_oura/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source oura test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-oura:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-oura:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-oura:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-oura:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-oura:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-oura:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-oura:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-oura:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-oura/__init__.py b/airbyte-integrations/connectors/source-oura/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-oura/acceptance-test-config.yml b/airbyte-integrations/connectors/source-oura/acceptance-test-config.yml new file mode 100644 index 000000000000..d28931b81703 --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/acceptance-test-config.yml @@ -0,0 +1,20 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-oura:dev +tests: + spec: + - spec_path: "source_oura/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-oura/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-oura/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-oura/build.gradle b/airbyte-integrations/connectors/source-oura/build.gradle new file mode 100644 index 000000000000..b91b54e0f6ee --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_oura' +} diff --git a/airbyte-integrations/connectors/source-oura/integration_tests/__init__.py b/airbyte-integrations/connectors/source-oura/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-oura/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-oura/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-oura/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-oura/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-oura/integration_tests/catalog.json b/airbyte-integrations/connectors/source-oura/integration_tests/catalog.json new file mode 100644 index 000000000000..97f78c44769d --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/integration_tests/catalog.json @@ -0,0 +1,60 @@ +{ + "streams": [ + { + "name": "daily_activity", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": "timestamp", + "json_schema": {} + }, + { + "name": "daily_readiness", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": "timestamp", + "json_schema": {} + }, + { + "name": "daily_sleep", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": "timestamp", + "json_schema": {} + }, + { + "name": "heart_rate", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": "timestamp", + "json_schema": {} + }, + { + "name": "sessions", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": "start_datetime", + "json_schema": {} + }, + { + "name": "sleep_periods", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": "bedtime_start", + "json_schema": {} + }, + { + "name": "tags", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": "timestamp", + "json_schema": {} + }, + { + "name": "workouts", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": "start_datetime", + "json_schema": {} + } + ] +} diff --git a/airbyte-integrations/connectors/source-oura/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-oura/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..b1d3828eafe6 --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "daily_activity", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "heart_rate", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-oura/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-oura/integration_tests/invalid_config.json new file mode 100644 index 000000000000..99fa9f0cf423 --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/integration_tests/invalid_config.json @@ -0,0 +1,5 @@ +{ + "api_key": "INVALID_API_KEY", + "start_datetime": "2032-10-19T21:12:25Z", + "end_datetime": "2022-10-31T23:59:59Z" +} diff --git a/airbyte-integrations/connectors/source-oura/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-oura/integration_tests/sample_config.json new file mode 100644 index 000000000000..0cc1d58827d3 --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "api_key": "YOUR_API_KEY", + "start_datetime": "2022-10-19T21:12:25Z", + "end_datetime": "2022-10-31T23:59:59Z" +} diff --git a/airbyte-integrations/connectors/source-oura/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-oura/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-oura/main.py b/airbyte-integrations/connectors/source-oura/main.py new file mode 100644 index 000000000000..96826c0c518f --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_oura import SourceOura + +if __name__ == "__main__": + source = SourceOura() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-oura/requirements.txt b/airbyte-integrations/connectors/source-oura/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-oura/setup.py b/airbyte-integrations/connectors/source-oura/setup.py new file mode 100644 index 000000000000..947f2fa770b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.4", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_oura", + description="Source implementation for Oura.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-oura/source_oura/__init__.py b/airbyte-integrations/connectors/source-oura/source_oura/__init__.py new file mode 100644 index 000000000000..aeb9c1b104db --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/source_oura/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceOura + +__all__ = ["SourceOura"] diff --git a/airbyte-integrations/connectors/source-oura/source_oura/oura.yaml b/airbyte-integrations/connectors/source-oura/source_oura/oura.yaml new file mode 100644 index 000000000000..4683195946f8 --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/source_oura/oura.yaml @@ -0,0 +1,118 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: ["data"] + base_requester: + url_base: "https://api.ouraring.com/v2/usercollection" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['api_key'] }}" + date_requester: + $ref: "*ref(definitions.base_requester)" + request_options_provider: + request_parameters: + start_date: "{{ config['start_date'].split('T')[0] }}" + end_date: "{{ config['end_date'].split('T')[0] }}" + datetime_requester: + $ref: "*ref(definitions.base_requester)" + request_options_provider: + request_parameters: + start_datetime: "{{ config['start_datetime'] }}" + end_datetime: "{{ config['end_datetime'] }}" + paginator: + type: DefaultPaginator + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response['next_token'] }}" + page_size: 100 # Not used, but check fails without it + page_token_option: + field_name: "next_token" + inject_into: "request_parameter" + url_base: "*ref(definitions.base_requester.url_base)" + page_size_option: # Not used, but check fails without it + field_name: "" + inject_into: "request_parameter" + base_retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + $ref: "*ref(definitions.paginator)" + date_retriever: + $ref: "*ref(definitions.base_retriever)" + requester: + $ref: "*ref(definitions.date_requester)" + datetime_retriever: + $ref: "*ref(definitions.base_retriever)" + requester: + $ref: "*ref(definitions.datetime_requester)" + date_stream: + retriever: + $ref: "*ref(definitions.date_retriever)" + datetime_stream: + retriever: + $ref: "*ref(definitions.datetime_retriever)" + daily_activity_stream: + $ref: "*ref(definitions.date_stream)" + $options: + name: "daily_activity" + primary_key: "timestamp" + path: "/daily_activity" + daily_readiness_stream: + $ref: "*ref(definitions.date_stream)" + $options: + name: "daily_readiness" + primary_key: "timestamp" + path: "/daily_readiness" + daily_sleep_stream: + $ref: "*ref(definitions.date_stream)" + $options: + name: "daily_sleep" + primary_key: "timestamp" + path: "/daily_sleep" + heart_rate_stream: + $ref: "*ref(definitions.datetime_stream)" + $options: + name: "heart_rate" + primary_key: "timestamp" + path: "/heartrate" + sessions_stream: + $ref: "*ref(definitions.date_stream)" + $options: + name: "sessions" + primary_key: "start_datetime" + path: "/session" + sleep_periods_stream: + $ref: "*ref(definitions.date_stream)" + $options: + name: "sleep_periods" + primary_key: "bedtime_start" + path: "/sleep" + tags_stream: + $ref: "*ref(definitions.date_stream)" + $options: + name: "tags" + primary_key: "timestamp" + path: "/tag" + workouts_stream: + $ref: "*ref(definitions.date_stream)" + $options: + name: "workouts" + primary_key: "start_datetime" + path: "/workout" + +streams: + - "*ref(definitions.daily_activity_stream)" + - "*ref(definitions.daily_readiness_stream)" + - "*ref(definitions.daily_sleep_stream)" + - "*ref(definitions.heart_rate_stream)" + - "*ref(definitions.sessions_stream)" + - "*ref(definitions.sleep_periods_stream)" + - "*ref(definitions.tags_stream)" + - "*ref(definitions.workouts_stream)" + +check: + stream_names: + - "heart_rate" diff --git a/airbyte-integrations/connectors/source-oura/source_oura/schemas/daily_activity.json b/airbyte-integrations/connectors/source-oura/source_oura/schemas/daily_activity.json new file mode 100644 index 000000000000..92fffa40f7fa --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/source_oura/schemas/daily_activity.json @@ -0,0 +1,118 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "class_5_min": { + "type": ["string", "null"] + }, + "score": { + "type": ["number", "null"] + }, + "active_calories": { + "type": ["number", "null"] + }, + "average_met_minutes": { + "type": ["number", "null"] + }, + "contributors": { + "type": "object", + "properties": { + "meet_daily_targets": { + "type": ["number", "null"] + }, + "move_every_hour": { + "type": ["number", "null"] + }, + "recovery_time": { + "type": ["number", "null"] + }, + "stay_active": { + "type": ["number", "null"] + }, + "training_frequency": { + "type": ["number", "null"] + }, + "training_volume": { + "type": ["number", "null"] + } + } + }, + "equivalent_walking_distance": { + "type": ["number", "null"] + }, + "high_activity_met_minutes": { + "type": ["number", "null"] + }, + "high_activity_time": { + "type": ["number", "null"] + }, + "inactivity_alerts": { + "type": ["number", "null"] + }, + "low_activity_met_minutes": { + "type": ["number", "null"] + }, + "low_activity_time": { + "type": ["number", "null"] + }, + "medium_activity_met_minutes": { + "type": ["number", "null"] + }, + "medium_activity_time": { + "type": ["number", "null"] + }, + "met": { + "type": "object", + "properties": { + "interval": { + "type": ["number", "null"] + }, + "items": { + "type": "array", + "items": { + "type": ["number", "null"] + } + }, + "timestamp": { + "type": ["string", "null"], + "format": "date-time" + } + } + }, + "meters_to_target": { + "type": ["number", "null"] + }, + "non_wear_time": { + "type": ["number", "null"] + }, + "resting_time": { + "type": ["number", "null"] + }, + "sedentary_met_minutes": { + "type": ["number", "null"] + }, + "sedentary_time": { + "type": ["number", "null"] + }, + "steps": { + "type": ["number", "null"] + }, + "target_calories": { + "type": ["number", "null"] + }, + "target_meters": { + "type": ["number", "null"] + }, + "total_calories": { + "type": ["number", "null"] + }, + "day": { + "type": ["string", "null"], + "format": "date" + }, + "timestamp": { + "type": ["string", "null"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-oura/source_oura/schemas/daily_readiness.json b/airbyte-integrations/connectors/source-oura/source_oura/schemas/daily_readiness.json new file mode 100644 index 000000000000..22d5b637fe91 --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/source_oura/schemas/daily_readiness.json @@ -0,0 +1,50 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "contributors": { + "type": "object", + "properties": { + "activity_balance": { + "type": ["number", "null"] + }, + "body_temperature": { + "type": ["number", "null"] + }, + "hrv_balance": { + "type": ["number", "null"] + }, + "previous_day_activity": {}, + "previous_night": { + "type": ["number", "null"] + }, + "recovery_index": { + "type": ["number", "null"] + }, + "resting_heart_rate": { + "type": ["number", "null"] + }, + "sleep_balance": { + "type": ["number", "null"] + } + } + }, + "day": { + "type": ["string", "null"], + "format": "date" + }, + "score": { + "type": ["number", "null"] + }, + "temperature_deviation": { + "type": ["number", "null"] + }, + "temperature_trend_deviation": { + "type": ["number", "null"] + }, + "timestamp": { + "type": ["string", "null"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-oura/source_oura/schemas/daily_sleep.json b/airbyte-integrations/connectors/source-oura/source_oura/schemas/daily_sleep.json new file mode 100644 index 000000000000..fd9aaffc0d92 --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/source_oura/schemas/daily_sleep.json @@ -0,0 +1,43 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "contributors": { + "type": "object", + "properties": { + "deep_sleep": { + "type": ["number", "null"] + }, + "efficiency": { + "type": ["number", "null"] + }, + "latency": { + "type": ["number", "null"] + }, + "rem_sleep": { + "type": ["number", "null"] + }, + "restfulness": { + "type": ["number", "null"] + }, + "timing": { + "type": ["number", "null"] + }, + "total_sleep": { + "type": ["number", "null"] + } + } + }, + "day": { + "type": ["string", "null"], + "format": "date" + }, + "score": { + "type": ["number", "null"] + }, + "timestamp": { + "type": ["string", "null"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-oura/source_oura/schemas/heart_rate.json b/airbyte-integrations/connectors/source-oura/source_oura/schemas/heart_rate.json new file mode 100644 index 000000000000..c544fc08b073 --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/source_oura/schemas/heart_rate.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "bpm": { + "type": ["number", "null"] + }, + "source": { + "type": ["string", "null"] + }, + "timestamp": { + "type": ["string", "null"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-oura/source_oura/schemas/sessions.json b/airbyte-integrations/connectors/source-oura/source_oura/schemas/sessions.json new file mode 100644 index 000000000000..b8a45e3f6db7 --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/source_oura/schemas/sessions.json @@ -0,0 +1,42 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "day": { + "type": ["string", "null"], + "format": "date" + }, + "start_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "end_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "type": { + "type": ["string", "null"] + }, + "heart_rate": {}, + "heart_rate_variability": {}, + "mood": {}, + "motion_count": { + "type": "object", + "properties": { + "interval": { + "type": ["number", "null"] + }, + "items": { + "type": "array", + "items": { + "type": ["number", "null"] + } + }, + "timestamp": { + "type": ["string", "null"], + "format": "date-time" + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-oura/source_oura/schemas/sleep_periods.json b/airbyte-integrations/connectors/source-oura/source_oura/schemas/sleep_periods.json new file mode 100644 index 000000000000..fba2a15402f9 --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/source_oura/schemas/sleep_periods.json @@ -0,0 +1,112 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "average_breath": { + "type": ["number", "null"] + }, + "average_heart_rate": { + "type": ["number", "null"] + }, + "average_hrv": { + "type": ["number", "null"] + }, + "awake_time": { + "type": ["number", "null"] + }, + "bedtime_end": { + "type": ["string", "null"], + "format": "date-time" + }, + "bedtime_start": { + "type": ["string", "null"], + "format": "date-time" + }, + "day": { + "type": ["string", "null"], + "format": "date" + }, + "deep_sleep_duration": { + "type": ["number", "null"] + }, + "efficiency": { + "type": ["number", "null"] + }, + "heart_rate": { + "type": "object", + "properties": { + "interval": { + "type": ["number", "null"] + }, + "items": { + "type": "array", + "items": { + "type": ["number", "null"] + } + }, + "timestamp": { + "type": ["string", "null"], + "format": "date-time" + } + } + }, + "hrv": { + "type": "object", + "properties": { + "interval": { + "type": ["number", "null"] + }, + "items": { + "type": "array", + "items": { + "type": ["number", "null"] + } + }, + "timestamp": { + "type": ["string", "null"], + "format": "date-time" + } + } + }, + "latency": { + "type": ["number", "null"] + }, + "light_sleep_duration": { + "type": ["number", "null"] + }, + "low_battery_alert": { + "type": "boolean" + }, + "lowest_heart_rate": { + "type": ["number", "null"] + }, + "movement_30_sec": { + "type": ["string", "null"] + }, + "period": { + "type": ["number", "null"] + }, + "readiness_score_delta": { + "type": ["number", "null"] + }, + "rem_sleep_duration": { + "type": ["number", "null"] + }, + "restless_periods": { + "type": ["number", "null"] + }, + "sleep_phase_5_min": { + "type": ["string", "null"] + }, + "sleep_score_delta": { + "type": ["number", "null"] + }, + "time_in_bed": { + "type": ["number", "null"] + }, + "total_sleep_duration": {}, + "type": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-oura/source_oura/schemas/tags.json b/airbyte-integrations/connectors/source-oura/source_oura/schemas/tags.json new file mode 100644 index 000000000000..a2302b4b54fd --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/source_oura/schemas/tags.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "day": { + "type": ["string", "null"], + "format": "date" + }, + "text": { + "type": ["string", "null"] + }, + "timestamp": { + "type": ["string", "null"], + "format": "date-time" + }, + "tags": { + "type": "array", + "items": { + "type": ["string", "null"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-oura/source_oura/schemas/workouts.json b/airbyte-integrations/connectors/source-oura/source_oura/schemas/workouts.json new file mode 100644 index 000000000000..e67f8e530cfc --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/source_oura/schemas/workouts.json @@ -0,0 +1,34 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "activity": { + "type": ["string", "null"] + }, + "calories": { + "type": ["number", "null"] + }, + "day": { + "type": ["string", "null"], + "format": "date" + }, + "distance": { + "type": ["number", "null"] + }, + "end_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "intensity": { + "type": ["string", "null"] + }, + "label": {}, + "source": { + "type": ["string", "null"] + }, + "start_datetime": { + "type": ["string", "null"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-oura/source_oura/source.py b/airbyte-integrations/connectors/source-oura/source_oura/source.py new file mode 100644 index 000000000000..644f972f110a --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/source_oura/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceOura(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "oura.yaml"}) diff --git a/airbyte-integrations/connectors/source-oura/source_oura/spec.yaml b/airbyte-integrations/connectors/source-oura/source_oura/spec.yaml new file mode 100644 index 000000000000..8e94256a8eb4 --- /dev/null +++ b/airbyte-integrations/connectors/source-oura/source_oura/spec.yaml @@ -0,0 +1,26 @@ +documentationUrl: https://docsurl.com +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Oura Spec + type: object + required: + - api_key + additionalProperties: true + properties: + api_key: + type: string + description: API Key + airbyte_secret: true + order: 0 + start_datetime: + type: string + description: | + Start datetime to sync from. Default is current UTC datetime minus 1 + day. + pattern: ^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(Z|[+-](?:2[0-3]|[01][0-9]):[0-5][0-9])?$ + order: 1 + end_datetime: + type: string + description: End datetime to sync until. Default is current UTC datetime. + pattern: ^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(Z|[+-](?:2[0-3]|[01][0-9]):[0-5][0-9])?$ + order: 2 diff --git a/airbyte-integrations/connectors/source-pokeapi/acceptance-test-config.yml b/airbyte-integrations/connectors/source-pokeapi/acceptance-test-config.yml index 6e648fb7f970..a98c1c1a9a9e 100644 --- a/airbyte-integrations/connectors/source-pokeapi/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-pokeapi/acceptance-test-config.yml @@ -1,9 +1,8 @@ -# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) -# for more information about how to configure these tests connector_image: airbyte/source-pokeapi:dev +test_strictness_level: high acceptance_tests: spec: - bypass_reason: "TODO: activate this test!" + bypass_reason: "The spec is currently invalid: it has additionalProperties set to false" connection: tests: - config_path: "integration_tests/config.json" @@ -19,3 +18,5 @@ acceptance_tests: tests: - config_path: "integration_tests/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + incremental: + bypass_reason: "This connector does not support incremental syncs." diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index 4b79ea19f4b7..4730a7787898 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.19 +LABEL io.airbyte.version=1.0.21 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 029074f82560..fe732c06f6c5 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.19 +LABEL io.airbyte.version=1.0.21 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java index 0facfaadf8f8..ddabe5f727b7 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java @@ -16,6 +16,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.functional.CheckedConsumer; @@ -78,6 +79,7 @@ public class PostgresSource extends AbstractJdbcSource implements Sour private static final int INTERMEDIATE_STATE_EMISSION_FREQUENCY = 10_000; public static final String PARAM_SSLMODE = "sslmode"; + public static final String SSL_MODE = "ssl_mode"; public static final String PARAM_SSL = "ssl"; public static final String PARAM_SSL_TRUE = "true"; public static final String PARAM_SSL_FALSE = "false"; @@ -87,11 +89,13 @@ public class PostgresSource extends AbstractJdbcSource implements Sour public static final String CA_CERTIFICATE_PATH = "ca_certificate_path"; public static final String SSL_KEY = "sslkey"; public static final String SSL_PASSWORD = "sslpassword"; + public static final String MODE = "mode"; static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( "ssl", "true", "sslmode", "require"); private List schemas; private final FeatureFlags featureFlags; + private static final Set INVALID_CDC_SSL_MODES = ImmutableSet.of("allow", "prefer"); public static Source sshWrappedSource() { return new SshWrappedSource(new PostgresSource(), JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY); @@ -464,6 +468,23 @@ public static void main(final String[] args) throws Exception { LOGGER.info("completed source: {}", PostgresSource.class); } + @Override + public AirbyteConnectionStatus check(final JsonNode config) throws Exception { + if (PostgresUtils.isCdc(config)) { + if (config.has(SSL_MODE) && config.get(SSL_MODE).has(MODE)){ + String sslModeValue = config.get(SSL_MODE).get(MODE).asText(); + if (INVALID_CDC_SSL_MODES.contains(sslModeValue)) { + return new AirbyteConnectionStatus() + .withStatus(Status.FAILED) + .withMessage(String.format( + "In CDC replication mode ssl value '%s' is invalid. Please use one of the following SSL modes: disable, require, verify-ca, verify-full", + sslModeValue)); + } + } + } + return super.check(config); + } + @Override protected String toSslJdbcParam(final SslMode sslMode) { return toSslJdbcParamInternal(sslMode); diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java index 2b4cdce416a8..b4b7aec93524 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java @@ -25,6 +25,7 @@ import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.protocol.models.AirbyteCatalog; +import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteStream; import io.airbyte.protocol.models.CatalogHelpers; @@ -35,6 +36,7 @@ import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.math.BigDecimal; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -197,4 +199,34 @@ void testIsCdc() { assertTrue(PostgresUtils.isCdc(config)); } + @Test + void testAllowSSLWithCdcReplicationMethod() throws Exception { + + JsonNode config = getCDCAndSslModeConfig("allow"); + + final AirbyteConnectionStatus actual = new PostgresSource().check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, actual.getStatus()); + assertTrue(actual.getMessage().contains("In CDC replication mode ssl value 'allow' is invalid")); + } + + @Test + void testPreferSSLWithCdcReplicationMethod() throws Exception { + + JsonNode config = getCDCAndSslModeConfig("prefer"); + + final AirbyteConnectionStatus actual = new PostgresSource().check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, actual.getStatus()); + assertTrue(actual.getMessage().contains("In CDC replication mode ssl value 'prefer' is invalid")); + } + + private JsonNode getCDCAndSslModeConfig(String sslMode) { + return Jsons.jsonNode(ImmutableMap.builder() + .put(JdbcUtils.SSL_KEY, true) + .put(JdbcUtils.SSL_MODE_KEY, Map.of(JdbcUtils.MODE_KEY, sslMode)) + .put("replication_method", Map.of("method", "CDC", + "replication_slot", "slot", + "publication", "ab_pub")) + .build()); + } + } diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/.dockerignore b/airbyte-integrations/connectors/source-rd-station-marketing/.dockerignore new file mode 100644 index 000000000000..1ee8b479485f --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_rd_station_marketing +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/Dockerfile b/airbyte-integrations/connectors/source-rd-station-marketing/Dockerfile new file mode 100644 index 000000000000..327eabb2010a --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_rd_station_marketing ./source_rd_station_marketing + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-rd-station-marketing diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/README.md b/airbyte-integrations/connectors/source-rd-station-marketing/README.md new file mode 100644 index 000000000000..8337e8a1f952 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/README.md @@ -0,0 +1,133 @@ +# RD Station Marketing Source + +This is the repository for the RD Station Marketing source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/rd-station-marketing). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python3 -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-rd-station-marketing:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/rd-station-marketing) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_rd_station_marketing/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source rd-station test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-rd-station-marketing:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-rd-station-marketing:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-rd-station-marketing:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-rd-station-marketing:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-rd-station-marketing:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-rd-station-marketing:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). + +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-rd-station-marketing:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-rd-station-marketing:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-rd-station-marketing/acceptance-test-config.yml new file mode 100644 index 000000000000..7315fa50eda0 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/acceptance-test-config.yml @@ -0,0 +1,22 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-rd-station-marketing:dev +tests: + spec: + - spec_path: "source_rd_station_marketing/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + timeout_seconds: 3600 + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 3600 diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-rd-station-marketing/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/bootstrap.md b/airbyte-integrations/connectors/source-rd-station-marketing/bootstrap.md new file mode 100644 index 000000000000..fa71c6f5ca39 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/bootstrap.md @@ -0,0 +1,29 @@ +# RD Station Marketing + +## Overview + +RD Station Marketing is the leading Marketing Automation tool in Latin America. It is a software application that helps your company carry out better campaigns, nurture Leads, generate qualified business opportunities and achieve more results. From social media to email, Landing Pages, Pop-ups, even Automations and Analytics. + +## Authentication + +RD Station Marketing uses Oauth2 to authenticate. To get the credentials, you need first to create an App for private use in this [link](https://appstore.rdstation.com/en/publisher) (needs to be loged in to access). After that, follow [these](https://developers.rdstation.com/reference/autenticacao?lng=en) instructions to create the client_id and client_secret. + +## Endpoints + +There are eleven endpoints in RD Station Marketing Connector: + +- [Analytics Conversions](https://developers.rdstation.com/reference/get_platform-analytics-conversions?lng=en): Responds with conversion statistics for campaings and other marketing assets. +- [Analytics Emails](https://developers.rdstation.com/reference/get_platform-analytics-emails?lng=en): Responds with statistics about the emails sent with this tool. +- [Analytics Funnel](https://developers.rdstation.com/reference/get_platform-analytics-funnel): Responds with the sales funnel for a given period, grouped by day. +- [Analytics Workflow Emails Statistics](https://developers.rdstation.com/reference/get_platform-analytics-workflow-emails): Responds with statistics about emails sent via an automation flow. +- [Emails](https://developers.rdstation.com/reference/get_platform-emails): List all sent emails. +- [Embeddables](https://developers.rdstation.com/reference/get_platform-embeddables): Returns a list of all forms for an account. +- [Fields](https://developers.rdstation.com/reference/get_platform-contacts-fields): Returns all fields, customized and default, and its attributes. +- [Landing Pages](https://developers.rdstation.com/reference/get_platform-landing-pages): Returns a list of all landing pages for an account. +- [Pop-ups](https://developers.rdstation.com/reference/get_platform-popups): Returns a list of all pop-ups for an account. +- [Segmentations](https://developers.rdstation.com/reference/get_platform-segmentations): List all segmentations, custom and default. +- [Workflows](https://developers.rdstation.com/reference/get_platform-workflows): Returns all automation flows. + +## Quick Notes + +- The analytics streams are only supported if you have a Pro or Enterprise RD Station Account. The usage is available only to these plans. diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/build.gradle b/airbyte-integrations/connectors/source-rd-station-marketing/build.gradle new file mode 100644 index 000000000000..171dd1799b82 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_rd_station_marketing' +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/__init__.py b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..0f178109f511 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/abnormal_state.json @@ -0,0 +1,14 @@ +{ + "analytics_emails": { + "send_at": "2217-06-26 21:20:07" + }, + "analytics_funnel": { + "reference_day": "2217-06-26 21:20:07" + }, + "analytics_conversions": { + "asset_updated_at": "2217-06-26 21:20:07" + }, + "analytics_workflow_emails_statistics": { + "asset_updated_at": "2217-06-26 21:20:07" + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..c76a23d355d2 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/configured_catalog.json @@ -0,0 +1,49 @@ +{ + "streams": [ + { + "stream": { + "name": "emails", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "update_time" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "landing_pages", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "update_time" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "segmentations", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "update_time" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/invalid_config.json new file mode 100644 index 000000000000..178618cff3b3 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/invalid_config.json @@ -0,0 +1,11 @@ +{ + "authorization": + { + "auth_type": "Client", + "client_id": "fake-client-id", + "client_secret": "fake-client-secret", + "refresh_token": "fake-refresh-token" + }, + "replication_start_date": "2022-01-01T00:00:00Z", + "all_contacts_segmentation": "9999999999999" +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/sample_config.json new file mode 100644 index 000000000000..5149ad122f2a --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/sample_config.json @@ -0,0 +1,11 @@ +{ + "authorization": + { + "auth_type": "Client", + "client_id": "", + "client_secret": "", + "refresh_token": "" + }, + "replication_start_date": "2022-01-01T00:00:00Z", + "all_contacts_segmentation": "2050455" +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/sample_state.json new file mode 100644 index 000000000000..e72298059ea4 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/sample_state.json @@ -0,0 +1,14 @@ +{ + "analytics_emails": { + "send_at": "2022-06-26 21:20:07" + }, + "analytics_funnel": { + "reference_day": "2022-06-26 21:20:07" + }, + "analytics_conversions": { + "asset_updated_at": "2022-06-26 21:20:07" + }, + "analytics_workflow_emails_statistics": { + "updated_at": "2022-06-26 21:20:07" + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/main.py b/airbyte-integrations/connectors/source-rd-station-marketing/main.py new file mode 100644 index 000000000000..3e89331aff62 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_rd_station_marketing import SourceRDStationMarketing + +if __name__ == "__main__": + source = SourceRDStationMarketing() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/requirements.txt b/airbyte-integrations/connectors/source-rd-station-marketing/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/setup.py b/airbyte-integrations/connectors/source-rd-station-marketing/setup.py new file mode 100644 index 000000000000..b3a9ab68c439 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/setup.py @@ -0,0 +1,31 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.2", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", + "responses~=0.13.3", + "requests-mock", +] + +setup( + name="source_rd_station_marketing", + description="Source implementation for RD Station Marketing.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/__init__.py b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/__init__.py new file mode 100644 index 000000000000..c548f26f7235 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceRDStationMarketing + +__all__ = ["SourceRDStationMarketing"] diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_conversions.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_conversions.json new file mode 100644 index 000000000000..b285d069ca0d --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_conversions.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "asset_id": { + "type": ["null", "integer"] + }, + "asset_identifier": { + "type": ["null", "string"] + }, + "asset_created_at": { + "type": ["null", "string"] + }, + "asset_updated_at": { + "type": ["null", "string"] + }, + "asset_type": { + "type": ["null", "string"] + }, + "conversion_count": { + "type": ["null", "integer"] + }, + "visits_count": { + "type": ["null", "integer"] + }, + "conversion_rate": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_emails.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_emails.json new file mode 100644 index 000000000000..6b74fb2db79b --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_emails.json @@ -0,0 +1,51 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "send_at": { + "type": ["null", "string"] + }, + "campaign_id": { + "type": ["null", "integer"] + }, + "campaign_name": { + "type": ["null", "string"] + }, + "email_dropped_count": { + "type": ["null", "integer"] + }, + "email_delivered_count": { + "type": ["null", "integer"] + }, + "email_bounced_count": { + "type": ["null", "integer"] + }, + "email_opened_count": { + "type": ["null", "integer"] + }, + "email_clicked_count": { + "type": ["null", "integer"] + }, + "email_unsubscribed_count": { + "type": ["null", "integer"] + }, + "email_spam_reported_count": { + "type": ["null", "integer"] + }, + "email_delivered_rate": { + "type": ["null", "number"] + }, + "email_opened_rate": { + "type": ["null", "number"] + }, + "email_clicked_rate": { + "type": ["null", "number"] + }, + "email_spam_reported_rate": { + "type": ["null", "number"] + }, + "contacts_count": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_funnel.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_funnel.json new file mode 100644 index 000000000000..8f144ccfa7de --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_funnel.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "reference_day": { + "type": ["null", "string"] + }, + "contacts_count": { + "type": ["null", "integer"] + }, + "qualified_contacts_count": { + "type": ["null", "integer"] + }, + "opportunities_count": { + "type": ["null", "integer"] + }, + "sales_count": { + "type": ["null", "integer"] + }, + "visitors_count": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_workflow_emails_statistics.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_workflow_emails_statistics.json new file mode 100644 index 000000000000..ffe7866f912d --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_workflow_emails_statistics.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "workflow_name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "email_name": { + "type": ["null", "string"] + }, + "workflow_action_id": { + "type": ["null", "string"] + }, + "workflow_id": { + "type": ["null", "string"] + }, + "contacts_count": { + "type": ["null", "integer"] + }, + "count_processed": { + "type": ["null", "integer"] + }, + "email_delivered_count": { + "type": ["null", "integer"] + }, + "email_opened_unique_count": { + "type": ["null", "integer"] + }, + "email_clicked_unique_count": { + "type": ["null", "integer"] + }, + "email_dropped_count": { + "type": ["null", "integer"] + }, + "email_unsubscribed_count": { + "type": ["null", "integer"] + }, + "email_spam_reported_count": { + "type": ["null", "integer"] + }, + "email_delivered_rate": { + "type": ["null", "number"] + }, + "email_opened_rate": { + "type": ["null", "number"] + }, + "email_clicked_rate": { + "type": ["null", "number"] + }, + "email_spam_reported_rate": { + "type": ["null", "number"] + }, + "email_bounced_unique_count": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/emails.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/emails.json new file mode 100644 index 000000000000..d69b22ee0d19 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/emails.json @@ -0,0 +1,56 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "campaign_id": { + "type": ["null", "integer"] + }, + "behavior_score_info": { + "type": ["null", "object"], + "properties": { + "engaged": { + "type": ["null", "boolean"] + }, + "disengaged": { + "type": ["null", "boolean"] + }, + "indeterminate": { + "type": ["null", "boolean"] + } + } + }, + "send_at": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "sending_is_imminent": { + "type": ["null", "boolean"] + }, + "is_predictive_sending": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "component_template_id": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "leads_count": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/embeddables.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/embeddables.json new file mode 100644 index 000000000000..052bb931c79c --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/embeddables.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "conversion_identifier": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/fields.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/fields.json new file mode 100644 index 000000000000..d740af4b0d0c --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/fields.json @@ -0,0 +1,64 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "uuid": { + "type": ["null", "string"] + }, + "label": { + "type": ["null", "object"], + "properties": { + "en-UD": { + "type": ["null", "string"] + }, + "en-US": { + "type": ["null", "string"] + }, + "es-ES": { + "type": ["null", "string"] + }, + "pt-BR": { + "type": ["null", "string"] + }, + "default": { + "type": ["null", "string"] + } + } + }, + "name": { + "type": ["null", "object"], + "properties": { + "en-UD": { + "type": ["null", "string"] + }, + "en-US": { + "type": ["null", "string"] + }, + "es-ES": { + "type": ["null", "string"] + }, + "pt-BR": { + "type": ["null", "string"] + }, + "default": { + "type": ["null", "string"] + } + } + }, + "api_identifier": { + "type": ["null", "string"] + }, + "custom_field": { + "type": ["null", "boolean"] + }, + "validation_rules": { + "type": ["null", "object"] + }, + "presentation_type": { + "type": ["null", "string"] + }, + "data_type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/landing_pages.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/landing_pages.json new file mode 100644 index 000000000000..dc8e55175ab1 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/landing_pages.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "conversion_identifier": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "has_active_experiment": { + "type": ["null", "boolean"] + }, + "had_experiment": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/popups.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/popups.json new file mode 100644 index 000000000000..250310cc7748 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/popups.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "conversion_identifier": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "trigger": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/segmentations.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/segmentations.json new file mode 100644 index 000000000000..04a2d97658fb --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/segmentations.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "standard": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "process_status": { + "type": ["null", "string"] + }, + "links": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "rel": { + "type": ["null", "string"] + }, + "href": { + "type": ["null", "string"] + }, + "media": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/workflows.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/workflows.json new file mode 100644 index 000000000000..c9bd95a32671 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/workflows.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "user_email_created": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "user_email_updated": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/source.py b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/source.py new file mode 100644 index 000000000000..ee56f5e7d7d6 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/source.py @@ -0,0 +1,74 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from typing import Any, List, Mapping, Tuple + +import pendulum +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator +from source_rd_station_marketing.streams import ( + AnalyticsConversions, + AnalyticsEmails, + AnalyticsFunnel, + AnalyticsWorkflowEmailsStatistics, + Emails, + Embeddables, + Fields, + LandingPages, + Popups, + Segmentations, + Workflows, +) + + +class SourceRDStationMarketing(AbstractSource): + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: + try: + stream_kwargs = self.get_stream_kwargs(config) + segmentations = Segmentations(**stream_kwargs) + segmentations_gen = segmentations.read_records(sync_mode=SyncMode.full_refresh) + next(segmentations_gen) + return True, None + except Exception as error: + return ( + False, + f"Unable to connect to RD Station Marketing API with the provided credentials - {repr(error)}", + ) + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + stream_kwargs = self.get_stream_kwargs(config) + incremental_kwargs = {**stream_kwargs, "start_date": pendulum.parse(config["start_date"])} + streams = [ + AnalyticsEmails(**incremental_kwargs), + AnalyticsConversions(**incremental_kwargs), + AnalyticsFunnel(**incremental_kwargs), + AnalyticsWorkflowEmailsStatistics(**incremental_kwargs), + Emails(**stream_kwargs), + Embeddables(**stream_kwargs), + Fields(**stream_kwargs), + LandingPages(**stream_kwargs), + Popups(**stream_kwargs), + Segmentations(**stream_kwargs), + Workflows(**stream_kwargs), + ] + return streams + + @staticmethod + def get_stream_kwargs(config: Mapping[str, Any]) -> Mapping[str, Any]: + authorization = config.get("authorization", {}) + stream_kwargs = dict() + + stream_kwargs["authenticator"] = Oauth2Authenticator( + token_refresh_endpoint="https://api.rd.services/auth/token", + client_secret=authorization.get("client_secret"), + client_id=authorization.get("client_id"), + refresh_token=authorization.get("refresh_token"), + ) + return stream_kwargs diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/spec.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/spec.json new file mode 100644 index 000000000000..72eeecef2168 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/spec.json @@ -0,0 +1,85 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/rd-station-marketing", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "RD Station Marketing Spec", + "type": "object", + "required": [ + "start_date" + ], + "additionalProperties": true, + "properties": { + "authorization": { + "type": "object", + "title": "Authentication Type", + "description": "Choose one of the possible authorization method", + "oneOf": [ + { + "title": "Sign in via RD Station (OAuth)", + "type": "object", + "required": [ + "auth_type" + ], + "properties": { + "auth_type": { + "type": "string", + "const": "Client", + "order": 0 + }, + "client_id": { + "title": "Client ID", + "type": "string", + "description": "The Client ID of your RD Station developer application.", + "airbyte_secret": true + }, + "client_secret": { + "title": "Client Secret", + "type": "string", + "description": "The Client Secret of your RD Station developer application", + "airbyte_secret": true + }, + "refresh_token": { + "title": "Refresh Token", + "type": "string", + "description": "The token for obtaining the new access token.", + "airbyte_secret": true + } + } + } + ] + }, + "start_date": { + "title": "Start Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. When specified and not None, then stream will behave as incremental", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "examples": [ + "2017-01-25T00:00:00Z" + ], + "type": "string" + } + } + }, + "supportsIncremental": true, + "authSpecification": { + "auth_type": "oauth2.0", + "oauth2Specification": { + "rootObject": [ + "authorization", + 0 + ], + "oauthFlowInitParameters": [ + [ + "client_id" + ], + [ + "client_secret" + ] + ], + "oauthFlowOutputParameters": [ + [ + "refresh_token" + ] + ] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py new file mode 100755 index 000000000000..25bca0799280 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py @@ -0,0 +1,201 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from abc import ABC +from datetime import date +from typing import Any, Iterable, Mapping, MutableMapping, Optional + +import pendulum +import requests +from airbyte_cdk.sources.streams.http import HttpStream + + +class RDStationMarketingStream(HttpStream, ABC): + data_field = None + extra_params = {} + page = 1 + page_size_limit = 125 + primary_key = None + url_base = "https://api.rd.services" + + def __init__(self, authenticator, start_date=None, **kwargs): + super().__init__(authenticator=authenticator, **kwargs) + self._start_date = start_date + + def path(self, **kwargs) -> str: + class_name = self.__class__.__name__ + return f"/platform/{class_name[0].lower()}{class_name[1:]}" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + if self.data_field: + json_response = response.json().get(self.data_field) + else: + json_response = response.json() + if json_response: + self.page = self.page + 1 + return {"next_page": self.page} + else: + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + params = {"page_size": self.page_size_limit, "page": self.page} + if next_page_token: + params = {"page_size": self.page_size_limit, "page": next_page_token["next_page"]} + return params + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + if self.data_field: + records = response.json().get(self.data_field) + else: + records = response.json() + yield from records + + +class IncrementalRDStationMarketingStream(RDStationMarketingStream): + def path(self, **kwargs) -> str: + return f"/platform/analytics/{self.data_field}" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + start_date = self._start_date + + if start_date and stream_state.get(self.cursor_field): + start_date = max(pendulum.parse(stream_state[self.cursor_field]), start_date) + + params = {} + params.update( + { + "start_date": start_date.strftime("%Y-%m-%d"), + "end_date": date.today().strftime("%Y-%m-%d"), + } + ) + + params.update(self.extra_params) + return params + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + latest_benchmark = latest_record[self.cursor_field] + if current_stream_state.get(self.cursor_field): + return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} + return {self.cursor_field: latest_benchmark} + + +class AnalyticsConversions(IncrementalRDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-analytics-conversions + """ + + data_field = "conversions" + cursor_field = "asset_updated_at" + primary_key = "asset_id" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + records = response.json().get(self.data_field)[0].get(self.data_field) + yield from records + + +class AnalyticsEmails(IncrementalRDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-analytics-emails + """ + + data_field = "emails" + cursor_field = "send_at" + primary_key = "campaign_id" + + +class AnalyticsFunnel(IncrementalRDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-analytics-funnel + """ + + data_field = "funnel" + cursor_field = "reference_day" + primary_key = "reference_day" + + +class AnalyticsWorkflowEmailsStatistics(IncrementalRDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-analytics-workflow-emails + """ + + data_field = "workflow_email_statistics" + cursor_field = "updated_at" + primary_key = "workflow_id" + + def path(self, **kwargs) -> str: + return "/platform/analytics/workflow_emails_statistics" + + +class Emails(RDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-emails + """ + + data_field = "items" + primary_key = "id" + + +class Embeddables(RDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-embeddables + """ + + primary_key = "id" + + +class Fields(RDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-contacts-fields + """ + + data_field = "fields" + primary_key = "uuid" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def path(self, **kwargs) -> str: + return "/platform/contacts/fields" + + +class LandingPages(RDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-landing-pages + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "/platform/landing_pages" + + +class Popups(RDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-popups + """ + + primary_key = "id" + + +class Segmentations(RDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-segmentations + """ + + data_field = "segmentations" + primary_key = "id" + + +class Workflows(RDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-workflows + """ + + data_field = "workflows" + primary_key = "id" diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/__init__.py b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_incremental_streams.py new file mode 100644 index 000000000000..6c99b32f1cc0 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_incremental_streams.py @@ -0,0 +1,60 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_rd_station_marketing.streams import IncrementalRDStationMarketingStream + + +@fixture +def test_current_stream_state(): + return {"updated_time": "2021-10-22"} + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(IncrementalRDStationMarketingStream, "path", "v0/example_endpoint") + mocker.patch.object(IncrementalRDStationMarketingStream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalRDStationMarketingStream, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + stream = IncrementalRDStationMarketingStream(authenticator=MagicMock()) + expected_cursor_field = [] + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_base_class, test_current_stream_state, mocker): + mocker.patch.object(IncrementalRDStationMarketingStream, "cursor_field", "updated_time") + stream = IncrementalRDStationMarketingStream(authenticator=MagicMock()) + inputs = {"current_stream_state": test_current_stream_state, "latest_record": test_current_stream_state} + expected_state = {"updated_time": "2021-10-22"} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_stream_slices(patch_incremental_base_class): + stream = IncrementalRDStationMarketingStream(authenticator=MagicMock()) + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + expected_stream_slice = [None] + assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + mocker.patch.object(IncrementalRDStationMarketingStream, "cursor_field", "dummy_field") + stream = IncrementalRDStationMarketingStream(authenticator=MagicMock()) + assert stream.supports_incremental + + +def test_source_defined_cursor(patch_incremental_base_class): + stream = IncrementalRDStationMarketingStream(authenticator=MagicMock()) + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(patch_incremental_base_class): + stream = IncrementalRDStationMarketingStream(authenticator=MagicMock()) + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_source.py new file mode 100644 index 000000000000..9b11721310f0 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_source.py @@ -0,0 +1,71 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +import responses +from pytest import fixture +from source_rd_station_marketing.source import SourceRDStationMarketing + + +@fixture +def test_config(): + return { + "authorization": { + "auth_type": "Client", + "client_id": "test_client_id", + "client_secret": "test_client_secret", + "refresh_token": "test_refresh_token", + }, + "start_date": "2022-01-01T00:00:00Z", + } + + +def setup_responses(): + responses.add( + responses.POST, + "https://api.rd.services/auth/token", + json={"access_token": "fake_access_token", "expires_in": 3600}, + ) + responses.add( + responses.GET, + "https://api.rd.services/platform/segmentations", + json={ + "segmentations": [ + { + "id": 71625167165, + "name": "A mock segmentation", + "standard": True, + "created_at": "2019-09-04T18:05:42.638-03:00", + "updated_at": "2019-09-04T18:05:42.638-03:00", + "process_status": "processed", + "links": [ + { + "rel": "SEGMENTATIONS.CONTACTS", + "href": "https://api.rd.services/platform/segmentations/71625167165/contacts", + "media": "application/json", + "type": "GET", + } + ], + } + ] + }, + ) + + +@responses.activate +def test_check_connection(test_config): + setup_responses() + source = SourceRDStationMarketing() + logger_mock = MagicMock() + assert source.check_connection(logger_mock, test_config) == (True, None) + + +@responses.activate +def test_streams(test_config): + setup_responses() + source = SourceRDStationMarketing() + streams = source.streams(test_config) + expected_streams_number = 11 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_streams.py new file mode 100644 index 000000000000..7f3199465464 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_streams.py @@ -0,0 +1,112 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_rd_station_marketing.streams import RDStationMarketingStream, Segmentations + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(RDStationMarketingStream, "primary_key", "test_primary_key") + mocker.patch.object(RDStationMarketingStream, "__abstractmethods__", set()) + + +def test_path(patch_base_class): + stream = Segmentations(authenticator=MagicMock()) + assert stream.path() == "/platform/segmentations" + + +def test_request_params(patch_base_class): + stream = RDStationMarketingStream(authenticator=MagicMock()) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {"page": 1, "page_size": 125} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = RDStationMarketingStream(authenticator=MagicMock()) + inputs = {"response": MagicMock()} + expected_token = {"next_page": 2} + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = RDStationMarketingStream(authenticator=MagicMock()) + response = MagicMock() + response.json.return_value = [ + { + "id": 71625167165, + "name": "A mock segmentation", + "standard": True, + "created_at": "2019-09-04T18:05:42.638-03:00", + "updated_at": "2019-09-04T18:05:42.638-03:00", + "process_status": "processed", + "links": [ + { + "rel": "SEGMENTATIONS.CONTACTS", + "href": "https://api.rd.services/platform/segmentations/71625167165/contacts", + "media": "application/json", + "type": "GET", + } + ], + } + ] + inputs = {"response": response, "stream_state": None} + expected_parsed_object = { + "id": 71625167165, + "name": "A mock segmentation", + "standard": True, + "created_at": "2019-09-04T18:05:42.638-03:00", + "updated_at": "2019-09-04T18:05:42.638-03:00", + "process_status": "processed", + "links": [ + { + "rel": "SEGMENTATIONS.CONTACTS", + "href": "https://api.rd.services/platform/segmentations/71625167165/contacts", + "media": "application/json", + "type": "GET", + } + ], + } + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = RDStationMarketingStream(authenticator=MagicMock()) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = RDStationMarketingStream(authenticator=MagicMock()) + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = RDStationMarketingStream(authenticator=MagicMock()) + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = RDStationMarketingStream(authenticator=MagicMock()) + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java index c77a61fb9845..738f7eba3193 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java @@ -29,8 +29,8 @@ public class StateDecoratingIterator extends AbstractIterator im private final JsonSchemaPrimitive cursorType; private final String initialCursor; - private String maxCursor; - private long maxCursorRecordCount = 0L; + private String currentMaxCursor; + private long currentMaxCursorRecordCount = 0L; private boolean hasEmittedFinalState; /** @@ -82,7 +82,7 @@ public StateDecoratingIterator(final Iterator messageIterator, this.cursorField = cursorField; this.cursorType = cursorType; this.initialCursor = initialCursor; - this.maxCursor = initialCursor; + this.currentMaxCursor = initialCursor; this.stateEmissionFrequency = stateEmissionFrequency; } @@ -126,17 +126,18 @@ protected AirbyteMessage computeNext() { final AirbyteMessage message = messageIterator.next(); if (message.getRecord().getData().hasNonNull(cursorField)) { final String cursorCandidate = getCursorCandidate(message); - final int cursorComparison = IncrementalUtils.compareCursors(maxCursor, cursorCandidate, cursorType); + final int cursorComparison = IncrementalUtils.compareCursors(currentMaxCursor, cursorCandidate, cursorType); if (cursorComparison < 0) { - if (stateEmissionFrequency > 0 && !Objects.equals(maxCursor, initialCursor) && messageIterator.hasNext()) { + // Update the current max cursor only when current max cursor < cursor candidate from the message + if (stateEmissionFrequency > 0 && !Objects.equals(currentMaxCursor, initialCursor) && messageIterator.hasNext()) { // Only emit an intermediate state when it is not the first or last record message, // because the last state message will be taken care of in a different branch. intermediateStateMessage = createStateMessage(false); } - maxCursor = cursorCandidate; - maxCursorRecordCount = 1L; + currentMaxCursor = cursorCandidate; + currentMaxCursorRecordCount = 1L; } else if (cursorComparison == 0) { - maxCursorRecordCount++; + currentMaxCursorRecordCount++; } } @@ -188,7 +189,7 @@ protected final Optional getIntermediateMessage() { * @return AirbyteMessage which includes information on state of records read so far */ public AirbyteMessage createStateMessage(final boolean isFinalState) { - final AirbyteStateMessage stateMessage = stateManager.updateAndEmit(pair, maxCursor, maxCursorRecordCount); + final AirbyteStateMessage stateMessage = stateManager.updateAndEmit(pair, currentMaxCursor, currentMaxCursorRecordCount); final Optional cursorInfo = stateManager.getCursorInfo(pair); LOGGER.info("State report for stream {} - original: {} = {} (count {}) -> latest: {} = {} (count {})", pair, diff --git a/airbyte-integrations/connectors/source-s3/Dockerfile b/airbyte-integrations/connectors/source-s3/Dockerfile index c61095cc1484..2413c0c799b7 100644 --- a/airbyte-integrations/connectors/source-s3/Dockerfile +++ b/airbyte-integrations/connectors/source-s3/Dockerfile @@ -17,5 +17,5 @@ COPY source_s3 ./source_s3 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.24 +LABEL io.airbyte.version=0.1.25 LABEL io.airbyte.name=airbyte/source-s3 diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json b/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json index c5b35c593f9d..6bf9fc09c36d 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json +++ b/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json @@ -6,7 +6,7 @@ "aws_access_key_id": "123456", "aws_secret_access_key": "123456key", "path_prefix": "", - "endpoint": "http://10.0.92.4:9000" + "endpoint": "http://10.0.56.135:9000" }, "format": { "filetype": "csv" diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/integration_test_abstract.py b/airbyte-integrations/connectors/source-s3/integration_tests/integration_test_abstract.py index 2885a9662444..91283a032aa5 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/integration_test_abstract.py +++ b/airbyte-integrations/connectors/source-s3/integration_tests/integration_test_abstract.py @@ -128,8 +128,9 @@ def _stream_records_test_logic( for file_dict in stream_slice["files"]: # TODO: if we ever test other filetypes in these tests this will need fixing file_reader = CsvParser(format) - with file_dict["storage_file"].open(file_reader.is_binary) as f: - expected_columns.extend(list(file_reader.get_inferred_schema(f).keys())) + storage_file = file_dict["storage_file"] + with storage_file.open(file_reader.is_binary) as f: + expected_columns.extend(list(file_reader.get_inferred_schema(f, storage_file.file_info).keys())) expected_columns = set(expected_columns) # de-dupe for record in fs.read_records(sync_mode, stream_slice=stream_slice): diff --git a/airbyte-integrations/connectors/source-s3/source_s3/exceptions.py b/airbyte-integrations/connectors/source-s3/source_s3/exceptions.py new file mode 100644 index 000000000000..09e3a571b9da --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/source_s3/exceptions.py @@ -0,0 +1,35 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from typing import List, Optional, Union + +from airbyte_cdk.models import FailureType +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + +from .source_files_abstract.file_info import FileInfo + + +class S3Exception(AirbyteTracedException): + def __init__( + self, + file_info: Union[List[FileInfo], FileInfo], + internal_message: Optional[str] = None, + message: Optional[str] = None, + failure_type: FailureType = FailureType.system_error, + exception: BaseException = None, + ): + file_info = ( + file_info + if isinstance(file_info, (list, tuple)) + else [ + file_info, + ] + ) + file_names = ", ".join([file.key for file in file_info]) + user_friendly_message = f""" + The connector encountered an error while processing the file(s): {file_names}. + {message} + This can be an input configuration error as well, please double check your connection settings. + """ + super().__init__(internal_message=internal_message, message=user_friendly_message, failure_type=failure_type, exception=exception) diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/abstract_file_parser.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/abstract_file_parser.py index 76f110b5bc73..44579b5a0999 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/abstract_file_parser.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/abstract_file_parser.py @@ -45,12 +45,13 @@ def is_binary(self) -> bool: """ @abstractmethod - def get_inferred_schema(self, file: Union[TextIO, BinaryIO]) -> dict: + def get_inferred_schema(self, file: Union[TextIO, BinaryIO], file_info: FileInfo) -> dict: """ Override this with format-specifc logic to infer the schema of file Note: needs to return inferred schema with JsonSchema datatypes :param file: file-like object (opened via StorageFile) + :param file_info: file metadata :return: mapping of {columns:datatypes} where datatypes are JsonSchema types """ diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/avro_parser.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/avro_parser.py index 501188dc2520..7f439f3e7c89 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/avro_parser.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/avro_parser.py @@ -6,6 +6,7 @@ import fastavro from fastavro import reader +from source_s3.source_files_abstract.file_info import FileInfo from .abstract_file_parser import AbstractFileParser @@ -69,18 +70,20 @@ def _get_avro_schema(self, file: Union[TextIO, BinaryIO]) -> dict: else: return schema - def get_inferred_schema(self, file: Union[TextIO, BinaryIO]) -> dict: + def get_inferred_schema(self, file: Union[TextIO, BinaryIO], file_info: FileInfo) -> dict: """Return schema :param file: file-like object (opened via StorageFile) + :param file_info: file metadata :return: mapping of {columns:datatypes} where datatypes are JsonSchema types """ avro_schema = self._get_avro_schema(file) schema_dict = self._parse_data_type(data_type_mapping, avro_schema) return schema_dict - def stream_records(self, file: Union[TextIO, BinaryIO]) -> Iterator[Mapping[str, Any]]: + def stream_records(self, file: Union[TextIO, BinaryIO], file_info: FileInfo) -> Iterator[Mapping[str, Any]]: """Stream the data using a generator :param file: file-like object (opened via StorageFile) + :param file_info: file metadata :yield: data record as a mapping of {columns:values} """ avro_reader = reader(file) diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py index 49eef269a233..9b832653b932 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py @@ -11,6 +11,8 @@ import pyarrow as pa import six # type: ignore[import] from pyarrow import csv as pa_csv +from source_s3.exceptions import S3Exception +from source_s3.source_files_abstract.file_info import FileInfo from source_s3.utils import get_value_or_json_if_empty_string, run_in_external_process from .abstract_file_parser import AbstractFileParser @@ -20,6 +22,19 @@ TMP_FOLDER = tempfile.mkdtemp() +def wrap_exception(exceptions: Tuple[type, ...]): + def wrapper(fn: callable): + def inner(self, file: Union[TextIO, BinaryIO], file_info: FileInfo): + try: + return fn(self, file, file_info) + except exceptions as e: + raise S3Exception(file_info, str(e), str(e), exception=e) + + return inner + + return wrapper + + class CsvParser(AbstractFileParser): def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) @@ -74,7 +89,8 @@ def _convert_options(self, json_schema: Mapping[str, Any] = None) -> Mapping[str **json.loads(additional_reader_options), } - def get_inferred_schema(self, file: Union[TextIO, BinaryIO]) -> Mapping[str, Any]: + @wrap_exception((ValueError,)) + def get_inferred_schema(self, file: Union[TextIO, BinaryIO], file_info: FileInfo) -> Mapping[str, Any]: """ https://arrow.apache.org/docs/python/generated/pyarrow.csv.open_csv.html This now uses multiprocessing in order to timeout the schema inference as it can hang. @@ -146,7 +162,8 @@ def _get_schema_dict_without_inference(self, file: Union[TextIO, BinaryIO]) -> M field_names = next(reader) return {field_name.strip(): pyarrow.string() for field_name in field_names} - def stream_records(self, file: Union[TextIO, BinaryIO]) -> Iterator[Mapping[str, Any]]: + @wrap_exception((ValueError,)) + def stream_records(self, file: Union[TextIO, BinaryIO], file_info: FileInfo) -> Iterator[Mapping[str, Any]]: """ https://arrow.apache.org/docs/python/generated/pyarrow.csv.open_csv.html PyArrow returns lists of values for each column so we zip() these up into records which we then yield diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/jsonl_parser.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/jsonl_parser.py index f3e4595bcbd7..b6d408dc1e4f 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/jsonl_parser.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/jsonl_parser.py @@ -7,6 +7,7 @@ import pyarrow as pa from pyarrow import json as pa_json +from source_s3.source_files_abstract.file_info import FileInfo from .abstract_file_parser import AbstractFileParser from .jsonl_spec import JsonlFormat @@ -73,7 +74,7 @@ def _read_table(self, file: Union[TextIO, BinaryIO], json_schema: Mapping[str, A file, pa.json.ReadOptions(**self._read_options()), pa.json.ParseOptions(**self._parse_options(json_schema)) ) - def get_inferred_schema(self, file: Union[TextIO, BinaryIO]) -> Mapping[str, Any]: + def get_inferred_schema(self, file: Union[TextIO, BinaryIO], file_info: FileInfo) -> Mapping[str, Any]: """ https://arrow.apache.org/docs/python/generated/pyarrow.json.read_json.html Json reader support multi thread hence, donot need to add external process @@ -93,7 +94,7 @@ def field_type_to_str(type_: Any) -> str: schema_dict = {field.name: field_type_to_str(field.type) for field in table.schema} return self.json_schema_to_pyarrow_schema(schema_dict, reverse=True) - def stream_records(self, file: Union[TextIO, BinaryIO]) -> Iterator[Mapping[str, Any]]: + def stream_records(self, file: Union[TextIO, BinaryIO], file_info: FileInfo) -> Iterator[Mapping[str, Any]]: """ https://arrow.apache.org/docs/python/generated/pyarrow.json.read_json.html diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/parquet_parser.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/parquet_parser.py index a274502a0ef1..9a290b940775 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/parquet_parser.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/parquet_parser.py @@ -5,7 +5,10 @@ from typing import Any, BinaryIO, Iterator, List, Mapping, TextIO, Tuple, Union import pyarrow.parquet as pq +from airbyte_cdk.models import FailureType from pyarrow.parquet import ParquetFile +from source_s3.exceptions import S3Exception +from source_s3.source_files_abstract.file_info import FileInfo from .abstract_file_parser import AbstractFileParser from .parquet_spec import ParquetFormat @@ -85,7 +88,7 @@ def convert_field_data(logical_type: str, field_value: Any) -> Any: return func(field_value) if func else field_value raise TypeError(f"unsupported field type: {logical_type}, value: {field_value}") - def get_inferred_schema(self, file: Union[TextIO, BinaryIO]) -> dict: + def get_inferred_schema(self, file: Union[TextIO, BinaryIO], file_info: FileInfo) -> dict: """ https://arrow.apache.org/docs/python/parquet.html#finer-grained-reading-and-writing @@ -97,10 +100,10 @@ def get_inferred_schema(self, file: Union[TextIO, BinaryIO]) -> dict: } if not schema_dict: # pyarrow can parse empty parquet files but a connector can't generate dynamic schema - raise OSError("empty Parquet file") + raise S3Exception(file_info, "empty Parquet file", "The .parquet file is empty!", FailureType.config_error) return schema_dict - def stream_records(self, file: Union[TextIO, BinaryIO]) -> Iterator[Mapping[str, Any]]: + def stream_records(self, file: Union[TextIO, BinaryIO], file_info: FileInfo) -> Iterator[Mapping[str, Any]]: """ https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetFile.html PyArrow reads streaming batches from a Parquet file @@ -116,7 +119,7 @@ def stream_records(self, file: Union[TextIO, BinaryIO]) -> Iterator[Mapping[str, } if not reader.schema: # pyarrow can parse empty parquet files but a connector can't generate dynamic schema - raise OSError("empty Parquet file") + raise S3Exception(file_info, "empty Parquet file", "The .parquet file is empty!", FailureType.config_error) args = self._select_options("columns", "batch_size") # type: ignore[arg-type] self.logger.debug(f"Found the {reader.num_row_groups} Parquet groups") diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/stream.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/stream.py index e45297f793f6..bf20111ab53a 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/stream.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/stream.py @@ -12,10 +12,12 @@ from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Union from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import FailureType from airbyte_cdk.models.airbyte_protocol import SyncMode from airbyte_cdk.sources.streams import Stream from wcmatch.glob import GLOBSTAR, SPLIT, globmatch +from ..exceptions import S3Exception from .file_info import FileInfo from .formats.abstract_file_parser import AbstractFileParser from .formats.avro_parser import AvroParser @@ -221,6 +223,7 @@ def _get_master_schema(self, min_datetime: datetime = None) -> Dict[str, Any]: file_reader = self.fileformatparser_class(self._format) + processed_files = [] for file_info in self.get_time_ordered_file_infos(): # skip this file if it's earlier than min_datetime if (min_datetime is not None) and (file_info.last_modified < min_datetime): @@ -228,7 +231,8 @@ def _get_master_schema(self, min_datetime: datetime = None) -> Dict[str, Any]: storagefile = self.storagefile_class(file_info, self._provider) with storagefile.open(file_reader.is_binary) as f: - this_schema = file_reader.get_inferred_schema(f) + this_schema = file_reader.get_inferred_schema(f, file_info) + processed_files.append(file_info) if this_schema == master_schema: continue # exact schema match so go to next file @@ -249,15 +253,18 @@ def _get_master_schema(self, min_datetime: datetime = None) -> Dict[str, Any]: master_schema[col] = broadest_of_types if override_type or type_explicitly_defined: LOGGER.warn( - f"Detected mismatched datatype on column '{col}', in file '{storagefile.url}'. " + f"Detected mismatched datatype on column '{col}', in file '{file_info}'. " + f"Should be '{master_schema[col]}', but found '{this_schema[col]}'. " + f"Airbyte will attempt to coerce this to {master_schema[col]} on read." ) continue # otherwise throw an error on mismatching datatypes - raise RuntimeError( - f"Detected mismatched datatype on column '{col}', in file '{storagefile.url}'. " - + f"Should be '{master_schema[col]}', but found '{this_schema[col]}'." + raise S3Exception( + processed_files, + "Column type mismatch", + f"Detected mismatched datatype on column '{col}', in file '{file_info}'. " + + f"Should be '{master_schema[col]}', but found '{this_schema[col]}'.", + failure_type=FailureType.config_error, ) # missing columns in this_schema doesn't affect our master_schema, so we don't check for it here @@ -343,7 +350,7 @@ def _read_from_slice( storage_file: StorageFile = file_item["storage_file"] with storage_file.open(file_reader.is_binary) as f: # TODO: make this more efficient than mutating every record one-by-one as they stream - for record in file_reader.stream_records(f): + for record in file_reader.stream_records(f, storage_file.file_info): schema_matched_record = self._match_target_schema(record, list(self._get_schema_map().keys())) complete_record = self._add_extra_fields_from_map( schema_matched_record, diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/abstract_test_parser.py b/airbyte-integrations/connectors/source-s3/unit_tests/abstract_test_parser.py index 433120053b4b..7fe028297e77 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/abstract_test_parser.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/abstract_test_parser.py @@ -132,25 +132,28 @@ def _get_readmode(self, file_info: Mapping[str, Any]) -> str: @memory_limit(1024) def test_suite_inferred_schema(self, file_info: Mapping[str, Any]) -> None: + file_info_instance = FileInfo(key=file_info["filepath"], size=os.stat(file_info["filepath"]).st_size, last_modified=datetime.now()) with smart_open(file_info["filepath"], self._get_readmode(file_info)) as f: if "test_get_inferred_schema" in file_info["fails"]: with pytest.raises(Exception) as e_info: - file_info["AbstractFileParser"].get_inferred_schema(f) + file_info["AbstractFileParser"].get_inferred_schema(f), file_info_instance self.logger.debug(str(e_info)) else: - assert file_info["AbstractFileParser"].get_inferred_schema(f) == file_info["inferred_schema"] + assert file_info["AbstractFileParser"].get_inferred_schema(f, file_info_instance) == file_info["inferred_schema"] @memory_limit(1024) def test_stream_suite_records(self, file_info: Mapping[str, Any]) -> None: filepath = file_info["filepath"] - self.logger.info(f"read the file: {filepath}, size: {os.stat(filepath).st_size / (1024 ** 2)}Mb") + file_size = os.stat(filepath).st_size + file_info_instance = FileInfo(key=filepath, size=file_size, last_modified=datetime.now()) + self.logger.info(f"read the file: {filepath}, size: {file_size / (1024 ** 2)}Mb") with smart_open(filepath, self._get_readmode(file_info)) as f: if "test_stream_records" in file_info["fails"]: with pytest.raises(Exception) as e_info: - [print(r) for r in file_info["AbstractFileParser"].stream_records(f)] + [print(r) for r in file_info["AbstractFileParser"].stream_records(f, file_info_instance)] self.logger.debug(str(e_info)) else: - records = [r for r in file_info["AbstractFileParser"].stream_records(f)] + records = [r for r in file_info["AbstractFileParser"].stream_records(f, file_info_instance)] assert len(records) == file_info["num_records"] for index, expected_record in file_info["line_checks"].items(): diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/test_csv_parser.py b/airbyte-integrations/connectors/source-s3/unit_tests/test_csv_parser.py index c6c8748af6ad..5cc5bd6999f8 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/test_csv_parser.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/test_csv_parser.py @@ -10,8 +10,10 @@ from pathlib import Path from typing import Any, List, Mapping, Tuple +import pendulum import pytest from smart_open import open as smart_open +from source_s3.source_files_abstract.file_info import FileInfo from source_s3.source_files_abstract.formats.csv_parser import CsvParser from .abstract_test_parser import AbstractTestParser, memory_limit @@ -403,7 +405,7 @@ def test_big_file(self) -> None: next(expected_file) read_count = 0 with smart_open(filepath, self._get_readmode({"AbstractFileParser": parser})) as f: - for record in parser.stream_records(f): + for record in parser.stream_records(f, FileInfo(key=filepath, size=file_size, last_modified=pendulum.now())): record_line = ",".join("" if v is None else str(v) for v in record.values()) expected_line = next(expected_file).strip("\n") assert record_line == expected_line diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-s3/unit_tests/test_stream.py index 5aee69d50ee3..81fb5cf9769b 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/test_stream.py @@ -9,6 +9,7 @@ import pytest from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import SyncMode +from source_s3.exceptions import S3Exception from source_s3.source_files_abstract.file_info import FileInfo from source_s3.source_files_abstract.storagefile import StorageFile from source_s3.source_files_abstract.stream import IncrementalFileStream @@ -642,7 +643,7 @@ def test_master_schema( dataset="dummy", provider={}, format={"filetype": "csv"}, schema=user_schema, path_pattern="**/prefix*.csv" ) if error_expected: - with pytest.raises(RuntimeError): + with pytest.raises(S3Exception): stream_instance._get_master_schema(min_datetime=min_datetime) else: assert stream_instance._get_master_schema(min_datetime=min_datetime) == expected_schema diff --git a/airbyte-integrations/connectors/source-salesforce/setup.py b/airbyte-integrations/connectors/source-salesforce/setup.py index beabe0dd8bf3..517429e3d7d1 100644 --- a/airbyte-integrations/connectors/source-salesforce/setup.py +++ b/airbyte-integrations/connectors/source-salesforce/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "vcrpy==4.1.1", "pandas"] +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.2", "vcrpy==4.1.1", "pandas"] TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6", "requests_mock", "source-acceptance-test", "pytest-timeout"] diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py index aadaef4cae19..38144ceae4c7 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py @@ -460,7 +460,9 @@ def test_forwarding_sobject_options(stream_config, stream_names, catalog_stream_ catalog = ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( - stream=AirbyteStream(name=catalog_stream_name, json_schema={"type": "object"}), + stream=AirbyteStream(name=catalog_stream_name, + supported_sync_modes=[SyncMode.full_refresh], + json_schema={"type": "object"}), sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite, ) diff --git a/airbyte-integrations/connectors/source-sftp-bulk/.dockerignore b/airbyte-integrations/connectors/source-sftp-bulk/.dockerignore new file mode 100644 index 000000000000..c35a8c56feb8 --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_sftp_bulk +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-sftp-bulk/Dockerfile b/airbyte-integrations/connectors/source-sftp-bulk/Dockerfile new file mode 100644 index 000000000000..1a2ed98a6c48 --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.9-slim +# FROM python:3.9.11-alpine3.15 + +# Bash is installed for more convenient debugging. +# RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +WORKDIR /airbyte/integration_code +COPY source_sftp_bulk ./source_sftp_bulk +COPY main.py ./ +COPY setup.py ./ +RUN pip install . + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-sftp-bulk diff --git a/airbyte-integrations/connectors/source-sftp-bulk/README.md b/airbyte-integrations/connectors/source-sftp-bulk/README.md new file mode 100644 index 000000000000..47c0ab30ae94 --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/README.md @@ -0,0 +1,129 @@ +# SFTP Bulk Source + +This is the repository for the FTP source connector, written in Python, that helps you bulk ingest files with the same data format from an FTP server into a single stream. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/sftp-bulk). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +From the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-sftp-bulk:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/sftp-bulk) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_sftp_bulk/spec.json` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source ftp test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-sftp-bulk:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-sftp-bulk:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-sftp-bulk:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sftp-bulk:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sftp-bulk:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-sftp-bulk:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-sftp-bulk:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-sftp-bulk:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-sftp-bulk/acceptance-test-config.yml b/airbyte-integrations/connectors/source-sftp-bulk/acceptance-test-config.yml new file mode 100644 index 000000000000..ff24b6e3bd3c --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/acceptance-test-config.yml @@ -0,0 +1,27 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-sftp-bulk:dev +tests: + spec: + - spec_path: "source_sftp_bulk/spec.json" + timeout_seconds: 60 + connection: + - config_path: "integration_tests/valid_config.json" + status: "succeed" + timeout_seconds: 60 + - config_path: "integration_tests/invalid_config.json" + status: "failed" + timeout_seconds: 60 + discovery: + - config_path: "integration_tests/valid_config.json" + basic_read: + - config_path: "integration_tests/valid_config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + incremental: + - config_path: "integration_tests/valid_config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "integration_tests/valid_config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-sftp-bulk/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-sftp-bulk/acceptance-test-docker.sh new file mode 100755 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-sftp-bulk/build.gradle b/airbyte-integrations/connectors/source-sftp-bulk/build.gradle new file mode 100644 index 000000000000..158db78f5330 --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_sftp_bulk' +} diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/__init__.py b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..0e79da99238e --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/abnormal_state.json @@ -0,0 +1,9 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "last_modified": "2522-10-04T13:31:15" }, + "stream_descriptor": { "name": "test_stream" } + } + } +] diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/acceptance.py new file mode 100644 index 000000000000..02ff76980f3f --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/acceptance.py @@ -0,0 +1,44 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import os +import shutil +import time +import uuid + +import docker +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + +TMP_FOLDER = "/tmp/test_sftp_source" + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + dir_path = os.getcwd() + "/integration_tests/files" + + if os.path.exists(TMP_FOLDER): + shutil.rmtree(TMP_FOLDER) + shutil.copytree(dir_path, TMP_FOLDER) + + docker_client = docker.from_env() + + container = docker_client.containers.run( + "atmoz/sftp", + "foo:pass", + name=f"mysftpacceptance_{uuid.uuid4().hex}", + ports={22: 1122}, + volumes={ + f"{TMP_FOLDER}": {"bind": "/home/foo/files", "mode": "rw"}, + }, + detach=True, + ) + + time.sleep(5) + yield + + shutil.rmtree(TMP_FOLDER) + container.kill() + container.remove() diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..c16d82b80a12 --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/configured_catalog.json @@ -0,0 +1,24 @@ +{ + "streams": [ + { + "stream": { + "name": "test_stream", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "string_col": { "type": "str" }, + "int_col": { "type": "integer" }, + "last_modified": { "type": "string", "format": "date-time" } + } + }, + "supported_sync_modes": ["incremental"], + "default_cursor_field": ["last_modified"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["last_modified"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/files/csv/test_1.csv b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/files/csv/test_1.csv new file mode 100644 index 000000000000..a88d0791f99f --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/files/csv/test_1.csv @@ -0,0 +1,3 @@ +string_col,int_col +"hello",1 +"foo",2 \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/files/empty/empty.json b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/files/empty/empty.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/files/null_values/null_values.csv b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/files/null_values/null_values.csv new file mode 100644 index 000000000000..431e3f0a1a95 --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/files/null_values/null_values.csv @@ -0,0 +1,6 @@ +string_col,int_col +"hello",1 +"foo",2 +"bar", +"baz",3 +,4 diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/files/test_1.json b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/files/test_1.json new file mode 100644 index 000000000000..83508f135089 --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/files/test_1.json @@ -0,0 +1 @@ +{ "string_col": "foo", "int_col": 2 } diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/files/test_2.json b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/files/test_2.json new file mode 100644 index 000000000000..38e9836a6f02 --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/files/test_2.json @@ -0,0 +1 @@ +{ "string_col": "hello", "int_col": 1 } diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py new file mode 100644 index 000000000000..e29ef1312295 --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py @@ -0,0 +1,248 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import logging +import os +import shutil +import time +from io import StringIO +from socket import socket +from typing import Mapping + +import docker +import paramiko +import pytest +from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, Status, SyncMode, Type +from source_sftp_bulk import SourceFtp + +pytest_plugins = ("source_acceptance_test.plugin",) + +logger = logging.getLogger("airbyte") + +TMP_FOLDER = "/tmp/test_sftp_source" + + +def generate_ssh_keys(): + key = paramiko.RSAKey.generate(2048) + privateString = StringIO() + key.write_private_key(privateString) + + return privateString.getvalue(), "ssh-rsa " + key.get_base64() + + +@pytest.fixture(scope="session") +def docker_client(): + return docker.from_env() + + +@pytest.fixture(name="config", scope="session") +def config_fixture(docker_client): + with socket() as s: + s.bind(("", 0)) + available_port = s.getsockname()[1] + + dir_path = os.getcwd() + "/integration_tests" + + config = { + "host": "localhost", + "port": available_port, + "username": "foo", + "password": "pass", + "file_type": "json", + "start_date": "2021-01-01T00:00:00Z", + "folder_path": "/files", + "stream_name": "overwrite_stream", + } + + container = docker_client.containers.run( + "atmoz/sftp", + f"{config['username']}:{config['password']}", + name="mysftp", + ports={22: config["port"]}, + volumes={ + f"{dir_path}/files": {"bind": "/home/foo/files", "mode": "rw"}, + }, + detach=True, + ) + + time.sleep(20) + yield config + + container.kill() + container.remove() + + +@pytest.fixture(name="config_pk", scope="session") +def config_fixture_pk(docker_client): + with socket() as s: + s.bind(("", 0)) + available_port = s.getsockname()[1] + + ssh_path = TMP_FOLDER + "/ssh" + dir_path = os.getcwd() + "/integration_tests" + + if os.path.exists(ssh_path): + shutil.rmtree(ssh_path) + + os.makedirs(ssh_path) + + pk, pubk = generate_ssh_keys() + + pub_key_path = ssh_path + "/id_rsa.pub" + with open(pub_key_path, "w") as f: + f.write(pubk) + + config = { + "host": "localhost", + "port": available_port, + "username": "foo", + "password": "pass", + "file_type": "json", + "private_key": pk, + "start_date": "2021-01-01T00:00:00Z", + "folder_path": "/files", + "stream_name": "overwrite_stream", + } + + container = docker_client.containers.run( + "atmoz/sftp", + f"{config['username']}:{config['password']}:1001", + name="mysftpssh", + ports={22: config["port"]}, + volumes={ + f"{dir_path}/files": {"bind": "/home/foo/files", "mode": "rw"}, + f"{pub_key_path}": {"bind": "/home/foo/.ssh/keys/id_rsa.pub", "mode": "ro"}, + }, + detach=True, + ) + + time.sleep(20) + yield config + + shutil.rmtree(ssh_path) + container.kill() + container.remove() + + +@pytest.fixture(name="configured_catalog") +def configured_catalog_fixture() -> ConfiguredAirbyteCatalog: + stream_schema = { + "type": "object", + "properties": {"string_col": {"type": "str"}, "int_col": {"type": "integer"}}, + } + + overwrite_stream = ConfiguredAirbyteStream( + stream=AirbyteStream( + name="overwrite_stream", json_schema=stream_schema, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental] + ), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + + return ConfiguredAirbyteCatalog(streams=[overwrite_stream]) + + +def test_check_valid_config_pk(config_pk: Mapping): + outcome = SourceFtp().check(logger, config_pk) + assert outcome.status == Status.SUCCEEDED + + +def test_check_valid_config_pk_bad_pk(config_pk: Mapping): + outcome = SourceFtp().check( + logger, {**config_pk, "private_key": "-----BEGIN OPENSSH PRIVATE KEY-----\nbaddata\n-----END OPENSSH PRIVATE KEY-----"} + ) + assert outcome.status == Status.FAILED + + +def test_check_invalid_config(config: Mapping): + outcome = SourceFtp().check(logger, {**config, "password": "wrongpass"}) + assert outcome.status == Status.FAILED + + +def test_check_valid_config(config: Mapping): + outcome = SourceFtp().check(logger, config) + assert outcome.status == Status.SUCCEEDED + + +def test_get_files_no_pattern_json(config: Mapping, configured_catalog: ConfiguredAirbyteCatalog): + source = SourceFtp() + result_iter = source.read(logger, config, configured_catalog, None) + result = list(result_iter) + assert len(result) == 2 + for res in result: + assert res.type == Type.RECORD + assert res.record.data["string_col"] in ["foo", "hello"] + assert res.record.data["int_col"] in [1, 2] + + +def test_get_files_pattern_json(config: Mapping, configured_catalog: ConfiguredAirbyteCatalog): + source = SourceFtp() + result_iter = source.read(logger, {**config, "file_pattern": "test_1.+"}, configured_catalog, None) + result = list(result_iter) + assert len(result) == 1 + for res in result: + assert res.type == Type.RECORD + assert res.record.data["string_col"] == "foo" + assert res.record.data["int_col"] == 2 + + +def test_get_files_pattern_no_match_json(config: Mapping, configured_catalog: ConfiguredAirbyteCatalog): + source = SourceFtp() + result = source.read(logger, {**config, "file_pattern": "bad_pattern.+"}, configured_catalog, None) + assert len(list(result)) == 0 + + +def test_get_files_no_pattern_csv(config: Mapping, configured_catalog: ConfiguredAirbyteCatalog): + source = SourceFtp() + result_iter = source.read(logger, {**config, "file_type": "csv", "folder_path": "files/csv"}, configured_catalog, None) + result = list(result_iter) + assert len(result) == 2 + for res in result: + assert res.type == Type.RECORD + assert res.record.data["string_col"] in ["foo", "hello"] + assert res.record.data["int_col"] in [1, 2] + + +def test_get_files_pattern_csv(config: Mapping, configured_catalog: ConfiguredAirbyteCatalog): + source = SourceFtp() + result_iter = source.read( + logger, {**config, "file_type": "csv", "folder_path": "files/csv", "file_pattern": "test_1.+"}, configured_catalog, None + ) + result = list(result_iter) + assert len(result) == 2 + for res in result: + assert res.type == Type.RECORD + assert res.record.data["string_col"] in ["foo", "hello"] + assert res.record.data["int_col"] in [1, 2] + + +def test_get_files_pattern_no_match_csv(config: Mapping, configured_catalog: ConfiguredAirbyteCatalog): + source = SourceFtp() + result = source.read( + logger, {**config, "file_type": "csv", "folder_path": "files/csv", "file_pattern": "badpattern.+"}, configured_catalog, None + ) + assert len(list(result)) == 0 + + +def test_get_files_empty_files(config: Mapping, configured_catalog: ConfiguredAirbyteCatalog): + source = SourceFtp() + result = source.read(logger, {**config, "folder_path": "files/empty"}, configured_catalog, None) + assert len(list(result)) == 0 + + +def test_get_files_handle_null_values(config: Mapping, configured_catalog: ConfiguredAirbyteCatalog): + source = SourceFtp() + result_iter = source.read(logger, {**config, "folder_path": "files/null_values", "file_type": "csv"}, configured_catalog, None) + result = list(result_iter) + assert len(result) == 5 + + res = result[2] + assert res.type == Type.RECORD + assert res.record.data["string_col"] == "bar" + assert res.record.data["int_col"] is None + + res = result[4] + assert res.type == Type.RECORD + assert res.record.data["string_col"] is None + assert res.record.data["int_col"] == 4 diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/invalid_config.json new file mode 100644 index 000000000000..0fd7d47ba99a --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/invalid_config.json @@ -0,0 +1,11 @@ +{ + "host": "localhost", + "port": 1122, + "username": "foo", + "password": "badpass", + "file_type": "json", + "start_date": "2021-01-01T00:00:00Z", + "folder_path": "/files", + "stream_name": "test_stream", + "file_most_recent": false +} diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/sample_state.json new file mode 100644 index 000000000000..7070334fd07c --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/sample_state.json @@ -0,0 +1,9 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "last_modified": "2022-10-04T13:31:15" }, + "stream_descriptor": { "name": "test_stream" } + } + } +] diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/valid_config.json b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/valid_config.json new file mode 100644 index 000000000000..f8093d1e610e --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/valid_config.json @@ -0,0 +1,11 @@ +{ + "host": "localhost", + "port": 1122, + "username": "foo", + "password": "pass", + "file_type": "json", + "start_date": "2021-01-01T00:00:00Z", + "folder_path": "/files", + "stream_name": "test_stream", + "file_most_recent": false +} diff --git a/airbyte-integrations/connectors/source-sftp-bulk/main.py b/airbyte-integrations/connectors/source-sftp-bulk/main.py new file mode 100644 index 000000000000..9f22f4e8fa01 --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_sftp_bulk import SourceFtp + +if __name__ == "__main__": + source = SourceFtp() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zoom-singer/requirements.txt b/airbyte-integrations/connectors/source-sftp-bulk/requirements.txt similarity index 75% rename from airbyte-integrations/connectors/source-zoom-singer/requirements.txt rename to airbyte-integrations/connectors/source-sftp-bulk/requirements.txt index 7b9114ed5867..7be17a56d745 100644 --- a/airbyte-integrations/connectors/source-zoom-singer/requirements.txt +++ b/airbyte-integrations/connectors/source-sftp-bulk/requirements.txt @@ -1,2 +1,3 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. +-e ../../bases/source-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-sftp-bulk/setup.py b/airbyte-integrations/connectors/source-sftp-bulk/setup.py new file mode 100644 index 000000000000..9d5c69d0a614 --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.2", + "paramiko==2.11.0", + "backoff==1.8.0", + "terminaltables==3.1.0", + "pandas==1.5.0", +] + +TEST_REQUIREMENTS = ["pytest~=6.1", "source-acceptance-test", "docker==5.0.3"] + +setup( + name="source_sftp_bulk", + description="Source implementation for SFTP Bulk.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/__init__.py b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/__init__.py new file mode 100644 index 000000000000..5e55118d5656 --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceFtp + +__all__ = ["SourceFtp"] diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/client.py b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/client.py new file mode 100644 index 000000000000..96518ae297f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/client.py @@ -0,0 +1,205 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import io +import logging +import os +import re +import socket +import stat +from datetime import datetime +from typing import Any, Dict, List, Mapping, Tuple + +import backoff +import numpy as np +import pandas as pd +import paramiko +from paramiko.ssh_exception import AuthenticationException + +# set default timeout to 300 seconds +REQUEST_TIMEOUT = 300 + +logger = logging.getLogger("airbyte") + +File = Dict[str, Any] + + +class SFTPClient: + _connection = None + + def __init__(self, host, username, password=None, private_key=None, port=None, timeout=REQUEST_TIMEOUT): + self.host = host + self.username = username + self.password = password + self.port = int(port) or 22 + + self.key = paramiko.RSAKey.from_private_key(io.StringIO(private_key)) if private_key else None + self.timeout = float(timeout) if timeout else REQUEST_TIMEOUT + + if not self.password and not self.key: + raise Exception("Either password or private key must be provided") + + self._connect() + + def handle_backoff(details): + logger.warning("SSH Connection closed unexpectedly. Waiting {wait} seconds and retrying...".format(**details)) + + # If connection is snapped during connect flow, retry up to a + # minute for SSH connection to succeed. 2^6 + 2^5 + ... + @backoff.on_exception(backoff.expo, (EOFError), max_tries=6, on_backoff=handle_backoff, jitter=None, factor=2) + def _connect(self): + if self._connection is not None: + return + + try: + self.transport = paramiko.Transport((self.host, self.port)) + self.transport.use_compression(True) + self.transport.connect(username=self.username, password=self.password, hostkey=None, pkey=self.key) + self._connection = paramiko.SFTPClient.from_transport(self.transport) + + # get 'socket' to set the timeout + socket = self._connection.get_channel() + # set request timeout + socket.settimeout(self.timeout) + + except (AuthenticationException) as ex: + raise Exception("Authentication failed: %s" % ex) + except Exception as ex: + raise Exception("SSH Connection failed: %s" % ex) + + def __enter__(self): + self._connect() + return self + + def __exit__(self): + """Clean up the socket when this class gets garbage collected.""" + self.close() + + def close(self): + if self._connection is not None: + try: + self._connection.close() + self.transport.close() + self._connection = None + # Known paramiko issue: https://github.com/paramiko/paramiko/issues/1617 + except Exception as e: + if str(e) != "'NoneType' object has no attribute 'time'": + raise + + @staticmethod + def get_files_matching_pattern(files, pattern) -> List[File]: + """Takes a file dict {"filepath": "...", "last_modified": "..."} and a regex pattern string, and returns files matching that pattern.""" + matcher = re.compile(pattern) + return [f for f in files if matcher.search(f["filepath"])] + + # backoff for 60 seconds as there is possibility the request will backoff again in 'discover.get_schema' + @backoff.on_exception(backoff.constant, (socket.timeout), max_time=60, interval=10, jitter=None) + def get_files_by_prefix(self, prefix: str) -> List[File]: + def is_empty(a): + return a.st_size == 0 + + def is_directory(a): + return stat.S_ISDIR(a.st_mode) + + files = [] + + if prefix is None or prefix == "": + prefix = "." + + try: + result = self._connection.listdir_attr(prefix) + except FileNotFoundError as e: + raise Exception("Directory '{}' does not exist".format(prefix)) from e + + for file_attr in result: + # NB: This only looks at the immediate level beneath the prefix directory + if is_directory(file_attr): + logger.info("Skipping directory: %s", file_attr.filename) + else: + if is_empty(file_attr): + logger.info("Skipping empty file: %s", file_attr.filename) + continue + + last_modified = file_attr.st_mtime + if last_modified is None: + logger.warning( + "Cannot read m_time for file %s, defaulting to current epoch time", os.path.join(prefix, file_attr.filename) + ) + last_modified = datetime.utcnow().timestamp() + + files.append( + { + "filepath": prefix + "/" + file_attr.filename, + "last_modified": datetime.utcfromtimestamp(last_modified).replace(tzinfo=None), + } + ) + + return files + + def get_files(self, prefix, search_pattern=None, modified_since=None, most_recent_only=False) -> List[File]: + files = self.get_files_by_prefix(prefix) + + if files: + logger.info('Found %s files in "%s"', len(files), prefix) + else: + logger.warning('Found no files on specified SFTP server at "%s"', prefix) + + matching_files = files + + if search_pattern is not None: + matching_files = self.get_files_matching_pattern(files, search_pattern) + + if matching_files and search_pattern: + logger.info('Found %s files in "%s" matching "%s"', len(matching_files), prefix, search_pattern) + + if not matching_files and search_pattern: + logger.warning('Found no files on specified SFTP server at "%s" matching "%s"', prefix, search_pattern) + + if modified_since is not None: + matching_files = [f for f in matching_files if f["last_modified"] > modified_since] + + # sort files in increasing order of "last_modified" + sorted_files = sorted(matching_files, key=lambda x: (x["last_modified"]).timestamp()) + + if most_recent_only: + logger.info(f"Returning only the most recently modified file: {sorted_files[-1]}.") + sorted_files = sorted_files[-1:] + + return sorted_files + + @backoff.on_exception(backoff.expo, (socket.timeout), max_tries=5, factor=2) + def fetch_file(self, fn: Mapping[str, Any], file_type="csv") -> pd.DataFrame: + try: + with self._connection.open(fn["filepath"], "rb") as f: + df: pd.DataFrame = None + + # Using pandas to make reading files in different formats easier + if file_type == "csv": + df = pd.read_csv(f) + elif file_type == "json": + df = pd.read_json(f, lines=True) + else: + raise Exception("Unsupported file type: %s" % file_type) + + # Replace nan with None for correct + # json serialization when emitting records + df = df.replace({np.nan: None}) + df["last_modified"] = fn["last_modified"] + return df + + except OSError as e: + if "Permission denied" in str(e): + logger.warning("Skipping %s file because you do not have enough permissions.", f["filepath"]) + else: + logger.warning("Skipping %s file because it is unable to be read.", f["filepath"]) + + raise Exception("Unable to read file: %s" % e) from e + + def fetch_files(self, files, file_type="csv") -> Tuple[datetime, Dict[str, Any]]: + logger.info("Fetching %s files", len(files)) + for fn in files: + records = self.fetch_file(fn, file_type) + yield (fn["last_modified"], records.to_dict("records")) + + self.close() diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/source.py b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/source.py new file mode 100644 index 000000000000..d7f70d86e143 --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/source.py @@ -0,0 +1,135 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import json +import logging +from datetime import datetime +from typing import Any, Dict, List, Mapping, Tuple + +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import AirbyteCatalog, AirbyteConnectionStatus, AirbyteStream, Status, SyncMode +from airbyte_cdk.sources import AbstractSource + +from .client import SFTPClient +from .streams import FTPStream + +logger = logging.getLogger("airbyte") + + +class SourceFtp(AbstractSource): + @property + def _default_json_schema(self): + return { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {}, + } + + def _generate_json_schema(self, dtypes: Dict[str, Any]) -> Dict[str, Any]: + json_schema = self._default_json_schema + + for key, val in dtypes.items(): + if val == "int64": + json_schema["properties"][key] = {"type": ["null", "integer"]} + elif val == "float64": + json_schema["properties"][key] = {"type": ["null", "number"]} + elif val == "bool": + json_schema["properties"][key] = {"type": ["null", "boolean"]} + # Special case for last_modified timestamp + elif key == "last_modified": + json_schema["properties"][key] = {"type": ["null", "string"], "format": "date-time"} + # Default to string + else: + json_schema["properties"][key] = {"type": ["null", "string"]} + + return json_schema + + def _infer_json_schema(self, config: Mapping[str, Any], connection: SFTPClient) -> Dict[str, Any]: + file_pattern = config.get("file_pattern") + files = connection.get_files(config["folder_path"], file_pattern) + + if len(files) == 0: + logger.warning(f"No files found in folder {config['folder_path']} with pattern {file_pattern}") + return self._default_json_schema + + # Get last file to infer schema + # Use pandas `infer_objects` to infer dtypes + df = connection.fetch_file(files[-1], config["file_type"]) + df = df.infer_objects() + + # Default column used for incremental sync + # Contains the date when a file was last modified or added + df["last_modified"] = files[-1]["last_modified"] + + if len(df) < 1: + logger.warning(f"No records found in file {files[0]}, can't infer json schema") + return self._default_json_schema + + return self._generate_json_schema(df.dtypes.to_dict()) + + def _get_connection(self, config: Mapping[str, Any]) -> SFTPClient: + return SFTPClient( + host=config["host"], + username=config["username"], + password=config["password"], + private_key=config.get("private_key", None), + port=config["port"], + ) + + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, AirbyteConnectionStatus]: + try: + conn = self._get_connection(config) + conn._connect() + conn.close() + return (True, AirbyteConnectionStatus(status=Status.SUCCEEDED)) + except Exception as ex: + logger.error( + f"Failed to connect to FTP server: {ex}", + ) + return (False, AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {str(ex)}")) + + def check(self, logger: AirbyteLogger, config: json) -> AirbyteConnectionStatus: + _, status = self.check_connection(logger, config) + return status + + def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog: + conn = self._get_connection(config) + json_schema = self._infer_json_schema(config, conn) + + stream_name = config["stream_name"] + streams = [] + + sync_modes = [SyncMode.full_refresh] + + file_most_recent = config.get("file_most_recent", False) + if not file_most_recent: + logger.debug("File most recent is false, enabling incremental sync mode") + sync_modes.append(SyncMode.incremental) + + streams.append( + AirbyteStream( + name=stream_name, + json_schema=json_schema, + supported_sync_modes=sync_modes, + source_defined_cursor=True, + default_cursor_field=[] if file_most_recent else ["last_modified"], + ) + ) + + conn.close() + return AirbyteCatalog(streams=streams) + + def streams(self, config: json) -> List[AirbyteStream]: + conn = SFTPClient( + host=config["host"], + username=config["username"], + password=config["password"], + private_key=config.get("private_key", None), + port=config["port"], + ) + + start_date = datetime.strptime(config["start_date"], "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=None) + json_schema = self._infer_json_schema(config, conn) + + return [FTPStream(config=config, start_date=start_date, connection=conn, json_schema=json_schema)] diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/spec.json b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/spec.json new file mode 100644 index 000000000000..c6d72d8ce46f --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/spec.json @@ -0,0 +1,104 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/source/ftp", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "FTP Source Spec", + "type": "object", + "required": [ + "username", + "host", + "port", + "stream_name", + "start_date", + "folder_path" + ], + "additionalProperties": true, + "properties": { + "username": { + "title": "User Name", + "description": "The server user", + "type": "string", + "order": 0 + }, + "password": { + "title": "Password", + "description": "OS-level password for logging into the jump server host", + "type": "string", + "airbyte_secret": true, + "order": 1 + }, + "private_key": { + "title": "Private key", + "description": "The private key", + "type": "string", + "multiline": true, + "order": 1 + }, + "host": { + "title": "Host Address", + "description": "The server host address", + "type": "string", + "examples": ["www.host.com", "192.0.2.1"], + "order": 1 + }, + "port": { + "title": "Port", + "description": "The server port", + "type": "integer", + "default": 22, + "examples": ["22"], + "order": 2 + }, + "stream_name": { + "title": "Stream name", + "description": "The name of the stream or table you want to create", + "type": "string", + "examples": ["ftp_contacts"], + "order": 1 + }, + "file_type": { + "title": "File type", + "description": "The file type you want to sync. Currently only 'csv' and 'json' files are supported.", + "type": "string", + "default": "csv", + "enum": ["csv", "json"], + "order": 4, + "examples": ["csv", "json"] + }, + "folder_path": { + "title": "Folder Path (Optional)", + "description": "The directory to search files for sync", + "type": "string", + "default": "", + "examples": ["/logs/2022"], + "order": 5 + }, + "file_pattern": { + "title": "File Pattern (Optional)", + "description": "The regular expression to specify files for sync in a chosen Folder Path", + "type": "string", + "default": "", + "examples": [ + "log-([0-9]{4})([0-9]{2})([0-9]{2}) - This will filter files which `log-yearmmdd`" + ], + "order": 6 + }, + "file_most_recent": { + "title": "Most recent file (Optional)", + "description": "Sync only the most recent file for the configured folder path and file pattern", + "type": "boolean", + "default": false, + "order": 7 + }, + "start_date": { + "type": "string", + "title": "Start Date", + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "examples": ["2017-01-25T00:00:00Z"], + "description": "The date from which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", + "order": 8 + } + } + } +} diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/streams.py b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/streams.py new file mode 100644 index 000000000000..155d9bd5622c --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/streams.py @@ -0,0 +1,77 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from datetime import datetime +from typing import Any, Iterable, List, Mapping + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams import IncrementalMixin, Stream + +from .client import SFTPClient + + +class FTPStream(Stream, IncrementalMixin): + primary_key = None + cursor_field = "last_modified" + + def __init__(self, config: Mapping[str, Any], start_date: datetime, connection: SFTPClient, json_schema: Mapping[str, Any], **kwargs): + super(Stream, self).__init__(**kwargs) + + self.config = config + self.start_date = start_date + self.connection = connection + + self._cursor_value: float = None + self._name = config["stream_name"] + self._only_most_recent_file: bool = config.get("file_most_recent", False) + self._json_schema = json_schema + + if self._only_most_recent_file: + self.cursor_field = None + + @property + def name(self) -> str: + """Source name""" + return self._name + + @property + def state(self) -> Mapping[str, Any]: + if self._cursor_value: + return {self.cursor_field: self._cursor_value.isoformat()} + + return {self.cursor_field: self.start_date.isoformat()} + + @state.setter + def state(self, value: Mapping[str, Any]): + self._cursor_value = datetime.fromisoformat(value[self.cursor_field]) + + def get_json_schema(self) -> Mapping[str, Any]: + return self._json_schema + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + if stream_state and sync_mode == SyncMode.incremental: + self._cursor_value = datetime.fromisoformat(stream_state[self.cursor_field]) + + if not stream_state and sync_mode == SyncMode.incremental: + self._cursor_value = self.start_date + + files = self.connection.get_files( + self.config.get("folder_path"), + self.config.get("file_pattern"), + modified_since=self._cursor_value, + most_recent_only=self._only_most_recent_file, + ) + + for cursor, records in self.connection.fetch_files(files, self.config["file_type"]): + if cursor and sync_mode == SyncMode.incremental: + if self._cursor_value and cursor > self._cursor_value: + self._cursor_value = cursor + + yield from records diff --git a/airbyte-integrations/connectors/source-sftp-bulk/unit_tests/client_test.py b/airbyte-integrations/connectors/source-sftp-bulk/unit_tests/client_test.py new file mode 100644 index 000000000000..b0a8a545da39 --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/unit_tests/client_test.py @@ -0,0 +1,67 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from source_sftp_bulk.client import SFTPClient + + +def test_get_files_matching_pattern_match(): + files = [ + { + "filepath": "test.csv", + "last_modified": "2021-01-01 00:00:00", + }, + { + "filepath": "test2.csv", + "last_modified": "2021-01-01 00:00:00", + }, + ] + + result = SFTPClient.get_files_matching_pattern(files, "test.csv") + assert result == [ + { + "filepath": "test.csv", + "last_modified": "2021-01-01 00:00:00", + } + ] + + +def test_get_files_matching_pattern_no_match(): + files = [ + { + "filepath": "test.csv", + "last_modified": "2021-01-01 00:00:00", + }, + { + "filepath": "test2.csv", + "last_modified": "2021-01-01 00:00:00", + }, + ] + + result = SFTPClient.get_files_matching_pattern(files, "test3.csv") + assert result == [] + + +def test_get_files_matching_pattern_regex_match(): + files = [ + { + "filepath": "test.csv", + "last_modified": "2021-01-01 00:00:00", + }, + { + "filepath": "test2.csv", + "last_modified": "2021-01-01 00:00:00", + }, + ] + + result = SFTPClient.get_files_matching_pattern(files, "test.*") + assert result == [ + { + "filepath": "test.csv", + "last_modified": "2021-01-01 00:00:00", + }, + { + "filepath": "test2.csv", + "last_modified": "2021-01-01 00:00:00", + }, + ] diff --git a/airbyte-integrations/connectors/source-sftp-bulk/unit_tests/source_test.py b/airbyte-integrations/connectors/source-sftp-bulk/unit_tests/source_test.py new file mode 100644 index 000000000000..c19fe8f10021 --- /dev/null +++ b/airbyte-integrations/connectors/source-sftp-bulk/unit_tests/source_test.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from source_sftp_bulk import SourceFtp + +source = SourceFtp() + + +def test_generate_json_schema(): + dtypes = { + "col1": "int64", + "col2": "float64", + "col3": "bool", + "col4": "object", + "col5": "string", + "last_modified": "datetime64[ns]", + } + + result = source._generate_json_schema(dtypes) + assert result == { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "col1": {"type": ["null", "integer"]}, + "col2": {"type": ["null", "number"]}, + "col3": {"type": ["null", "boolean"]}, + "col4": {"type": ["null", "string"]}, + "col5": {"type": ["null", "string"]}, + "last_modified": {"format": "date-time", "type": ["null", "string"]}, + }, + "type": "object", + } diff --git a/airbyte-integrations/connectors/source-sonar-cloud/.dockerignore b/airbyte-integrations/connectors/source-sonar-cloud/.dockerignore new file mode 100644 index 000000000000..e6f4b237f50d --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_sonar_cloud +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-sonar-cloud/Dockerfile b/airbyte-integrations/connectors/source-sonar-cloud/Dockerfile new file mode 100644 index 000000000000..63228935dc97 --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_sonar_cloud ./source_sonar_cloud + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-sonar-cloud diff --git a/airbyte-integrations/connectors/source-sonar-cloud/README.md b/airbyte-integrations/connectors/source-sonar-cloud/README.md new file mode 100644 index 000000000000..8d583e13ea04 --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/README.md @@ -0,0 +1,79 @@ +# Sonar Cloud Source + +This is the repository for the Sonar Cloud configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/sonar-cloud). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-sonar-cloud:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/sonar-cloud) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_sonar_cloud/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source sonar-cloud test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-sonar-cloud:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-sonar-cloud:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-sonar-cloud:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sonar-cloud:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sonar-cloud:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-sonar-cloud:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-sonar-cloud:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-sonar-cloud:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-sonar-cloud/__init__.py b/airbyte-integrations/connectors/source-sonar-cloud/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-sonar-cloud/acceptance-test-config.yml b/airbyte-integrations/connectors/source-sonar-cloud/acceptance-test-config.yml new file mode 100644 index 000000000000..0c8c8a4f9be6 --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/acceptance-test-config.yml @@ -0,0 +1,38 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-sonar-cloud:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_sonar_cloud/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.txt" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" +# TODO uncomment this block this block if your connector implements incremental sync: +# tests: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-sonar-cloud/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-sonar-cloud/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-sonar-cloud/build.gradle b/airbyte-integrations/connectors/source-sonar-cloud/build.gradle new file mode 100644 index 000000000000..1d6d97599de5 --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_sonar_cloud' +} diff --git a/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/__init__.py b/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/acceptance.py new file mode 100644 index 000000000000..1302b2f57e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..119a8e122504 --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/configured_catalog.json @@ -0,0 +1,31 @@ +{ + "streams": [ + { + "stream": { + "name": "components", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "issues", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "metrics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/invalid_config.json new file mode 100644 index 000000000000..2357f4981350 --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "user_token": "9019e9de10e0f77347f27e195396ea10d07c7c96", + "organization": "airbyte" +} diff --git a/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/sample_config.json new file mode 100644 index 000000000000..9cd000a56d54 --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "user_token": "90347f27e19e9de10e0f77195396ea10d07c7c96", + "organization": "airbyte", + "component_keys": ["woop-bo-back-new", "woop-ref"] +} diff --git a/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-sonar-cloud/main.py b/airbyte-integrations/connectors/source-sonar-cloud/main.py new file mode 100644 index 000000000000..48a0494368ce --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_sonar_cloud import SourceSonarCloud + +if __name__ == "__main__": + source = SourceSonarCloud() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-sonar-cloud/requirements.txt b/airbyte-integrations/connectors/source-sonar-cloud/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-sonar-cloud/setup.py b/airbyte-integrations/connectors/source-sonar-cloud/setup.py new file mode 100644 index 000000000000..1159c40211a0 --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_sonar_cloud", + description="Source implementation for Sonar Cloud.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/__init__.py b/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/__init__.py new file mode 100644 index 000000000000..45a7987a982f --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceSonarCloud + +__all__ = ["SourceSonarCloud"] diff --git a/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/schemas/components.json b/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/schemas/components.json new file mode 100644 index 000000000000..044c0352b2ea --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/schemas/components.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "organization": { + "type": "string" + }, + "key": { + "type": "string" + }, + "name": { + "type": "string" + }, + "qualifier": { + "type": "string" + }, + "project": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/schemas/issues.json b/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/schemas/issues.json new file mode 100644 index 000000000000..ad820d631a52 --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/schemas/issues.json @@ -0,0 +1,66 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "rule": { + "type": "string" + }, + "severity": { + "type": "string" + }, + "component": { + "type": "string" + }, + "project": { + "type": "string" + }, + "resolution": { + "type": "string" + }, + "status": { + "type": "string" + }, + "message": { + "type": "string" + }, + "effort": { + "type": "string" + }, + "debt": { + "type": "string" + }, + "author": { + "type": "string" + }, + "creationDate": { + "type": "string" + }, + "updateDate": { + "type": "string" + }, + "type": { + "type": "string" + }, + "organization": { + "type": "string" + }, + "textRange": { + "type": "object" + }, + "tags": { + "type": "array" + }, + "line": { + "type": "integer" + }, + "hash": { + "type": "string" + }, + "flows": { + "type": "array" + } + } +} diff --git a/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/schemas/metrics.json b/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/schemas/metrics.json new file mode 100644 index 000000000000..8ede95f11d9b --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/schemas/metrics.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "key": { + "type": "string" + }, + "name": { + "type": "string" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "domain": { + "type": "string" + }, + "qualitative": { + "type": "boolean" + }, + "hidden": { + "type": "boolean" + }, + "direction": { + "type": "number" + } + } +} diff --git a/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/sonar_cloud.yaml b/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/sonar_cloud.yaml new file mode 100644 index 000000000000..7610d4cdb28b --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/sonar_cloud.yaml @@ -0,0 +1,67 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: + - "{{ options['name'] }}" + requester: + url_base: "https://sonarcloud.io/api" + http_method: "GET" + request_options_provider: + request_parameters: + organization: "{{ config['organization'] }}" + createdAfter: "{{ config['start_date'] }}" + createdBefore: "{{ config['end_date'] }}" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['user_token'] }}" + increment_paginator: + type: "DefaultPaginator" + url_base: "*ref(definitions.requester.url_base)" + page_size_option: + inject_into: "request_parameter" + field_name: "ps" + pagination_strategy: + type: "PageIncrement" + page_size: 100 + page_token_option: + inject_into: "request_parameter" + field_name: "p" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + $ref: "*ref(definitions.increment_paginator)" + requester: + $ref: "*ref(definitions.requester)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + components_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "components" + primary_key: "key" + path: "/components/search" + issues_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "issues" + primary_key: "key" + path: "/issues/search?componentKeys={{ ','.join(config.get('component_keys', [])) }}" + metrics_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "metrics" + primary_key: "id" + path: "/metrics/search" + +streams: + - "*ref(definitions.components_stream)" + - "*ref(definitions.issues_stream)" + - "*ref(definitions.metrics_stream)" + +check: + stream_names: + - "components" diff --git a/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/source.py b/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/source.py new file mode 100644 index 000000000000..9d7d52c06e94 --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceSonarCloud(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "sonar_cloud.yaml"}) diff --git a/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/spec.yaml b/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/spec.yaml new file mode 100644 index 000000000000..b58b244a3c78 --- /dev/null +++ b/airbyte-integrations/connectors/source-sonar-cloud/source_sonar_cloud/spec.yaml @@ -0,0 +1,48 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/sonar-cloud +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Sonar Cloud Spec + type: object + required: + - user_token + - organization + - component_keys + additionalProperties: true + properties: + user_token: + title: User Token + type: string + description: >- + Your User Token. See here. The token is + case sensitive. + airbyte_secret: true + organization: + title: Organization + type: string + description: >- + Organization key. See here. + examples: + - airbyte + component_keys: + title: Component Keys + type: array + description: Comma-separated list of component keys. + examples: + - airbyte-ws-order + - airbyte-ws-checkout + start_date: + title: Start date + type: string + description: To retrieve issues created after the given date (inclusive). + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + examples: + - YYYY-MM-DD + end_date: + title: End date + type: string + description: To retrieve issues created before the given date (inclusive). + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + examples: + - YYYY-MM-DD diff --git a/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml b/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml index 0feabb8df976..8ba77b8d308e 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml @@ -1,21 +1,27 @@ connector_image: airbyte/source-surveymonkey:dev -tests: +acceptance_tests: spec: + tests: - spec_path: "source_surveymonkey/spec.json" connection: + tests: - config_path: "secrets/config.json" status: "succeed" - config_path: "integration_tests/invalid_config.json" - status: "exception" + status: "failed" discovery: + tests: - config_path: "secrets/config.json" basic_read: + tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" incremental: + tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" full_refresh: + tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml index 971be94ebdb6..98613225c304 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml @@ -68,23 +68,23 @@ tests: incremental: - config_path: "secrets/prod_config.json" configured_catalog_path: "integration_tests/streams_basic.json" - timeout_seconds: 3600 + timeout_seconds: 7200 future_state_path: "integration_tests/abnormal_state.json" - config_path: "secrets/prod_config.json" configured_catalog_path: "integration_tests/streams_reports_daily.json" - timeout_seconds: 3600 + timeout_seconds: 7200 future_state_path: "integration_tests/abnormal_state.json" # LIFETIME granularity: does not support incremental sync full_refresh: - config_path: "secrets/prod_config.json" configured_catalog_path: "integration_tests/streams_basic.json" - timeout_seconds: 3600 + timeout_seconds: 7200 ignored_fields: # Important: sometimes some streams does not return the same records in subsequent syncs "ad_groups": ["dayparting", "enable_search_result", "display_mode", "schedule_infos", "feed_type", "status" ] - config_path: "secrets/prod_config.json" configured_catalog_path: "integration_tests/streams_reports_daily.json" - timeout_seconds: 3600 + timeout_seconds: 7200 - config_path: "secrets/prod_config.json" configured_catalog_path: "integration_tests/streams_reports_lifetime.json" - timeout_seconds: 3600 \ No newline at end of file + timeout_seconds: 7200 \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/.dockerignore b/airbyte-integrations/connectors/source-tvmaze-schedule/.dockerignore new file mode 100644 index 000000000000..2aaed5f638fb --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_tvmaze_schedule +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/Dockerfile b/airbyte-integrations/connectors/source-tvmaze-schedule/Dockerfile new file mode 100644 index 000000000000..4df013196a63 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_tvmaze_schedule ./source_tvmaze_schedule + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-tvmaze-schedule diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/README.md b/airbyte-integrations/connectors/source-tvmaze-schedule/README.md new file mode 100644 index 000000000000..8f6ee3c87b0c --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/README.md @@ -0,0 +1,79 @@ +# Tvmaze Schedule Source + +This is the repository for the Tvmaze Schedule configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/tvmaze-schedule). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-tvmaze-schedule:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/tvmaze-schedule) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_tvmaze_schedule/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source tvmaze-schedule test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-tvmaze-schedule:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-tvmaze-schedule:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-tvmaze-schedule:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-tvmaze-schedule:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-tvmaze-schedule:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-tvmaze-schedule:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-tvmaze-schedule:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-tvmaze-schedule:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/__init__.py b/airbyte-integrations/connectors/source-tvmaze-schedule/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/acceptance-test-config.yml b/airbyte-integrations/connectors/source-tvmaze-schedule/acceptance-test-config.yml new file mode 100644 index 000000000000..3a7509bbf359 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/acceptance-test-config.yml @@ -0,0 +1,20 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-tvmaze-schedule:dev +tests: + spec: + - spec_path: "source_tvmaze_schedule/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-tvmaze-schedule/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/build.gradle b/airbyte-integrations/connectors/source-tvmaze-schedule/build.gradle new file mode 100644 index 000000000000..0d46db9972fb --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_tvmaze_schedule' +} diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/__init__.py b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..3a09e12ca7c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/configured_catalog.json @@ -0,0 +1,31 @@ +{ + "streams": [ + { + "stream": { + "name": "domestic", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "web", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "future", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/invalid_config.json new file mode 100644 index 000000000000..8a0545197c81 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/invalid_config.json @@ -0,0 +1,6 @@ +{ + "start_date": "2022-10-20", + "end_date": "2022-10-20", + "domestic_schedule_country_code": "something-wrong", + "web_schedule_country_code": "something-wrong" +} diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/sample_config.json new file mode 100644 index 000000000000..c56b1884ef5d --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/sample_config.json @@ -0,0 +1,6 @@ +{ + "start_date": "2022-10-20", + "end_date": "2022-10-20", + "domestic_schedule_country_code": "GB", + "web_schedule_country_code": "NL" +} diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/main.py b/airbyte-integrations/connectors/source-tvmaze-schedule/main.py new file mode 100644 index 000000000000..7b7ed5efaffe --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_tvmaze_schedule import SourceTvmazeSchedule + +if __name__ == "__main__": + source = SourceTvmazeSchedule() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/requirements.txt b/airbyte-integrations/connectors/source-tvmaze-schedule/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/setup.py b/airbyte-integrations/connectors/source-tvmaze-schedule/setup.py new file mode 100644 index 000000000000..90cb9a01f789 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_tvmaze_schedule", + description="Source implementation for Tvmaze Schedule.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/__init__.py b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/__init__.py new file mode 100644 index 000000000000..21885552e59c --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceTvmazeSchedule + +__all__ = ["SourceTvmazeSchedule"] diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/schemas/domestic.json b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/schemas/domestic.json new file mode 100644 index 000000000000..f422bec2b627 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/schemas/domestic.json @@ -0,0 +1,218 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["number", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "season": { + "type": ["number", "null"] + }, + "number": { + "type": ["number", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "airdate": { + "type": ["string", "null"] + }, + "airtime": { + "type": ["string", "null"] + }, + "airstamp": { + "type": ["string", "null"] + }, + "runtime": { + "type": ["number", "null"] + }, + "rating": { + "type": "object", + "properties": { + "average": { + "type": ["number", "null"] + } + } + }, + "image": { + "type": ["object", "null"], + "properties": { + "medium": { + "type": ["string", "null"] + }, + "original": { + "type": ["string", "null"] + } + } + }, + "summary": { + "type": ["string", "null"] + }, + "show": { + "type": "object", + "properties": { + "id": { + "type": ["number", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "language": { + "type": ["string", "null"] + }, + "genres": { + "type": "array", + "items": { + "type": ["string", "null"] + } + }, + "status": { + "type": ["string", "null"] + }, + "runtime": { + "type": ["number", "null"] + }, + "averageRuntime": { + "type": ["number", "null"] + }, + "premiered": { + "type": ["string", "null"] + }, + "ended": { + "type": ["string", "null"] + }, + "officialSite": { "type": ["string", "null"] }, + "schedule": { + "type": "object", + "properties": { + "time": { + "type": ["string", "null"] + }, + "days": { + "type": "array", + "items": { + "type": ["string", "null"] + } + } + } + }, + "rating": { + "type": "object", + "properties": { + "average": { + "type": ["number", "null"] + } + } + }, + "weight": { + "type": ["number", "null"] + }, + "network": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["number", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "country": { + "type": "object", + "properties": { + "name": { + "type": ["string", "null"] + }, + "code": { + "type": ["string", "null"] + }, + "timezone": { + "type": ["string", "null"] + } + } + }, + "officialSite": {} + } + }, + "webChannel": {}, + "dvdCountry": {}, + "externals": { + "type": "object", + "properties": { + "tvrage": { + "type": ["number", "null"] + }, + "thetvdb": { + "type": ["number", "null"] + }, + "imdb": { + "type": ["string", "null"] + } + } + }, + "image": { + "type": ["object", "null"], + "properties": { + "medium": { + "type": ["string", "null"] + }, + "original": { + "type": ["string", "null"] + } + } + }, + "summary": { + "type": ["string", "null"] + }, + "updated": { + "type": ["number", "null"] + }, + "_links": { + "type": "object", + "properties": { + "self": { + "type": "object", + "properties": { + "href": { + "type": ["string", "null"] + } + } + }, + "previousepisode": { + "type": "object", + "properties": { + "href": { + "type": ["string", "null"] + } + } + } + } + } + } + }, + "_links": { + "type": "object", + "properties": { + "self": { + "type": "object", + "properties": { + "href": { + "type": ["string", "null"] + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/schemas/future.json b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/schemas/future.json new file mode 100644 index 000000000000..568914ae1711 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/schemas/future.json @@ -0,0 +1,202 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["number", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "season": { + "type": ["number", "null"] + }, + "number": { + "type": ["number", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "airdate": { + "type": ["string", "null"] + }, + "airtime": { + "type": ["string", "null"] + }, + "airstamp": { + "type": ["string", "null"] + }, + "runtime": { + "type": ["number", "null"] + }, + "rating": { + "type": "object", + "properties": { + "average": {} + } + }, + "image": {}, + "summary": { + "type": ["string", "null"] + }, + "show": { + "type": "object", + "properties": { + "id": { + "type": ["number", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "language": { + "type": ["string", "null"] + }, + "genres": { + "type": "array", + "items": { + "type": ["string", "null"] + } + }, + "status": { + "type": ["string", "null"] + }, + "runtime": { + "type": ["number", "null"] + }, + "averageRuntime": { + "type": ["number", "null"] + }, + "premiered": { + "type": ["string", "null"] + }, + "ended": { + "type": ["string", "null"] + }, + "officialSite": {}, + "schedule": { + "type": "object", + "properties": { + "time": { + "type": ["string", "null"] + }, + "days": { + "type": "array", + "items": { + "type": ["string", "null"] + } + } + } + }, + "rating": { + "type": "object", + "properties": { + "average": {} + } + }, + "weight": { + "type": ["number", "null"] + }, + "network": { + "type": "object", + "properties": { + "id": { + "type": ["number", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "country": { + "type": "object", + "properties": { + "name": { + "type": ["string", "null"] + }, + "code": { + "type": ["string", "null"] + }, + "timezone": { + "type": ["string", "null"] + } + } + }, + "officialSite": {} + } + }, + "webChannel": {}, + "dvdCountry": {}, + "externals": { + "type": "object", + "properties": { + "tvrage": {}, + "thetvdb": { + "type": ["number", "null"] + }, + "imdb": { + "type": ["string", "null"] + } + } + }, + "image": { + "type": "object", + "properties": { + "medium": { + "type": ["string", "null"] + }, + "original": { + "type": ["string", "null"] + } + } + }, + "summary": { + "type": ["string", "null"] + }, + "updated": { + "type": ["number", "null"] + }, + "_links": { + "type": "object", + "properties": { + "self": { + "type": "object", + "properties": { + "href": { + "type": ["string", "null"] + } + } + }, + "previousepisode": { + "type": "object", + "properties": { + "href": { + "type": ["string", "null"] + } + } + } + } + } + } + }, + "_links": { + "type": "object", + "properties": { + "self": { + "type": "object", + "properties": { + "href": { + "type": ["string", "null"] + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/schemas/web.json b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/schemas/web.json new file mode 100644 index 000000000000..4be56bdc5ca7 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/schemas/web.json @@ -0,0 +1,208 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["number", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "season": { + "type": ["number", "null"] + }, + "number": { + "type": ["number", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "airdate": { + "type": ["string", "null"] + }, + "airtime": { + "type": ["string", "null"] + }, + "airstamp": { + "type": ["string", "null"] + }, + "runtime": { + "type": ["number", "null"] + }, + "rating": { + "type": "object", + "properties": { + "average": { + "type": ["number", "null"] + } + } + }, + "image": { + "type": ["object", "null"] + }, + "summary": { + "type": ["string", "null"] + }, + "show": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["number", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "language": { + "type": ["string", "null"] + }, + "genres": { + "type": "array", + "items": { + "type": ["string", "null"] + } + }, + "status": { + "type": ["string", "null"] + }, + "runtime": { + "type": ["number", "null"] + }, + "averageRuntime": { + "type": ["number", "null"] + }, + "premiered": { + "type": ["string", "null"] + }, + "ended": { + "type": ["string", "null"] + }, + "officialSite": {}, + "schedule": { + "type": "object", + "properties": { + "time": { + "type": ["string", "null"] + }, + "days": { + "type": "array", + "items": { + "type": ["string", "null"] + } + } + } + }, + "rating": { + "type": "object", + "properties": { + "average": { + "type": ["number", "null"] + } + } + }, + "weight": { + "type": ["number", "null"] + }, + "network": { + "type": "object", + "properties": { + "id": { + "type": ["number", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "country": { + "type": "object", + "properties": { + "name": { + "type": ["string", "null"] + }, + "code": { + "type": ["string", "null"] + }, + "timezone": { + "type": ["string", "null"] + } + } + }, + "officialSite": {} + } + }, + "webChannel": {}, + "dvdCountry": {}, + "externals": { + "type": "object", + "properties": { + "tvrage": {}, + "thetvdb": { + "type": ["number", "null"] + }, + "imdb": { + "type": ["string", "null"] + } + } + }, + "image": { + "type": ["object", "null"], + "properties": { + "medium": { + "type": ["string", "null"] + }, + "original": { + "type": ["string", "null"] + } + } + }, + "summary": { + "type": ["string", "null"] + }, + "updated": { + "type": ["number", "null"] + }, + "_links": { + "type": "object", + "properties": { + "self": { + "type": "object", + "properties": { + "href": { + "type": ["string", "null"] + } + } + }, + "previousepisode": { + "type": "object", + "properties": { + "href": { + "type": ["string", "null"] + } + } + } + } + } + } + }, + "_links": { + "type": "object", + "properties": { + "self": { + "type": "object", + "properties": { + "href": { + "type": ["string", "null"] + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/source.py b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/source.py new file mode 100644 index 000000000000..49dc934451b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceTvmazeSchedule(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "tvmaze_schedule.yaml"}) diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/spec.yaml b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/spec.yaml new file mode 100644 index 000000000000..f53564a37652 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/spec.yaml @@ -0,0 +1,37 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/tvmaze-schedule +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: TVMaze Schedule Spec + type: object + required: + - start_date + - domestic_schedule_country_code + additionalProperties: true + properties: + start_date: + type: string + description: Start date for TV schedule retrieval. May be in the future. + order: 0 + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + end_date: + type: string + description: | + End date for TV schedule retrieval. May be in the future. Optional. + order: 1 + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + domestic_schedule_country_code: + type: string + description: Country code for domestic TV schedule retrieval. + examples: + - US + - GB + web_schedule_country_code: + type: string + description: | + ISO 3166-1 country code for web TV schedule retrieval. Leave blank for + all countries plus global web channels (e.g. Netflix). Alternatively, + set to 'global' for just global web channels. + examples: + - US + - GB + - global diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/tvmaze_schedule.yaml b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/tvmaze_schedule.yaml new file mode 100644 index 000000000000..0b39d36dece0 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/tvmaze_schedule.yaml @@ -0,0 +1,72 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: [] + requester: + url_base: "https://api.tvmaze.com" + http_method: "GET" + request_options_provider: + request_parameters: + country: | + {{ + config['domestic_schedule_country_code'] + if options['name'] == 'domestic' + else config['web_schedule_country_code'].replace('global', '') + if options['name'] == 'web' + else '' + }} + stream_slicer: + type: DatetimeStreamSlicer + start_datetime: + datetime: "{{ config['start_date'] }}" + format: "%Y-%m-%d" + end_datetime: + datetime: "{{ config['end_date'] or today_utc() }}" + format: "%Y-%m-%d" + step: 1d + datetime_format: "%Y-%m-%d" + cursor_field: "airdate" + start_time_option: + field_name: "date" + inject_into: "request_parameter" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + requester: + $ref: "*ref(definitions.requester)" + stream_slicer: + $ref: "*ref(definitions.stream_slicer)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + domestic_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "domestic" + primary_key: "id" + path: "/schedule" + web_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "web" + primary_key: "id" + path: "/schedule" + future_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "future" + primary_key: "id" + path: "/schedule/full" + +streams: + - "*ref(definitions.domestic_stream)" + - "*ref(definitions.web_stream)" + - "*ref(definitions.future_stream)" + +check: + stream_names: + - "domestic" diff --git a/airbyte-integrations/connectors/source-twilio/Dockerfile b/airbyte-integrations/connectors/source-twilio/Dockerfile index 1b7b749e874c..321fa362410a 100644 --- a/airbyte-integrations/connectors/source-twilio/Dockerfile +++ b/airbyte-integrations/connectors/source-twilio/Dockerfile @@ -12,5 +12,5 @@ COPY main.py ./ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.12 +LABEL io.airbyte.version=0.1.13 LABEL io.airbyte.name=airbyte/source-twilio diff --git a/airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml b/airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml index cd4f3a170c7a..e970764760a7 100644 --- a/airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml @@ -16,6 +16,8 @@ tests: configured_catalog_path: "integration_tests/no_empty_streams_catalog.json" expect_records: path: "integration_tests/expected_records.txt" + empty_streams: ["alerts"] + timeout_seconds: 600 incremental: - config_path: "secrets/config.json" # usage records stream produces and error if cursor date gte than current date diff --git a/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.txt index f3353926de87..7e0f8873a16e 100644 --- a/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.txt @@ -31,7 +31,6 @@ {"stream": "available_phone_number_countries", "data": {"country_code": "DK", "country": "Denmark", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DK.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DK/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DK/Mobile.json"}}, "emitted_at": 1664560270815} {"stream": "available_phone_number_countries", "data": {"country_code": "UG", "country": "Uganda", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/UG.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/UG/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/UG/TollFree.json"}}, "emitted_at": 1664560270815} {"stream": "available_phone_number_countries", "data": {"country_code": "MX", "country": "Mexico", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/MX.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/MX/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/MX/TollFree.json"}}, "emitted_at": 1664560270816} -{"stream": "available_phone_number_countries", "data": {"country_code": "IS", "country": "Iceland", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IS.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IS/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IS/Mobile.json"}}, "emitted_at": 1664560270816} {"stream": "available_phone_number_countries", "data": {"country_code": "DZ", "country": "Algeria", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DZ.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DZ/Local.json"}}, "emitted_at": 1664560270816} {"stream": "available_phone_number_countries", "data": {"country_code": "ZA", "country": "South Africa", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/ZA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/ZA/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/ZA/Mobile.json"}}, "emitted_at": 1664560270816} {"stream": "available_phone_number_countries", "data": {"country_code": "HR", "country": "Croatia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HR/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HR/TollFree.json"}}, "emitted_at": 1664560270816} @@ -98,7 +97,6 @@ {"stream": "available_phone_number_countries", "data": {"country_code": "DK", "country": "Denmark", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DK.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DK/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DK/Mobile.json"}}, "emitted_at": 1664560271158} {"stream": "available_phone_number_countries", "data": {"country_code": "UG", "country": "Uganda", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/UG.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/UG/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/UG/TollFree.json"}}, "emitted_at": 1664560271158} {"stream": "available_phone_number_countries", "data": {"country_code": "MX", "country": "Mexico", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/MX.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/MX/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/MX/TollFree.json"}}, "emitted_at": 1664560271158} -{"stream": "available_phone_number_countries", "data": {"country_code": "IS", "country": "Iceland", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IS.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IS/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IS/Mobile.json"}}, "emitted_at": 1664560271158} {"stream": "available_phone_number_countries", "data": {"country_code": "DZ", "country": "Algeria", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DZ.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DZ/Local.json"}}, "emitted_at": 1664560271158} {"stream": "available_phone_number_countries", "data": {"country_code": "ZA", "country": "South Africa", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/ZA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/ZA/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/ZA/Mobile.json"}}, "emitted_at": 1664560271158} {"stream": "available_phone_number_countries", "data": {"country_code": "HR", "country": "Croatia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HR/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HR/TollFree.json"}}, "emitted_at": 1664560271158} @@ -145,17 +143,17 @@ {"stream": "available_phone_number_countries", "data": {"country_code": "AR", "country": "Argentina", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AR/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AR/TollFree.json"}}, "emitted_at": 1664560271163} {"stream": "incoming_phone_numbers", "data": {"sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "friendly_name": "2FA Number - PLEASE DO NOT TOUCH. Use another number for anythin", "phone_number": "+12056561170", "voice_url": "https://handler.twilio.com/twiml/EH7af811843f38093d724a5c2e80b3eabe", "voice_method": "POST", "voice_fallback_url": "", "voice_fallback_method": "POST", "voice_caller_id_lookup": false, "date_created": "2020-12-11T04:28:40Z", "date_updated": "2022-09-23T14:47:41Z", "sms_url": "https://webhooks.twilio.com/v1/Accounts/ACdade166c12e160e9ed0a6088226718fb/Flows/FWbd726b7110b21294a9f27a47f4ab0080", "sms_method": "POST", "sms_fallback_url": "", "sms_fallback_method": "POST", "address_requirements": "none", "beta": false, "capabilities": {"voice": true, "sms": true, "mms": true}, "status_callback": "", "status_callback_method": "POST", "api_version": "2010-04-01", "voice_application_sid": "", "sms_application_sid": "", "origin": "twilio", "trunk_sid": null, "emergency_status": "Active", "emergency_address_sid": null, "emergency_address_status": "unregistered", "address_sid": null, "identity_sid": null, "bundle_sid": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PNe40bd7f3ac343b32fd51275d2d5b3dcc.json", "status": "in-use"}, "emitted_at": 1655893245291} {"stream": "keys", "data": {"date_updated": "2021-02-01T07:30:21Z", "date_created": "2021-02-01T07:30:21Z", "friendly_name": "Studio API Key", "sid": "SK60085e9cfc3d94aa1b987b25c78067a9"}, "emitted_at": 1655893247168} -{"stream": "calls", "data": {"date_updated": "2022-06-17T22:28:34Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 61, "from": "+15312726629", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CAe71d3c7533543b5c81b1be3fc5affa2b", "queue_time": 0, "price": -0.017, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-06-17T22:27:33Z", "date_created": "2022-06-17T22:27:32Z", "from_formatted": "(531) 272-6629", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-06-17T22:28:34Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1655893249727} -{"stream": "calls", "data": {"date_updated": "2022-06-17T13:36:17Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 96, "from": "+17372040136", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CA0a47223735162e1a7df2738327bda2ab", "queue_time": 0, "price": -0.017, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-06-17T13:34:41Z", "date_created": "2022-06-17T13:34:41Z", "from_formatted": "(737) 204-0136", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-06-17T13:36:17Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1655893249739} -{"stream": "calls", "data": {"date_updated": "2022-06-16T20:02:43Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 124, "from": "+17372040136", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CAace5c8813c499253bbbff29ad0da0ccb", "queue_time": 0, "price": -0.0255, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-06-16T20:00:39Z", "date_created": "2022-06-16T20:00:39Z", "from_formatted": "(737) 204-0136", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-06-16T20:02:43Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1655893249745} -{"stream": "calls", "data": {"date_updated": "2022-06-02T12:54:05Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 5, "from": "+12059675338", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CAa24e9fbcb6eba3c8cfefc248a3c0b5b4", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-06-02T12:54:00Z", "date_created": "2022-06-02T12:54:00Z", "from_formatted": "(205) 967-5338", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa24e9fbcb6eba3c8cfefc248a3c0b5b4.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-06-02T12:54:05Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa24e9fbcb6eba3c8cfefc248a3c0b5b4/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa24e9fbcb6eba3c8cfefc248a3c0b5b4/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa24e9fbcb6eba3c8cfefc248a3c0b5b4/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa24e9fbcb6eba3c8cfefc248a3c0b5b4/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa24e9fbcb6eba3c8cfefc248a3c0b5b4/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa24e9fbcb6eba3c8cfefc248a3c0b5b4/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa24e9fbcb6eba3c8cfefc248a3c0b5b4/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1655893249752} -{"stream": "calls", "data": {"date_updated": "2022-05-26T22:14:18Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 69, "from": "+13343585579", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CA65f8d6ee9f8783233750f2b0f99cf1b3", "queue_time": 0, "price": -0.017, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-05-26T22:13:09Z", "date_created": "2022-05-26T22:13:09Z", "from_formatted": "(334) 358-5579", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA65f8d6ee9f8783233750f2b0f99cf1b3.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-05-26T22:14:18Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA65f8d6ee9f8783233750f2b0f99cf1b3/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA65f8d6ee9f8783233750f2b0f99cf1b3/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA65f8d6ee9f8783233750f2b0f99cf1b3/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA65f8d6ee9f8783233750f2b0f99cf1b3/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA65f8d6ee9f8783233750f2b0f99cf1b3/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA65f8d6ee9f8783233750f2b0f99cf1b3/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA65f8d6ee9f8783233750f2b0f99cf1b3/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1655893249756} -{"stream": "calls", "data": {"date_updated": "2022-05-24T23:00:40Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 31, "from": "+14156896198", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CA5b6907d5ebca072c9bd0f46952b886b6", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-05-24T23:00:09Z", "date_created": "2022-05-24T23:00:09Z", "from_formatted": "(415) 689-6198", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA5b6907d5ebca072c9bd0f46952b886b6.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-05-24T23:00:40Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA5b6907d5ebca072c9bd0f46952b886b6/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA5b6907d5ebca072c9bd0f46952b886b6/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA5b6907d5ebca072c9bd0f46952b886b6/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA5b6907d5ebca072c9bd0f46952b886b6/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA5b6907d5ebca072c9bd0f46952b886b6/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA5b6907d5ebca072c9bd0f46952b886b6/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA5b6907d5ebca072c9bd0f46952b886b6/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1655893249759} -{"stream": "calls", "data": {"date_updated": "2022-05-11T18:21:15Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 23, "from": "+12137661124", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CA696bd2d2e37ef8501f443807dce444a9", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-05-11T18:20:52Z", "date_created": "2022-05-11T18:20:52Z", "from_formatted": "(213) 766-1124", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA696bd2d2e37ef8501f443807dce444a9.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-05-11T18:21:15Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA696bd2d2e37ef8501f443807dce444a9/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA696bd2d2e37ef8501f443807dce444a9/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA696bd2d2e37ef8501f443807dce444a9/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA696bd2d2e37ef8501f443807dce444a9/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA696bd2d2e37ef8501f443807dce444a9/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA696bd2d2e37ef8501f443807dce444a9/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA696bd2d2e37ef8501f443807dce444a9/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1655893249762} -{"stream": "calls", "data": {"date_updated": "2022-04-20T17:33:25Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 5, "from": "+12059736828", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CAe86d27d7aba7c857135b46f52f578d0b", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-04-20T17:33:20Z", "date_created": "2022-04-20T17:33:20Z", "from_formatted": "(205) 973-6828", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe86d27d7aba7c857135b46f52f578d0b.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-04-20T17:33:25Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe86d27d7aba7c857135b46f52f578d0b/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe86d27d7aba7c857135b46f52f578d0b/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe86d27d7aba7c857135b46f52f578d0b/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe86d27d7aba7c857135b46f52f578d0b/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe86d27d7aba7c857135b46f52f578d0b/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe86d27d7aba7c857135b46f52f578d0b/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe86d27d7aba7c857135b46f52f578d0b/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1655893249765} -{"stream": "calls", "data": {"date_updated": "2022-04-06T21:01:01Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 6, "from": "+13017951000", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CAade9599c9cf53091c1787898093e2675", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-04-06T21:00:55Z", "date_created": "2022-04-06T21:00:55Z", "from_formatted": "(301) 795-1000", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAade9599c9cf53091c1787898093e2675.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-04-06T21:01:01Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAade9599c9cf53091c1787898093e2675/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAade9599c9cf53091c1787898093e2675/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAade9599c9cf53091c1787898093e2675/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAade9599c9cf53091c1787898093e2675/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAade9599c9cf53091c1787898093e2675/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAade9599c9cf53091c1787898093e2675/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAade9599c9cf53091c1787898093e2675/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1655893249767} -{"stream": "calls", "data": {"date_updated": "2022-04-06T20:57:37Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 6, "from": "+13017951000", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CAa3887d4de4849a630bc369351f300171", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-04-06T20:57:31Z", "date_created": "2022-04-06T20:57:31Z", "from_formatted": "(301) 795-1000", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa3887d4de4849a630bc369351f300171.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-04-06T20:57:37Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa3887d4de4849a630bc369351f300171/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa3887d4de4849a630bc369351f300171/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa3887d4de4849a630bc369351f300171/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa3887d4de4849a630bc369351f300171/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa3887d4de4849a630bc369351f300171/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa3887d4de4849a630bc369351f300171/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa3887d4de4849a630bc369351f300171/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1655893249769} -{"stream": "calls", "data": {"date_updated": "2022-03-13T23:56:37Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 13, "from": "+12059203962", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CA78611ecf5e7f101b1a59be31b8f520f7", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-03-13T23:56:24Z", "date_created": "2022-03-13T23:56:24Z", "from_formatted": "(205) 920-3962", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA78611ecf5e7f101b1a59be31b8f520f7.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-03-13T23:56:37Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA78611ecf5e7f101b1a59be31b8f520f7/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA78611ecf5e7f101b1a59be31b8f520f7/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA78611ecf5e7f101b1a59be31b8f520f7/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA78611ecf5e7f101b1a59be31b8f520f7/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA78611ecf5e7f101b1a59be31b8f520f7/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA78611ecf5e7f101b1a59be31b8f520f7/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA78611ecf5e7f101b1a59be31b8f520f7/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1655893249771} +{"stream": "calls", "data": {"date_updated": "2022-06-17T22:28:34Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 61, "from": "+15312726629", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CAe71d3c7533543b5c81b1be3fc5affa2b", "queue_time": 0, "price": -0.017, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-06-17T22:27:33Z", "date_created": "2022-06-17T22:27:32Z", "from_formatted": "(531) 272-6629", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-06-17T22:28:34Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/UserDefinedMessages.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/UserDefinedMessageSubscriptions.json"}}, "emitted_at": 1655893249727} +{"stream": "calls", "data": {"date_updated": "2022-06-17T13:36:17Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 96, "from": "+17372040136", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CA0a47223735162e1a7df2738327bda2ab", "queue_time": 0, "price": -0.017, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-06-17T13:34:41Z", "date_created": "2022-06-17T13:34:41Z", "from_formatted": "(737) 204-0136", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-06-17T13:36:17Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/UserDefinedMessages.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/UserDefinedMessageSubscriptions.json"}}, "emitted_at": 1655893249739} +{"stream": "calls", "data": {"date_updated": "2022-06-16T20:02:43Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 124, "from": "+17372040136", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CAace5c8813c499253bbbff29ad0da0ccb", "queue_time": 0, "price": -0.0255, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-06-16T20:00:39Z", "date_created": "2022-06-16T20:00:39Z", "from_formatted": "(737) 204-0136", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-06-16T20:02:43Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/UserDefinedMessages.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/UserDefinedMessageSubscriptions.json"}}, "emitted_at": 1655893249745} +{"stream": "calls", "data": {"date_updated": "2022-06-02T12:54:05Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 5, "from": "+12059675338", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CAa24e9fbcb6eba3c8cfefc248a3c0b5b4", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-06-02T12:54:00Z", "date_created": "2022-06-02T12:54:00Z", "from_formatted": "(205) 967-5338", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa24e9fbcb6eba3c8cfefc248a3c0b5b4.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-06-02T12:54:05Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa24e9fbcb6eba3c8cfefc248a3c0b5b4/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa24e9fbcb6eba3c8cfefc248a3c0b5b4/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa24e9fbcb6eba3c8cfefc248a3c0b5b4/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa24e9fbcb6eba3c8cfefc248a3c0b5b4/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa24e9fbcb6eba3c8cfefc248a3c0b5b4/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa24e9fbcb6eba3c8cfefc248a3c0b5b4/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa24e9fbcb6eba3c8cfefc248a3c0b5b4/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa24e9fbcb6eba3c8cfefc248a3c0b5b4/UserDefinedMessages.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa24e9fbcb6eba3c8cfefc248a3c0b5b4/UserDefinedMessageSubscriptions.json"}}, "emitted_at": 1655893249752} +{"stream": "calls", "data": {"date_updated": "2022-05-26T22:14:18Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 69, "from": "+13343585579", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CA65f8d6ee9f8783233750f2b0f99cf1b3", "queue_time": 0, "price": -0.017, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-05-26T22:13:09Z", "date_created": "2022-05-26T22:13:09Z", "from_formatted": "(334) 358-5579", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA65f8d6ee9f8783233750f2b0f99cf1b3.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-05-26T22:14:18Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA65f8d6ee9f8783233750f2b0f99cf1b3/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA65f8d6ee9f8783233750f2b0f99cf1b3/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA65f8d6ee9f8783233750f2b0f99cf1b3/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA65f8d6ee9f8783233750f2b0f99cf1b3/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA65f8d6ee9f8783233750f2b0f99cf1b3/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA65f8d6ee9f8783233750f2b0f99cf1b3/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA65f8d6ee9f8783233750f2b0f99cf1b3/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA65f8d6ee9f8783233750f2b0f99cf1b3/UserDefinedMessages.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA65f8d6ee9f8783233750f2b0f99cf1b3/UserDefinedMessageSubscriptions.json"}}, "emitted_at": 1655893249756} +{"stream": "calls", "data": {"date_updated": "2022-05-24T23:00:40Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 31, "from": "+14156896198", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CA5b6907d5ebca072c9bd0f46952b886b6", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-05-24T23:00:09Z", "date_created": "2022-05-24T23:00:09Z", "from_formatted": "(415) 689-6198", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA5b6907d5ebca072c9bd0f46952b886b6.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-05-24T23:00:40Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA5b6907d5ebca072c9bd0f46952b886b6/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA5b6907d5ebca072c9bd0f46952b886b6/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA5b6907d5ebca072c9bd0f46952b886b6/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA5b6907d5ebca072c9bd0f46952b886b6/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA5b6907d5ebca072c9bd0f46952b886b6/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA5b6907d5ebca072c9bd0f46952b886b6/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA5b6907d5ebca072c9bd0f46952b886b6/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA5b6907d5ebca072c9bd0f46952b886b6/UserDefinedMessages.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA5b6907d5ebca072c9bd0f46952b886b6/UserDefinedMessageSubscriptions.json"}}, "emitted_at": 1655893249759} +{"stream": "calls", "data": {"date_updated": "2022-05-11T18:21:15Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 23, "from": "+12137661124", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CA696bd2d2e37ef8501f443807dce444a9", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-05-11T18:20:52Z", "date_created": "2022-05-11T18:20:52Z", "from_formatted": "(213) 766-1124", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA696bd2d2e37ef8501f443807dce444a9.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-05-11T18:21:15Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA696bd2d2e37ef8501f443807dce444a9/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA696bd2d2e37ef8501f443807dce444a9/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA696bd2d2e37ef8501f443807dce444a9/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA696bd2d2e37ef8501f443807dce444a9/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA696bd2d2e37ef8501f443807dce444a9/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA696bd2d2e37ef8501f443807dce444a9/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA696bd2d2e37ef8501f443807dce444a9/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA696bd2d2e37ef8501f443807dce444a9/UserDefinedMessages.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA696bd2d2e37ef8501f443807dce444a9/UserDefinedMessageSubscriptions.json"}}, "emitted_at": 1655893249762} +{"stream": "calls", "data": {"date_updated": "2022-04-20T17:33:25Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 5, "from": "+12059736828", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CAe86d27d7aba7c857135b46f52f578d0b", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-04-20T17:33:20Z", "date_created": "2022-04-20T17:33:20Z", "from_formatted": "(205) 973-6828", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe86d27d7aba7c857135b46f52f578d0b.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-04-20T17:33:25Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe86d27d7aba7c857135b46f52f578d0b/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe86d27d7aba7c857135b46f52f578d0b/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe86d27d7aba7c857135b46f52f578d0b/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe86d27d7aba7c857135b46f52f578d0b/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe86d27d7aba7c857135b46f52f578d0b/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe86d27d7aba7c857135b46f52f578d0b/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe86d27d7aba7c857135b46f52f578d0b/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe86d27d7aba7c857135b46f52f578d0b/UserDefinedMessages.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe86d27d7aba7c857135b46f52f578d0b/UserDefinedMessageSubscriptions.json"}}, "emitted_at": 1655893249765} +{"stream": "calls", "data": {"date_updated": "2022-04-06T21:01:01Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 6, "from": "+13017951000", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CAade9599c9cf53091c1787898093e2675", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-04-06T21:00:55Z", "date_created": "2022-04-06T21:00:55Z", "from_formatted": "(301) 795-1000", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAade9599c9cf53091c1787898093e2675.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-04-06T21:01:01Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAade9599c9cf53091c1787898093e2675/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAade9599c9cf53091c1787898093e2675/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAade9599c9cf53091c1787898093e2675/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAade9599c9cf53091c1787898093e2675/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAade9599c9cf53091c1787898093e2675/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAade9599c9cf53091c1787898093e2675/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAade9599c9cf53091c1787898093e2675/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAade9599c9cf53091c1787898093e2675/UserDefinedMessages.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAade9599c9cf53091c1787898093e2675/UserDefinedMessageSubscriptions.json"}}, "emitted_at": 1655893249767} +{"stream": "calls", "data": {"date_updated": "2022-04-06T20:57:37Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 6, "from": "+13017951000", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CAa3887d4de4849a630bc369351f300171", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-04-06T20:57:31Z", "date_created": "2022-04-06T20:57:31Z", "from_formatted": "(301) 795-1000", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa3887d4de4849a630bc369351f300171.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-04-06T20:57:37Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa3887d4de4849a630bc369351f300171/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa3887d4de4849a630bc369351f300171/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa3887d4de4849a630bc369351f300171/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa3887d4de4849a630bc369351f300171/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa3887d4de4849a630bc369351f300171/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa3887d4de4849a630bc369351f300171/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa3887d4de4849a630bc369351f300171/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa3887d4de4849a630bc369351f300171/UserDefinedMessages.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAa3887d4de4849a630bc369351f300171/UserDefinedMessageSubscriptions.json"}}, "emitted_at": 1655893249769} +{"stream": "calls", "data": {"date_updated": "2022-03-13T23:56:37Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 13, "from": "+12059203962", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CA78611ecf5e7f101b1a59be31b8f520f7", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-03-13T23:56:24Z", "date_created": "2022-03-13T23:56:24Z", "from_formatted": "(205) 920-3962", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA78611ecf5e7f101b1a59be31b8f520f7.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-03-13T23:56:37Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA78611ecf5e7f101b1a59be31b8f520f7/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA78611ecf5e7f101b1a59be31b8f520f7/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA78611ecf5e7f101b1a59be31b8f520f7/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA78611ecf5e7f101b1a59be31b8f520f7/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA78611ecf5e7f101b1a59be31b8f520f7/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA78611ecf5e7f101b1a59be31b8f520f7/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA78611ecf5e7f101b1a59be31b8f520f7/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA78611ecf5e7f101b1a59be31b8f520f7/UserDefinedMessages.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA78611ecf5e7f101b1a59be31b8f520f7/UserDefinedMessageSubscriptions.json"}}, "emitted_at": 1655893249771} {"stream": "conferences", "data": {"status": "completed", "reason_conference_ended": "last-participant-left", "date_updated": "2022-09-23T14:44:41Z", "region": "us1", "friendly_name": "test_conference", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CFca0fa08200f55a6d60779d18b644a675.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "call_sid_ending_conference": "CA8858f240bdccfb3393def1682c2dbdf0", "sid": "CFca0fa08200f55a6d60779d18b644a675", "date_created": "2022-09-23T14:44:11Z", "api_version": "2010-04-01", "subresource_uris": {"participants": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CFca0fa08200f55a6d60779d18b644a675/Participants.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CFca0fa08200f55a6d60779d18b644a675/Recordings.json"}}, "emitted_at": 1663955824121} {"stream": "outgoing_caller_ids", "data": {"phone_number": "+14153597503", "date_updated": "2020-11-17T04:17:37Z", "friendly_name": "(415) 359-7503", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/OutgoingCallerIds/PN16ba111c0df5756cfe37044ed0ee3136.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "sid": "PN16ba111c0df5756cfe37044ed0ee3136", "date_created": "2020-11-17T04:17:37Z"}, "emitted_at": 1655893253929} {"stream": "outgoing_caller_ids", "data": {"phone_number": "+18023494963", "date_updated": "2020-12-11T04:28:02Z", "friendly_name": "(802) 349-4963", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/OutgoingCallerIds/PN726d635f970c30193cd12e7b994510a1.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "sid": "PN726d635f970c30193cd12e7b994510a1", "date_created": "2020-12-11T04:28:02Z"}, "emitted_at": 1655893253943} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json index 7ed73f1fb168..68bf0132a1a3 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json @@ -109,6 +109,12 @@ }, "streams": { "type": ["null", "string"] + }, + "user_defined_message_subscriptions": { + "type": ["null", "string"] + }, + "user_defined_messages": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/source.py b/airbyte-integrations/connectors/source-twilio/source_twilio/source.py index 274f60e347c7..e591df1a0843 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/source.py +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/source.py @@ -69,6 +69,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: "authenticator": auth, "start_date": config["start_date"], "lookback_window": config.get("lookback_window", 0), + "slice_step_map": config.get("slice_step_map", {}), } # Fix for `Date range specified in query is partially or entirely outside of retention window of 400 days` diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py index 1c3ea2412491..8a1c10307925 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py @@ -2,8 +2,10 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +import copy from abc import ABC, abstractmethod -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional +from functools import cached_property +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union from urllib.parse import parse_qsl, urlparse import pendulum @@ -11,7 +13,9 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import IncrementalMixin from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from requests.auth import AuthBase TWILIO_API_URL_BASE = "https://api.twilio.com" TWILIO_API_URL_BASE_VERSIONED = f"{TWILIO_API_URL_BASE}/2010-04-01/" @@ -24,9 +28,6 @@ class TwilioStream(HttpStream, ABC): page_size = 1000 transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) - def __init__(self, **kwargs): - super().__init__(**kwargs) - @property def data_field(self): return self.name @@ -73,9 +74,12 @@ def backoff_time(self, response: requests.Response) -> Optional[float]: return float(backoff_time) def request_params( - self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) + params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) params["PageSize"] = self.page_size if next_page_token: params.update(**next_page_token) @@ -98,17 +102,41 @@ def custom_transform_function(original_value: Any, field_schema: Mapping[str, An class IncrementalTwilioStream(TwilioStream, IncrementalMixin): time_filter_template = "YYYY-MM-DD HH:mm:ss[Z]" + # This attribute allows balancing between sync speed and memory consumption. + # The greater a slice is - the bigger memory consumption and the faster syncs are since fewer requests are made. + slice_step_default = pendulum.duration(years=1) + # time gap between when previous slice ends and current slice begins + slice_granularity = pendulum.duration(microseconds=1) state_checkpoint_interval = 1000 - def __init__(self, start_date: str = None, lookback_window: int = 0, **kwargs): - super().__init__(**kwargs) + def __init__( + self, + authenticator: Union[AuthBase, HttpAuthenticator], + start_date: str = None, + lookback_window: int = 0, + slice_step_map: Mapping[str, int] = None, + ): + super().__init__(authenticator) + slice_step = (slice_step_map or {}).get(self.name) + self._slice_step = slice_step and pendulum.duration(days=slice_step) self._start_date = start_date if start_date is not None else "1970-01-01T00:00:00Z" self._lookback_window = lookback_window self._cursor_value = None + @property + def slice_step(self): + return self._slice_step or self.slice_step_default + @property @abstractmethod - def incremental_filter_field(self) -> str: + def lower_boundary_filter_field(self) -> str: + """ + return: date filter query parameter name + """ + + @property + @abstractmethod + def upper_boundary_filter_field(self) -> str: """ return: date filter query parameter name """ @@ -123,7 +151,7 @@ def state(self) -> Mapping[str, Any]: return {} @state.setter - def state(self, value: Mapping[str, Any]): + def state(self, value: MutableMapping[str, Any]): if self._lookback_window and value.get(self.cursor_field): new_start_date = ( pendulum.parse(value[self.cursor_field]) - pendulum.duration(minutes=self._lookback_window) @@ -132,12 +160,38 @@ def state(self, value: Mapping[str, Any]): value[self.cursor_field] = new_start_date self._cursor_value = value.get(self.cursor_field) + def generate_date_ranges(self, super_slice: MutableMapping[str, Any]) -> Iterable[Optional[MutableMapping[str, Any]]]: + end_datetime = pendulum.now() + start_datetime = min(end_datetime, pendulum.parse(self.state.get(self.cursor_field, self._start_date))) + current_start = start_datetime + current_end = start_datetime + while current_end < end_datetime: + current_end = min(end_datetime, current_start + self.slice_step) + slice_ = copy.deepcopy(super_slice) if super_slice else {} + slice_[self.lower_boundary_filter_field] = current_start.format(self.time_filter_template) + slice_[self.upper_boundary_filter_field] = current_end.format(self.time_filter_template) + yield slice_ + current_start = current_end + self.slice_granularity + + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + for super_slice in super().stream_slices(sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state): + yield from self.generate_date_ranges(super_slice) + def request_params( - self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) - start_date = self.state.get(self.cursor_field, self._start_date) - params[self.incremental_filter_field] = pendulum.parse(start_date).format(self.time_filter_template) + params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + lower_bound = stream_slice and stream_slice.get(self.lower_boundary_filter_field) + upper_bound = stream_slice and stream_slice.get(self.upper_boundary_filter_field) + if lower_bound: + params[self.lower_boundary_filter_field] = lower_bound + if upper_bound: + params[self.upper_boundary_filter_field] = upper_bound return params def read_records( @@ -165,6 +219,7 @@ class TwilioNestedStream(TwilioStream): """ media_exist_validation = {} + uri_from_subresource = True def path(self, stream_slice: Mapping[str, Any], **kwargs): return stream_slice["subresource_uri"] @@ -180,21 +235,30 @@ def parent_stream(self) -> TwilioStream: :return: parent stream class """ + @cached_property + def parent_stream_instance(self): + return self.parent_stream(authenticator=self.authenticator) + + def parent_record_to_stream_slice(self, record: Mapping[str, Any]) -> Mapping[str, Any]: + return {"subresource_uri": record["subresource_uris"][self.subresource_uri_key]} + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - stream_instance = self.parent_stream(authenticator=self.authenticator) + stream_instance = self.parent_stream_instance stream_slices = stream_instance.stream_slices(sync_mode=SyncMode.full_refresh, cursor_field=stream_instance.cursor_field) for stream_slice in stream_slices: for item in stream_instance.read_records( sync_mode=SyncMode.full_refresh, stream_slice=stream_slice, cursor_field=stream_instance.cursor_field ): - if item.get("subresource_uris", {}).get(self.subresource_uri_key): + if not self.uri_from_subresource: + yield self.parent_record_to_stream_slice(item) + elif item.get("subresource_uris", {}).get(self.subresource_uri_key): validated = True for key, value in self.media_exist_validation.items(): validated = item.get(key) and item.get(key) != value if not validated: break if validated: - yield {"subresource_uri": item["subresource_uris"][self.subresource_uri_key]} + yield self.parent_record_to_stream_slice(item) class Accounts(TwilioStream): @@ -214,18 +278,13 @@ class DependentPhoneNumbers(TwilioNestedStream): parent_stream = Addresses url_base = TWILIO_API_URL_BASE_VERSIONED + uri_from_subresource = False def path(self, stream_slice: Mapping[str, Any], **kwargs): return f"Accounts/{stream_slice['account_sid']}/Addresses/{stream_slice['sid']}/DependentPhoneNumbers.json" - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - stream_instance = self.parent_stream(authenticator=self.authenticator) - stream_slices = stream_instance.stream_slices(sync_mode=SyncMode.full_refresh, cursor_field=stream_instance.cursor_field) - for stream_slice in stream_slices: - for item in stream_instance.read_records( - sync_mode=SyncMode.full_refresh, stream_slice=stream_slice, cursor_field=stream_instance.cursor_field - ): - yield {"sid": item["sid"], "account_sid": item["account_sid"]} + def parent_record_to_stream_slice(self, record: Mapping[str, Any]) -> Mapping[str, Any]: + return {"sid": record["sid"], "account_sid": record["account_sid"]} class Applications(TwilioNestedStream): @@ -287,22 +346,26 @@ class Keys(TwilioNestedStream): parent_stream = Accounts -class Calls(TwilioNestedStream, IncrementalTwilioStream): +class Calls(IncrementalTwilioStream, TwilioNestedStream): """https://www.twilio.com/docs/voice/api/call-resource#create-a-call-resource""" parent_stream = Accounts - incremental_filter_field = "EndTime>" + lower_boundary_filter_field = "EndTime>" + upper_boundary_filter_field = "EndTime<" cursor_field = "end_time" time_filter_template = "YYYY-MM-DD" + slice_granularity = pendulum.duration(days=1) -class Conferences(TwilioNestedStream, IncrementalTwilioStream): +class Conferences(IncrementalTwilioStream, TwilioNestedStream): """https://www.twilio.com/docs/voice/api/conference-resource#read-multiple-conference-resources""" parent_stream = Accounts - incremental_filter_field = "DateCreated>" + lower_boundary_filter_field = "DateCreated>" + upper_boundary_filter_field = "DateCreated<" cursor_field = "date_created" time_filter_template = "YYYY-MM-DD" + slice_granularity = pendulum.duration(days=1) class ConferenceParticipants(TwilioNestedStream): @@ -324,11 +387,12 @@ class OutgoingCallerIds(TwilioNestedStream): parent_stream = Accounts -class Recordings(TwilioNestedStream, IncrementalTwilioStream): +class Recordings(IncrementalTwilioStream, TwilioNestedStream): """https://www.twilio.com/docs/voice/api/recording#read-multiple-recording-resources""" parent_stream = Accounts - incremental_filter_field = "DateCreated>" + lower_boundary_filter_field = "DateCreated>" + upper_boundary_filter_field = "DateCreated<" cursor_field = "date_created" @@ -344,46 +408,35 @@ class Queues(TwilioNestedStream): parent_stream = Accounts -class Messages(TwilioNestedStream, IncrementalTwilioStream): +class Messages(IncrementalTwilioStream, TwilioNestedStream): """https://www.twilio.com/docs/sms/api/message-resource#read-multiple-message-resources""" parent_stream = Accounts - incremental_filter_field = "DateSent>" + slice_step_default = pendulum.duration(days=1) + lower_boundary_filter_field = "DateSent>" + upper_boundary_filter_field = "DateSent<" cursor_field = "date_sent" -class MessageMedia(TwilioNestedStream, IncrementalTwilioStream): +class MessageMedia(IncrementalTwilioStream, TwilioNestedStream): """https://www.twilio.com/docs/sms/api/media-resource#read-multiple-media-resources""" parent_stream = Messages data_field = "media_list" subresource_uri_key = "media" media_exist_validation = {"num_media": "0"} - incremental_filter_field = "DateCreated>" + lower_boundary_filter_field = "DateCreated>" + upper_boundary_filter_field = "DateCreated<" cursor_field = "date_created" - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - stream_instance = self.parent_stream( - authenticator=self.authenticator, start_date=self._start_date, lookback_window=self._lookback_window - ) - stream_slices = stream_instance.stream_slices(sync_mode=SyncMode.full_refresh, cursor_field=stream_instance.cursor_field) - for stream_slice in stream_slices: - for item in stream_instance.read_records( - sync_mode=SyncMode.full_refresh, stream_slice=stream_slice, cursor_field=stream_instance.cursor_field - ): - if item.get("subresource_uris", {}).get(self.subresource_uri_key): - validated = True - for key, value in self.media_exist_validation.items(): - validated = item.get(key) and item.get(key) != value - if not validated: - break - if validated: - - yield {"subresource_uri": item["subresource_uris"][self.subresource_uri_key]} + @cached_property + def parent_stream_instance(self): + return self.parent_stream(authenticator=self.authenticator, start_date=self._start_date, lookback_window=self._lookback_window) class UsageNestedStream(TwilioNestedStream): url_base = TWILIO_API_URL_BASE_VERSIONED + uri_from_subresource = False @property @abstractmethod @@ -395,23 +448,19 @@ def path_name(self) -> str: def path(self, stream_slice: Mapping[str, Any], **kwargs): return f"Accounts/{stream_slice['account_sid']}/Usage/{self.path_name}.json" - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - stream_instance = self.parent_stream(authenticator=self.authenticator) - stream_slices = stream_instance.stream_slices(sync_mode=SyncMode.full_refresh, cursor_field=stream_instance.cursor_field) - for stream_slice in stream_slices: - for item in stream_instance.read_records( - sync_mode=SyncMode.full_refresh, stream_slice=stream_slice, cursor_field=stream_instance.cursor_field - ): - yield {"account_sid": item["sid"]} + def parent_record_to_stream_slice(self, record: Mapping[str, Any]) -> Mapping[str, Any]: + return {"account_sid": record["sid"]} -class UsageRecords(UsageNestedStream, IncrementalTwilioStream): +class UsageRecords(IncrementalTwilioStream, UsageNestedStream): """https://www.twilio.com/docs/usage/api/usage-record#read-multiple-usagerecord-resources""" parent_stream = Accounts - incremental_filter_field = "StartDate" + lower_boundary_filter_field = "StartDate" + upper_boundary_filter_field = "EndDate" cursor_field = "start_date" time_filter_template = "YYYY-MM-DD" + slice_granularity = pendulum.duration(days=1) path_name = "Records" primary_key = [["account_sid"], ["category"]] changeable_fields = ["as_of"] @@ -429,7 +478,8 @@ class Alerts(IncrementalTwilioStream): """https://www.twilio.com/docs/usage/monitor-alert#read-multiple-alert-resources""" url_base = TWILIO_MONITOR_URL_BASE - incremental_filter_field = "StartDate" + lower_boundary_filter_field = "StartDate=" + upper_boundary_filter_field = "EndDate=" cursor_field = "date_generated" def path(self, **kwargs): diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py index b53d5412f046..a51d80aad627 100644 --- a/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py @@ -2,14 +2,16 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from contextlib import nullcontext from unittest.mock import patch +import pendulum import pytest import requests from airbyte_cdk.sources.streams.http import HttpStream from source_twilio.auth import HttpBasicAuthenticator from source_twilio.source import SourceTwilio -from source_twilio.streams import Accounts, Addresses, Calls, DependentPhoneNumbers, MessageMedia, Messages, UsageTriggers +from source_twilio.streams import Accounts, Addresses, Alerts, Calls, DependentPhoneNumbers, MessageMedia, TwilioNestedStream, UsageTriggers TEST_CONFIG = { "account_sid": "airbyte.io", @@ -135,29 +137,25 @@ class TestIncrementalTwilioStream: CONFIG.pop("auth_token") @pytest.mark.parametrize( - "stream_cls, expected", - [ - (Calls, "EndTime>"), - ], - ) - def test_incremental_filter_field(self, stream_cls, expected): - stream = stream_cls(**self.CONFIG) - result = stream.incremental_filter_field - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, next_page_token, expected", + "stream_cls, stream_slice, next_page_token, expected", [ ( Calls, + {"EndTime>": "2022-01-01", "EndTime<": "2022-01-02"}, {"Page": "2", "PageSize": "1000", "PageToken": "PAAD42931b949c0dedce94b2f93847fdcf95"}, - {"EndTime>": "2022-01-01", "Page": "2", "PageSize": "1000", "PageToken": "PAAD42931b949c0dedce94b2f93847fdcf95"}, + { + "EndTime>": "2022-01-01", + "EndTime<": "2022-01-02", + "Page": "2", + "PageSize": "1000", + "PageToken": "PAAD42931b949c0dedce94b2f93847fdcf95", + }, ), ], ) - def test_request_params(self, stream_cls, next_page_token, expected): + def test_request_params(self, stream_cls, stream_slice, next_page_token, expected): stream = stream_cls(**self.CONFIG) - result = stream.request_params(stream_state=None, next_page_token=next_page_token) + result = stream.request_params(stream_state=None, stream_slice=stream_slice, next_page_token=next_page_token) assert result == expected @pytest.mark.parametrize( @@ -172,6 +170,33 @@ def test_read_records(self, stream_cls, record, expected): result = stream.read_records(sync_mode=None) assert list(result) == expected + @pytest.mark.parametrize( + "stream_cls, parent_cls_records, extra_slice_keywords", + [ + (Calls, [{"subresource_uris": {"calls": "123"}}, {"subresource_uris": {"calls": "124"}}], ["subresource_uri"]), + (Alerts, [{}], []), + ], + ) + def test_stream_slices(self, mocker, stream_cls, parent_cls_records, extra_slice_keywords): + stream = stream_cls( + authenticator=TEST_CONFIG.get("authenticator"), start_date=pendulum.now().subtract(months=13).to_iso8601_string() + ) + expected_slices = 2 * len(parent_cls_records) # 2 per year slices per each parent slice + if isinstance(stream, TwilioNestedStream): + slices_mock_context = mocker.patch.object(stream.parent_stream_instance, "stream_slices", return_value=[{}]) + records_mock_context = mocker.patch.object(stream.parent_stream_instance, "read_records", return_value=parent_cls_records) + else: + slices_mock_context, records_mock_context = nullcontext(), nullcontext() + with slices_mock_context: + with records_mock_context: + slices = list(stream.stream_slices(sync_mode="incremental")) + assert len(slices) == expected_slices + for slice_ in slices: + if isinstance(stream, TwilioNestedStream): + for kw in extra_slice_keywords: + assert kw in slice_ + assert slice_[stream.lower_boundary_filter_field] <= slice_[stream.upper_boundary_filter_field] + class TestTwilioNestedStream: @@ -205,12 +230,6 @@ def test_media_exist_validation(self, stream_cls, expected): [{"subresource_uris": {"addresses": "123"}, "sid": "123", "account_sid": "456"}], [{"sid": "123", "account_sid": "456"}], ), - ( - MessageMedia, - Messages, - [{"subresource_uris": {"media": "1234"}, "num_media": "1", "sid": "123", "account_sid": "456"}], - [{"subresource_uri": "1234"}], - ), ], ) def test_stream_slices(self, stream_cls, parent_stream, record, expected): @@ -218,7 +237,7 @@ def test_stream_slices(self, stream_cls, parent_stream, record, expected): with patch.object(Accounts, "read_records", return_value=record): with patch.object(parent_stream, "stream_slices", return_value=record): with patch.object(parent_stream, "read_records", return_value=record): - result = stream.stream_slices() + result = stream.stream_slices(sync_mode="full_refresh") assert list(result) == expected diff --git a/airbyte-integrations/connectors/source-waiteraid/.dockerignore b/airbyte-integrations/connectors/source-waiteraid/.dockerignore new file mode 100644 index 000000000000..a89f0645c28d --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_waiteraid +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-waiteraid/Dockerfile b/airbyte-integrations/connectors/source-waiteraid/Dockerfile new file mode 100644 index 000000000000..a0d4d3a3c919 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_waiteraid ./source_waiteraid + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-waiteraid diff --git a/airbyte-integrations/connectors/source-waiteraid/README.md b/airbyte-integrations/connectors/source-waiteraid/README.md new file mode 100644 index 000000000000..a4f376768025 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/README.md @@ -0,0 +1,129 @@ +# Waiteraid Source + +This is the repository for the Waiteraid configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/waiteraid). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-waiteraid:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/waiteraid) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_waiteraid/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source waiteraid test creds` +and place them into `secrets/config.json`. +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-waiteraid:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-waiteraid:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-waiteraid:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-waiteraid:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-waiteraid:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-waiteraid:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-waiteraid:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-waiteraid:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-waiteraid/__init__.py b/airbyte-integrations/connectors/source-waiteraid/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-waiteraid/acceptance-test-config.yml b/airbyte-integrations/connectors/source-waiteraid/acceptance-test-config.yml new file mode 100644 index 000000000000..c5d65168c7ef --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-waiteraid:dev +tests: + spec: + - spec_path: "source_waiteraid/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.txt" + # extra_fields: no + # exact_order: no + # extra_records: yes + #incremental: # TODO if your connector does not implement incremental sync, remove this block + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-waiteraid/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-waiteraid/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-waiteraid/bootstrap.md b/airbyte-integrations/connectors/source-waiteraid/bootstrap.md new file mode 100644 index 000000000000..a92233214a44 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/bootstrap.md @@ -0,0 +1,11 @@ +## Streams + +Waiteraid is a REST API. Connector has the following streams, and all of them support full refresh only. + +* [Bookings](https://app.waiteraid.com/api-docs/index.html#api_get_bookings) + +## Authentication +Waiteraid API offers two types of [authentication methods](https://app.waiteraid.com/api-docs/index.html#auth_call). + +* API Keys - Keys are passed using HTTP Basic auth. +* Username and Password - Not supported by this connector. diff --git a/airbyte-integrations/connectors/source-waiteraid/build.gradle b/airbyte-integrations/connectors/source-waiteraid/build.gradle new file mode 100644 index 000000000000..6ea94362d508 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_waiteraid' +} diff --git a/airbyte-integrations/connectors/source-waiteraid/integration_tests/__init__.py b/airbyte-integrations/connectors/source-waiteraid/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-waiteraid/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-waiteraid/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..e6ccaa81eced --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "booking": { + "date": "2999-12-31" + } +} diff --git a/airbyte-integrations/connectors/source-waiteraid/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-waiteraid/integration_tests/acceptance.py new file mode 100644 index 000000000000..1302b2f57e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-waiteraid/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-waiteraid/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..7b8975e6ceac --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/integration_tests/configured_catalog.json @@ -0,0 +1,13 @@ +{ + "streams": [ + { + "stream": { + "name": "booking", + "json_schema": {}, + "supported_sync_modes": ["full_refresh","incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-waiteraid/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-waiteraid/integration_tests/invalid_config.json new file mode 100644 index 000000000000..1f9795718747 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/integration_tests/invalid_config.json @@ -0,0 +1 @@ +{"start_date": "2022-09-01", "auth_hash": "1nval1dk3y", "restid": "666"} diff --git a/airbyte-integrations/connectors/source-waiteraid/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-waiteraid/integration_tests/sample_config.json new file mode 100644 index 000000000000..ecc4913b84c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-waiteraid/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-waiteraid/integration_tests/sample_state.json new file mode 100644 index 000000000000..49e5c722434b --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "booking": { + "date": "2022-10-01" + } +} diff --git a/airbyte-integrations/connectors/source-waiteraid/main.py b/airbyte-integrations/connectors/source-waiteraid/main.py new file mode 100644 index 000000000000..c176f331c485 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_waiteraid import SourceWaiteraid + +if __name__ == "__main__": + source = SourceWaiteraid() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-waiteraid/requirements.txt b/airbyte-integrations/connectors/source-waiteraid/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-waiteraid/setup.py b/airbyte-integrations/connectors/source-waiteraid/setup.py new file mode 100644 index 000000000000..bb67ca6ef48a --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_waiteraid", + description="Source implementation for Waiteraid.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/__init__.py b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/__init__.py new file mode 100644 index 000000000000..0730ddfa3771 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceWaiteraid + +__all__ = ["SourceWaiteraid"] diff --git a/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/schemas/booking.json b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/schemas/booking.json new file mode 100644 index 000000000000..ec922ebe34dd --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/schemas/booking.json @@ -0,0 +1,151 @@ +{ + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "amount": { + "type": ["null", "number"] + }, + "children_amount": { + "type": ["null", "number"] + }, + "placed": { + "type": ["null", "number"] + }, + "placed_manually": { + "type": ["null", "number"] + }, + "start": { + "type": ["null", "number"] + }, + "end": { + "type": ["null", "number"] + }, + "length": { + "type": ["null", "number"] + }, + "status": { + "type": ["null", "string"] + }, + "arrived": { + "type": ["null", "number"] + }, + "all_seated": { + "type": ["null", "number"] + }, + "guest_left": { + "type": ["null", "number"] + }, + "comment": { + "type": ["null", "string"] + }, + "confirmed": { + "type": ["null", "number"] + }, + "waitinbar": { + "type": ["null", "number"] + }, + "internet_booking": { + "type": ["null", "number"] + }, + "internet_booking_confirmed": { + "type": ["null", "number"] + }, + "paid": { + "type": ["null", "number"] + }, + "langid": { + "type": ["null", "number"] + }, + "meal": { + "type": ["null", "string"] + }, + "tables": { + "type": ["null", "number"] + }, + "meal_abbr": { + "type": ["null", "string"] + }, + "table_ids": { + "type": ["null", "number"] + }, + "products": { + "type": ["null", "number"] + }, + "waitinlist": { + "type": ["null", "number"] + }, + "date": { + "type": ["null", "string"] + }, + "time": { + "type": ["null", "string"] + }, + "guest": { + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "firstname": { + "type": ["null", "string"] + }, + "lastname": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "postalcode": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "telephone": { + "type": ["null", "string"] + }, + "mobile": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "comment": { + "type": ["null", "string"] + }, + "other_contact": { + "type": ["null", "string"] + }, + "restaurant_newsletter": { + "type": ["null", "boolean"] + } + } + }, + "booking_date": { + "type": ["null", "number"] + }, + "payStarted": { + "type": ["null", "boolean"] + }, + "payClosed": { + "type": ["null", "boolean"] + }, + "payCharged": { + "type": ["null", "boolean"] + }, + "payActivated": { + "type": ["null", "boolean"] + }, + "has_message": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/source.py b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/source.py new file mode 100644 index 000000000000..7e2b4027a8ed --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceWaiteraid(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "waiteraid.yaml"}) diff --git a/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/spec.yaml b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/spec.yaml new file mode 100644 index 000000000000..7f00092a24d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/spec.yaml @@ -0,0 +1,28 @@ +documentationUrl: https://docsurl.com +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Waiteraid Spec + type: object + required: + - start_date + - auth_hash + - restid + additionalProperties: true + properties: + start_date: + title: Start Date + type: string + description: Start getting data from that date. + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + examples: + - YYYY-MM-DD + auth_hash: + title: Authentication Hash + type: string + description: Your WaiterAid API key, obtained from API request with Username and Password + airbyte_secret: true + restid: + title: Restaurant ID + type: string + description: Your WaiterAid restaurant id from API request to getRestaurants + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/waiteraid.yaml b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/waiteraid.yaml new file mode 100644 index 000000000000..123c84e0f031 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/waiteraid.yaml @@ -0,0 +1,48 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: [] + requester: + url_base: "https://app.waiteraid.com" + http_method: "POST" + request_options_provider: + request_parameters: + date: "{{ config['start_date'] }}" + auth_hash: "{{ config['auth_hash'] }}" + restid: "{{ config['restid'] }}" + stream_slicer: + type: "DatetimeStreamSlicer" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%d" + end_datetime: + datetime: "{{ now_utc() }}" + datetime_format: "%Y-%m-%d %H:%M:%S.%f" + step: "1d" + datetime_format: "%Y-%m-%d" + cursor_field: "{{ options['stream_cursor_field'] }}" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + requester: + $ref: "*ref(definitions.requester)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + booking_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "booking" + path: "/wa-api/searchBooking" + stream_cursor_field: "date" + +streams: + - "*ref(definitions.booking_stream)" + +check: + stream_names: + - "booking" diff --git a/airbyte-integrations/connectors/source-workable/.dockerignore b/airbyte-integrations/connectors/source-workable/.dockerignore new file mode 100644 index 000000000000..f0aecf092ee4 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_workable +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-workable/.python-version b/airbyte-integrations/connectors/source-workable/.python-version new file mode 100644 index 000000000000..a9f8d1be337f --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/.python-version @@ -0,0 +1 @@ +3.9.11 diff --git a/airbyte-integrations/connectors/source-workable/Dockerfile b/airbyte-integrations/connectors/source-workable/Dockerfile new file mode 100644 index 000000000000..625c27d740a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_workable ./source_workable + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-workable diff --git a/airbyte-integrations/connectors/source-workable/README.md b/airbyte-integrations/connectors/source-workable/README.md new file mode 100644 index 000000000000..88e9aaad1ca1 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/README.md @@ -0,0 +1,79 @@ +# Workable Source + +This is the repository for the Workable configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/workable). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-workable:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/workable) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_workable/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source workable test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-workable:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-workable:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-workable:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-workable:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-workable:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-workable:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-workable:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-workable:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-workable/__init__.py b/airbyte-integrations/connectors/source-workable/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-workable/acceptance-test-config.yml b/airbyte-integrations/connectors/source-workable/acceptance-test-config.yml new file mode 100644 index 000000000000..bd4e2a1b8f4c --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/acceptance-test-config.yml @@ -0,0 +1,20 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-workable:dev +tests: + spec: + - spec_path: "source_workable/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-workable/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-workable/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-workable/build.gradle b/airbyte-integrations/connectors/source-workable/build.gradle new file mode 100644 index 000000000000..0028e9d2c861 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_workable' +} diff --git a/airbyte-integrations/connectors/source-workable/integration_tests/__init__.py b/airbyte-integrations/connectors/source-workable/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-workable/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-workable/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..c1aa3481db58 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "jobs": { + "id": 0 + } +} diff --git a/airbyte-integrations/connectors/source-workable/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-workable/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-workable/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-workable/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..4830b6e1848f --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "jobs", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "candidates", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-workable/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-workable/integration_tests/invalid_config.json new file mode 100644 index 000000000000..8b389bcec35a --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/integration_tests/invalid_config.json @@ -0,0 +1,5 @@ +{ + "api_key": "", + "account_subdomain": "invalid_subdomain", + "start_date": "00001115T225616Z" +} diff --git a/airbyte-integrations/connectors/source-workable/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-workable/integration_tests/sample_config.json new file mode 100644 index 000000000000..c04684881f28 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "api_key": "", + "account_subdomain": "", + "start_date": "" +} diff --git a/airbyte-integrations/connectors/source-workable/main.py b/airbyte-integrations/connectors/source-workable/main.py new file mode 100644 index 000000000000..f52cd2dc1619 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_workable import SourceWorkable + +if __name__ == "__main__": + source = SourceWorkable() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-workable/requirements.txt b/airbyte-integrations/connectors/source-workable/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-workable/setup.py b/airbyte-integrations/connectors/source-workable/setup.py new file mode 100644 index 000000000000..257af451405d --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_workable", + description="Source implementation for Workable.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-workable/source_workable/__init__.py b/airbyte-integrations/connectors/source-workable/source_workable/__init__.py new file mode 100644 index 000000000000..36ebfc8b4f25 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/source_workable/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceWorkable + +__all__ = ["SourceWorkable"] diff --git a/airbyte-integrations/connectors/source-workable/source_workable/schemas/candidates.json b/airbyte-integrations/connectors/source-workable/source_workable/schemas/candidates.json new file mode 100644 index 000000000000..d06a94047edf --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/source_workable/schemas/candidates.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "headline": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "role": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-workable/source_workable/schemas/jobs.json b/airbyte-integrations/connectors/source-workable/source_workable/schemas/jobs.json new file mode 100644 index 000000000000..1b02f961a15d --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/source_workable/schemas/jobs.json @@ -0,0 +1,83 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "full_title": { + "type": ["null", "string"] + }, + "shortcode": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "department": { + "type": ["null", "string"] + }, + "department_hierarchy": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": "string" + } + } + }, + "minItems": 0 + }, + "url": { + "type": ["null", "string"] + }, + "application_url": { + "type": ["null", "string"] + }, + "shortlink": { + "type": ["null", "string"] + }, + "location": { + "type": "object", + "properties": { + "location_str": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "region": { + "type": ["null", "string"] + }, + "region_code": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "zip_code": { + "type": ["null", "string"] + }, + "telecommuting": { + "type": "boolean" + } + } + }, + "created_at": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-workable/source_workable/schemas/recruiters.json b/airbyte-integrations/connectors/source-workable/source_workable/schemas/recruiters.json new file mode 100644 index 000000000000..b776f6583ca0 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/source_workable/schemas/recruiters.json @@ -0,0 +1,15 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-workable/source_workable/schemas/stages.json b/airbyte-integrations/connectors/source-workable/source_workable/schemas/stages.json new file mode 100644 index 000000000000..14eb128a8922 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/source_workable/schemas/stages.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "slug": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "kind": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-workable/source_workable/source.py b/airbyte-integrations/connectors/source-workable/source_workable/source.py new file mode 100644 index 000000000000..cbb89faaacb7 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/source_workable/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceWorkable(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "workable.yaml"}) diff --git a/airbyte-integrations/connectors/source-workable/source_workable/spec.yaml b/airbyte-integrations/connectors/source-workable/source_workable/spec.yaml new file mode 100644 index 000000000000..00fd2eb461f9 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/source_workable/spec.yaml @@ -0,0 +1,28 @@ +documentationUrl: https://docs.airbyte.io/integrations/sources/workable +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Workable API Spec + type: object + required: + - api_key + - account_subdomain + - start_date + additionalProperties: true + properties: + api_key: + title: API Key + type: string + description: Your Workable API Key. See here. + airbyte_secret: true + account_subdomain: + title: Account Subdomain + type: string + description: Your Workable account subdomain, e.g. https://your_account_subdomain.workable.com. + start_date: + title: Start Date + type: string + description: "Get data that was created since this date (format: YYYYMMDDTHHMMSSZ)." + pattern: ^[0-9]{8}T[0-9]{6}Z$ + examples: + - 20150708T115616Z + - 20221115T225616Z diff --git a/airbyte-integrations/connectors/source-workable/source_workable/workable.yaml b/airbyte-integrations/connectors/source-workable/source_workable/workable.yaml new file mode 100644 index 000000000000..00d4a323b89f --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/source_workable/workable.yaml @@ -0,0 +1,84 @@ +version: "0.1.0" + +definitions: + page_size: 100 + schema_loader: + type: JsonSchema + file_path: "./source_sentry/schemas/{{ options.name }}.json" + selector: + extractor: + field_pointer: [] + requester: + url_base: "https://{{ config['account_subdomain'] }}.workable.com" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['api_key'] }}" + request_options_provider: + request_parameters: + created_after: "{{ config['start_date'] }}" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + requester: + $ref: "*ref(definitions.requester)" + paginator: + type: DefaultPaginator + url_base: "*ref(definitions.requester.url_base)" + limit_option: + inject_into: "request_parameter" + field_name: "" + page_token_option: + inject_into: "path" + page_size_option: + inject_into: "request_parameter" + field_name: "limit" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response.paging.next }}" + stop_condition: "{{ 'next' not in response['paging'] }}" + page_size: "*ref(definitions.page_size)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + jobs_stream: # https://workable.readme.io/reference/jobs + $ref: "*ref(definitions.base_stream)" + $options: + name: "jobs" + primary_key: "id" + path: "/spi/v3/jobs" + field_pointer: ["jobs"] + candidates_stream: # https://workable.readme.io/reference/job-candidates-index + $ref: "*ref(definitions.base_stream)" + $options: + name: "candidates" + primary_key: "id" + path: "/spi/v3/candidates" + field_pointer: ["candidates"] + stages_stream: # https://workable.readme.io/reference/stages + $ref: "*ref(definitions.base_stream)" + $options: + name: "stages" + primary_key: "slug" + path: "/spi/v3/stages" + field_pointer: ["stages"] + recruiters_stream: # https://workable.readme.io/reference/recruiters + $ref: "*ref(definitions.base_stream)" + $options: + name: "recruiters" + primary_key: "id" + path: "/spi/v3/recruiters" + field_pointer: ["recruiters"] + +streams: + - "*ref(definitions.jobs_stream)" + - "*ref(definitions.candidates_stream)" + - "*ref(definitions.stages_stream)" + - "*ref(definitions.recruiters_stream)" + +check: + stream_names: + - "jobs" + - "candidates" + - "stages" + - "recruiters" diff --git a/airbyte-integrations/connectors/source-zendesk-sell/.dockerignore b/airbyte-integrations/connectors/source-zendesk-sell/.dockerignore new file mode 100644 index 000000000000..7c3c867373cf --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_zendesk_sell +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-zendesk-sell/Dockerfile b/airbyte-integrations/connectors/source-zendesk-sell/Dockerfile new file mode 100644 index 000000000000..381fff970479 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.13-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_zendesk_sell ./source_zendesk_sell + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-zendesk-sell diff --git a/airbyte-integrations/connectors/source-zendesk-sell/README.md b/airbyte-integrations/connectors/source-zendesk-sell/README.md new file mode 100644 index 000000000000..741adc1a197c --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/README.md @@ -0,0 +1,133 @@ +# Zendesk Sell Source + +This is the repository for the Zendesk Sell source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/zendesk-sell). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-zendesk-sell:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/zendesk-sell) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zendesk_sell/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. +Note that the full process to generate api tokens is available [here](https://developer.zendesk.com/documentation/sales-crm/first-call/#1-generate-an-access-token) + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source zendesk-sell test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-zendesk-sell:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-zendesk-sell:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-zendesk-sell:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-sell:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-sell:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-zendesk-sell:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-zendesk-sell:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-zendesk-sell:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-config.yml new file mode 100644 index 000000000000..9e44467ce664 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-zendesk-sell:dev +tests: + spec: + - spec_path: "source_zendesk_sell/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.txt" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: # TODO if your connector does not implement incremental sync, remove this block + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-zendesk-sell/build.gradle b/airbyte-integrations/connectors/source-zendesk-sell/build.gradle new file mode 100644 index 000000000000..4b2aeaebf368 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_zendesk_sell' +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/__init__.py b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..871548c4aa59 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "contacts": { + "created_at": "33190962600" + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..3e127d892132 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/configured_catalog.json @@ -0,0 +1,220 @@ +{ + "streams": [ + { + "stream": { + "name": "pipelines", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "stages", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "deals", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "contacts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "leads", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "call_outcomes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "deal_sources", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "deal_unqualified_reasons", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "lead_conversions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "lead_sources", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "lead_unqualified_reasons", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "loss_reasons", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "notes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "orders", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "products", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "tags", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "tasks", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "text_messages", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "visit_outcomes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "visits", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/invalid_config.json new file mode 100644 index 000000000000..cb2463e43ef3 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "api_token": "" +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_config.json new file mode 100644 index 000000000000..84230b311b70 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "api_token": "c56afd675afe19b87a8bf810666927baa58854fd006da22795850ee957eec854" +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_state.json new file mode 100644 index 000000000000..b431595f17b8 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "contacts": { + "created_at": "1604479286" + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/main.py b/airbyte-integrations/connectors/source-zendesk-sell/main.py new file mode 100644 index 000000000000..e5cc433b0d36 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zendesk_sell import SourceZendeskSell + +if __name__ == "__main__": + source = SourceZendeskSell() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/requirements.txt b/airbyte-integrations/connectors/source-zendesk-sell/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-zendesk-sell/setup.py b/airbyte-integrations/connectors/source-zendesk-sell/setup.py new file mode 100644 index 000000000000..ab5314798e39 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.4", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_zendesk_sell", + description="Source implementation for Zendesk Sell.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/__init__.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/__init__.py new file mode 100644 index 000000000000..153d0c6b7b8e --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceZendeskSell + +__all__ = ["SourceZendeskSell"] diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/call_outcomes.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/call_outcomes.json new file mode 100644 index 000000000000..0ca39aa5f870 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/call_outcomes.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/calls.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/calls.json new file mode 100644 index 000000000000..94612ce6a249 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/calls.json @@ -0,0 +1,57 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "user_id": { + "type": ["null", "number"] + }, + "summary": { + "type": ["null", "string"] + }, + "recording_url": { + "type": ["null", "string"] + }, + "outcome_id": { + "type": ["null", "number"] + }, + "duration": { + "type": ["null", "number"] + }, + "phone_number": { + "type": ["null", "number"] + }, + "incoming": { + "type": ["null", "boolean"] + }, + "missed": { + "type": ["null", "boolean"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "resource_id": { + "type": ["null", "number"] + }, + "associated_deal_ids": { + "type": ["null", "array"] + }, + "made_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/collaborations.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/collaborations.json new file mode 100644 index 000000000000..89e857aa5031 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/collaborations.json @@ -0,0 +1,31 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "resource_id": { + "type": ["null", "number"] + }, + "collaborator_id": { + "type": ["null", "number"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/contacts.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/contacts.json new file mode 100644 index 000000000000..022461fce783 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/contacts.json @@ -0,0 +1,100 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "owner_id": { + "type": ["null", "number"] + }, + "is_organization": { + "type": ["null", "boolean"] + }, + "contact_id": { + "type": ["null", "number"] + }, + "parent_organization_id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "customer_status": { + "type": ["null", "string"] + }, + "prospect_status": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "industry": { + "type": ["null", "string"] + }, + "website": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "mobile": { + "type": ["null", "string"] + }, + "fax": { + "type": ["null", "string"] + }, + "twitter": { + "type": ["null", "string"] + }, + "facebook": { + "type": ["null", "string"] + }, + "linkedin": { + "type": ["null", "string"] + }, + "skype": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "billing_address": { + "type": ["null", "string"] + }, + "shipping_address": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "array"] + }, + "custom_fields": { + "type": ["null", "object"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_sources.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_sources.json new file mode 100644 index 000000000000..4c1d21218947 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_sources.json @@ -0,0 +1,28 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_unqualified_reasons.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_unqualified_reasons.json new file mode 100644 index 000000000000..0ca39aa5f870 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_unqualified_reasons.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deals.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deals.json new file mode 100644 index 000000000000..c0f3104de75a --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deals.json @@ -0,0 +1,87 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "owner_id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "number", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "hot": { + "type": ["null", "boolean"] + }, + "stage_id": { + "type": ["null", "number"] + }, + "last_stage_change_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "last_stage_change_by_id": { + "type": ["null", "number"] + }, + "last_activity_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "source_id": { + "type": ["null", "number"] + }, + "loss_reason_id": { + "type": ["null", "number"] + }, + "unqualified_reason_id": { + "type": ["null", "number"] + }, + "dropbox_email": { + "type": ["null", "string"] + }, + "contact_id": { + "type": ["null", "number"] + }, + "organization_id": { + "type": ["null", "number"] + }, + "estimated_close_date": { + "type": ["null", "string"] + }, + "customized_win_likelihood": { + "type": ["null", "number"] + }, + "tags": { + "type": ["null", "array"] + }, + "custom_fields": { + "type": ["null", "object"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "added_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_conversions.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_conversions.json new file mode 100644 index 000000000000..0f973d1cd534 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_conversions.json @@ -0,0 +1,29 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "lead_id": { + "type": ["null", "number"] + }, + "individual_id": { + "type": ["null", "number"] + }, + "organization_id": { + "type": ["null", "number"] + }, + "deal_id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_sources.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_sources.json new file mode 100644 index 000000000000..4c1d21218947 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_sources.json @@ -0,0 +1,28 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_unqualified_reasons.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_unqualified_reasons.json new file mode 100644 index 000000000000..f93853b912d9 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_unqualified_reasons.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/leads.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/leads.json new file mode 100644 index 000000000000..d31d0c53d6a0 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/leads.json @@ -0,0 +1,85 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "owner_id": { + "type": ["null", "number"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "organization_name": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "source_id": { + "type": ["null", "number"] + }, + "title": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "industry": { + "type": ["null", "string"] + }, + "website": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "mobile": { + "type": ["null", "string"] + }, + "fax": { + "type": ["null", "string"] + }, + "twitter": { + "type": ["null", "string"] + }, + "facebook": { + "type": ["null", "string"] + }, + "linkedin": { + "type": ["null", "string"] + }, + "skype": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "array"] + }, + "custom_fields": { + "type": ["null", "object"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/loss_reasons.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/loss_reasons.json new file mode 100644 index 000000000000..f93853b912d9 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/loss_reasons.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/notes.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/notes.json new file mode 100644 index 000000000000..3ac4eb3fe957 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/notes.json @@ -0,0 +1,40 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "resource_id": { + "type": ["null", "number"] + }, + "content": { + "type": ["null", "string"] + }, + "is_important": { + "type": ["null", "boolean"] + }, + "tags": { + "type": ["null", "array"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/orders.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/orders.json new file mode 100644 index 000000000000..d789fb749c46 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/orders.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "deal_id": { + "type": ["null", "number"] + }, + "discount": { + "type": ["null", "number"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/pipelines.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/pipelines.json new file mode 100644 index 000000000000..6d25e0550eef --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/pipelines.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "disabled": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/products.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/products.json new file mode 100644 index 000000000000..47f7873f7ec0 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/products.json @@ -0,0 +1,46 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "sku": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "max_discount": { + "type": ["null", "number"] + }, + "max_markup": { + "type": ["null", "number"] + }, + "cost": { + "type": ["null", "number"] + }, + "cost_currency": { + "type": ["null", "string"] + }, + "prices": { + "type": ["null", "array"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/stages.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/stages.json new file mode 100644 index 000000000000..0a826d2e8eae --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/stages.json @@ -0,0 +1,37 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "category": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "position": { + "type": ["null", "number"] + }, + "likelihood": { + "type": ["null", "number"] + }, + "pipeline_id": { + "type": ["null", "number"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tags.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tags.json new file mode 100644 index 000000000000..4c1d21218947 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tags.json @@ -0,0 +1,28 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tasks.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tasks.json new file mode 100644 index 000000000000..66a456a01b3e --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tasks.json @@ -0,0 +1,53 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "owner_id": { + "type": ["null", "number"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "resource_id": { + "type": ["null", "number"] + }, + "completed": { + "type": ["null", "boolean"] + }, + "completed_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "due_date": { + "type": ["null", "string"] + }, + "overdue": { + "type": ["null", "boolean"] + }, + "remind_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "content": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/text_messages.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/text_messages.json new file mode 100644 index 000000000000..2fbbc8db5c0e --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/text_messages.json @@ -0,0 +1,48 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "associated_deal_ids": { + "type": ["null", "array"] + }, + "content": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "incoming": { + "type": ["null", "boolean"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "resource_id": { + "type": ["null", "number"] + }, + "resource_phone_number": { + "type": ["null", "string"] + }, + "sent_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "user_id": { + "type": ["null", "number"] + }, + "user_phone_number": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/users.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/users.json new file mode 100644 index 000000000000..1c8c43003290 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/users.json @@ -0,0 +1,66 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "invited": { + "type": ["null", "boolean"] + }, + "confirmed": { + "type": ["null", "boolean"] + }, + "phone_number": { + "type": ["null", "string"] + }, + "role": { + "type": ["null", "string"] + }, + "roles": { + "type": ["null", "array"] + }, + "team_name": { + "type": ["null", "string"] + }, + "group": { + "type": ["null", "object"] + }, + "reports_to": { + "type": ["null", "number"] + }, + "timezone": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "deleted_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "zendesk_user_id": { + "type": ["null", "string"] + }, + "identity_type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visit_outcomes.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visit_outcomes.json new file mode 100644 index 000000000000..0ca39aa5f870 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visit_outcomes.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visits.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visits.json new file mode 100644 index 000000000000..5118a43d1929 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visits.json @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "outcome_id": { + "type": ["null", "number"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "resource_id": { + "type": ["null", "number"] + }, + "visited_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "resource_address": { + "type": ["null", "string"] + }, + "rep_location_verification_status": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py new file mode 100644 index 000000000000..d6868c021c71 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py @@ -0,0 +1,346 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import re +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + + +# Basic full refresh stream +class ZendeskSellStream(HttpStream, ABC): + """ + This class represents a stream output by the connector. + This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, + parsing responses etc.. + """ + + url_base = "https://api.getbase.com/v2/" + primary_key = None + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + try: + meta_links = {} + regex_page = r"[=?/]page[_=/-]?(\d{1,3})" + data = response.json() + if data: + meta_links = data.get("meta", {}).get("links") + if "next_page" in meta_links.keys(): + return {"page": int(re.findall(regex_page, meta_links["next_page"])[0])} + return None + except Exception as e: + self.logger.error(f"{e.__class__} occurred, while trying to get next page information from the following dict {meta_links}") + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + if next_page_token: + return {"page": next_page_token["page"]} + else: + return {} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + items = response.json()["items"] + yield from [item["data"] for item in items] + + +class Pipelines(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/pipelines/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "pipelines" + + +class Stages(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/stages/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "stages" + + +class Contacts(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/contacts/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "contacts" + + +class Deals(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deals/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "deals" + + +class Leads(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/leads/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "leads" + + +class CallOutcomes(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/call-outcomes/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "call_outcomes" + + +class Calls(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/calls/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "calls" + + +class Collaborations(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/collaborations/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "collaborations" + + +class DealSources(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deal-sources/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "deal_sources" + + +class DealUnqualifiedReasons(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deal-unqualified-reasons/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "deal_unqualified_reasons" + + +class LeadConversions(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/lead-conversions/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "lead_conversions" + + +class LeadSources(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/lead-sources/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "lead_sources" + + +class LeadUnqualifiedReasons(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/lead-unqualified-reasons/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "lead_unqualified_reasons" + + +class LossReasons(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/loss-reasons/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "loss_reasons" + + +class Notes(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/notes/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "notes" + + +class Orders(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/orders/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "orders" + + +class Products(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/products/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "products" + + +class Tags(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/tags/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "tags" + + +class Tasks(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/tasks/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "tasks" + + +class TextMessages(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/text-messages/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "text_messages" + + +class Users(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/users/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "users" + + +class VisitOutcomes(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/visit-outcomes/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "visit_outcomes" + + +class Visits(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/visits/ + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "visits" + + +# Source +class SourceZendeskSell(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + try: + authenticator = TokenAuthenticator(token=config["api_token"]) + stream = Contacts(authenticator=authenticator) + records = stream.read_records(sync_mode=SyncMode.full_refresh) + next(records) + return True, None + except Exception as e: + return False, e + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + auth = TokenAuthenticator(token=config["api_token"]) + return [ + Contacts(authenticator=auth), + Deals(authenticator=auth), + Leads(authenticator=auth), + Pipelines(authenticator=auth), + Stages(authenticator=auth), + CallOutcomes(authenticator=auth), + Calls(authenticator=auth), + Collaborations(authenticator=auth), + DealSources(authenticator=auth), + DealUnqualifiedReasons(authenticator=auth), + LeadConversions(authenticator=auth), + LeadSources(authenticator=auth), + LeadUnqualifiedReasons(authenticator=auth), + LossReasons(authenticator=auth), + Notes(authenticator=auth), + Orders(authenticator=auth), + Products(authenticator=auth), + Tags(authenticator=auth), + Tasks(authenticator=auth), + TextMessages(authenticator=auth), + Users(authenticator=auth), + VisitOutcomes(authenticator=auth), + Visits(authenticator=auth), + ] diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml new file mode 100644 index 000000000000..eb92f5a7083e --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml @@ -0,0 +1,15 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-sell +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Source Zendesk Sell Spec + type: object + required: + - api_token + properties: + api_token: + title: API token + type: string + description: "The API token for authenticating to Zendesk Sell" + examples: + - "f23yhd630otl94y85a8bf384958473pto95847fd006da49382716or937ruw059" + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/__init__.py b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py new file mode 100644 index 000000000000..063308dd0189 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from pytest import fixture +from source_zendesk_sell.source import SourceZendeskSell + + +@fixture +def config(): + return {"config": {"user_auth_key": "", "start_date": "2021-01-01T00:00:00Z", "outcome_names": ""}} + + +def test_check_connection(mocker, requests_mock, config): + source = SourceZendeskSell() + logger_mock, config_mock = MagicMock(), MagicMock() + requests_mock.get("https://api.getbase.com/v2/contacts", json={"items": [{"data": {}}]}) + + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker): + source = SourceZendeskSell() + config_mock = MagicMock() + streams = source.streams(config_mock) + expected_streams_number = 23 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py new file mode 100644 index 000000000000..aac02fc6c0e2 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py @@ -0,0 +1,151 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_zendesk_sell.source import ZendeskSellStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(ZendeskSellStream, "path", "v0/example_endpoint") + mocker.patch.object(ZendeskSellStream, "primary_key", "test_primary_key") + mocker.patch.object(ZendeskSellStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = ZendeskSellStream() + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +@pytest.mark.parametrize( + ("inputs", "expected_token"), + [ + ( + { + "items": [], + "meta": { + "type": "collection", + "count": 25, + "links": { + "self": "https://api.getbase.com/v2/contacts?page=2&per_page=25", + "first_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", + "prev_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", + "next_page": "https://api.getbase.com/v2/contacts?page=3&per_page=25", + }, + }, + }, + {"page": 3}, + ), + ( + { + "items": [], + "meta": { + "type": "collection", + "count": 25, + "links": { + "self": "https://api.getbase.com/v2/contacts?page=2&per_page=25", + "first_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", + "prev_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", + }, + }, + }, + None, + ), + ({None}, None), + ], +) +def test_next_page_token(mocker, requests_mock, patch_base_class, inputs, expected_token): + stream = ZendeskSellStream() + response = mocker.MagicMock() + response.json.return_value = inputs + assert stream.next_page_token(response) == expected_token + + +def test_parse_response(patch_base_class, mocker): + stream = ZendeskSellStream() + response = mocker.MagicMock() + response.json.return_value = { + "items": [ + { + "data": { + "id": 302488228, + "creator_id": 2393211, + "contact_id": 302488227, + "created_at": "2020-11-12T09:05:47Z", + "updated_at": "2022-03-23T16:53:22Z", + "title": None, + "name": "Octavia Squidington", + "first_name": "Octavia", + "last_name": "Squidington", + }, + "meta": {"version": 36, "type": "contact"}, + } + ], + "meta": { + "type": "collection", + "count": 25, + "links": { + "self": "https://api.getbase.com/v2/contacts?page=2&per_page=25", + "first_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", + "prev_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", + "next_page": "https://api.getbase.com/v2/contacts?page=3&per_page=25", + }, + }, + } + expected_parsed_object = { + "id": 302488228, + "creator_id": 2393211, + "contact_id": 302488227, + "created_at": "2020-11-12T09:05:47Z", + "updated_at": "2022-03-23T16:53:22Z", + "title": None, + "name": "Octavia Squidington", + "first_name": "Octavia", + "last_name": "Squidington", + } + assert next(stream.parse_response(response)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = ZendeskSellStream() + stream_slice = None + stream_state = None + next_page_token = {"page": 2} + expected_headers = {"page": 2} + assert stream.request_params(stream_slice, stream_state, next_page_token) == expected_headers + + +def test_http_method(patch_base_class): + stream = ZendeskSellStream() + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = ZendeskSellStream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = ZendeskSellStream() + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-zoom-singer/.dockerignore b/airbyte-integrations/connectors/source-zoom-singer/.dockerignore deleted file mode 100644 index 540fc080488d..000000000000 --- a/airbyte-integrations/connectors/source-zoom-singer/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -build -.venv - diff --git a/airbyte-integrations/connectors/source-zoom-singer/.gitignore b/airbyte-integrations/connectors/source-zoom-singer/.gitignore deleted file mode 100644 index 29fffc6a50cc..000000000000 --- a/airbyte-integrations/connectors/source-zoom-singer/.gitignore +++ /dev/null @@ -1 +0,0 @@ -NEW_SOURCE_CHECKLIST.md diff --git a/airbyte-integrations/connectors/source-zoom-singer/build.gradle b/airbyte-integrations/connectors/source-zoom-singer/build.gradle deleted file mode 100644 index 408979685851..000000000000 --- a/airbyte-integrations/connectors/source-zoom-singer/build.gradle +++ /dev/null @@ -1,21 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-standard-source-test-file' -} - -airbytePython { - moduleDirectory 'source_zoom_singer' -} - -airbyteStandardSourceTestFile { - specPath = "source_zoom_singer/spec.json" - configPath = "secrets/config.json" - configuredCatalogPath = "sample_files/configured_catalog.json" -} - - - -dependencies { - implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) -} diff --git a/airbyte-integrations/connectors/source-zoom-singer/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-zoom-singer/sample_files/configured_catalog.json deleted file mode 100644 index 0d10357b06a4..000000000000 --- a/airbyte-integrations/connectors/source-zoom-singer/sample_files/configured_catalog.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "users", - "json_schema": { - "properties": { - "id": { - "type": ["string"] - }, - "first_name": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "integer"] - }, - "status": { - "type": ["null", "string"] - }, - "pmi": { - "type": ["null", "integer"] - }, - "timezone": { - "type": ["null", "string"] - }, - "dept": { - "type": ["null", "string"] - }, - "created_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "last_login_time": { - "format": "date-time", - "type": ["null", "string"] - }, - "last_client_version": { - "type": ["null", "string"] - }, - "group_ids": { - "items": { - "type": ["string"] - }, - "type": ["null", "array"] - }, - "im_group_ids": { - "items": { - "type": ["string"] - }, - "type": ["null", "array"] - }, - "verified": { - "type": ["null", "integer"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-zoom-singer/sample_files/full_configured_catalog.json b/airbyte-integrations/connectors/source-zoom-singer/sample_files/full_configured_catalog.json deleted file mode 100644 index 6859a4e1e9e0..000000000000 --- a/airbyte-integrations/connectors/source-zoom-singer/sample_files/full_configured_catalog.json +++ /dev/null @@ -1,1462 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "meeting_polls", - "json_schema": { - "properties": { - "id": { - "type": ["string"] - }, - "meeting_id": { - "type": ["string"] - }, - "status": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "questions": { - "items": { - "properties": { - "name": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "answers": { - "items": { - "type": ["string"] - }, - "type": ["null", "array"] - } - }, - "type": ["object"], - "additionalProperties": false - }, - "type": ["null", "array"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "report_webinar_participants", - "json_schema": { - "properties": { - "id": { - "type": ["string"] - }, - "user_id": { - "type": ["string"] - }, - "webinar_id": { - "type": ["string"] - }, - "name": { - "type": ["null", "string"] - }, - "user_email": { - "type": ["null", "string"] - }, - "join_time": { - "format": "date-time", - "type": ["null", "string"] - }, - "leave_time": { - "format": "date-time", - "type": ["null", "string"] - }, - "duration": { - "type": ["null", "string"] - }, - "attentiveness_score": { - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "meeting_files", - "json_schema": { - "properties": { - "meeting_uuid": { - "type": ["string"] - }, - "file_name": { - "type": ["null", "string"] - }, - "download_url": { - "type": ["null", "string"] - }, - "file_size": { - "type": ["null", "integer"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "meeting_registrants", - "json_schema": { - "properties": { - "id": { - "type": ["string"] - }, - "meeting_id": { - "type": ["string"] - }, - "email": { - "type": ["null", "string"] - }, - "first_name": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "address": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "county": { - "type": ["null", "string"] - }, - "zip": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "industry": { - "type": ["null", "string"] - }, - "org": { - "type": ["null", "string"] - }, - "job_title": { - "type": ["null", "string"] - }, - "purchasing_time_frame": { - "type": ["null", "string"] - }, - "role_in_purchase_process": { - "type": ["null", "string"] - }, - "no_of_employees": { - "type": ["null", "string"] - }, - "comments": { - "type": ["null", "string"] - }, - "custom_questions": { - "items": { - "properties": { - "title": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - }, - "type": ["object"], - "additionalProperties": false - }, - "type": ["null", "array"] - }, - "status": { - "type": ["null", "string"] - }, - "create_time": { - "format": "date-time", - "type": ["null", "string"] - }, - "join_url": { - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "users", - "json_schema": { - "properties": { - "id": { - "type": ["string"] - }, - "first_name": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "integer"] - }, - "status": { - "type": ["null", "string"] - }, - "pmi": { - "type": ["null", "integer"] - }, - "timezone": { - "type": ["null", "string"] - }, - "dept": { - "type": ["null", "string"] - }, - "created_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "last_login_time": { - "format": "date-time", - "type": ["null", "string"] - }, - "last_client_version": { - "type": ["null", "string"] - }, - "group_ids": { - "items": { - "type": ["string"] - }, - "type": ["null", "array"] - }, - "im_group_ids": { - "items": { - "type": ["string"] - }, - "type": ["null", "array"] - }, - "verified": { - "type": ["null", "integer"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "webinar_poll_results", - "json_schema": { - "properties": { - "webinar_uuid": { - "type": ["string"] - }, - "email": { - "type": ["string"] - }, - "name": { - "type": ["null", "string"] - }, - "question_details": { - "items": { - "properties": { - "question": { - "type": ["null", "string"] - }, - "answer": { - "type": ["null", "string"] - } - }, - "type": ["object"], - "additionalProperties": false - }, - "type": ["null", "array"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "webinar_files", - "json_schema": { - "properties": { - "webinar_uuid": { - "type": ["string"] - }, - "file_name": { - "type": ["null", "string"] - }, - "download_url": { - "type": ["null", "string"] - }, - "file_size": { - "type": ["null", "integer"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "webinar_polls", - "json_schema": { - "properties": { - "id": { - "type": ["string"] - }, - "webinar_id": { - "type": ["string"] - }, - "status": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "questions": { - "items": { - "properties": { - "name": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "answers": { - "items": { - "type": ["string"] - }, - "type": ["null", "array"] - } - }, - "type": ["object"], - "additionalProperties": false - }, - "type": ["null", "array"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "meetings", - "json_schema": { - "properties": { - "id": { - "type": ["integer"] - }, - "meeting_id": { - "type": ["string"] - }, - "uuid": { - "type": ["string"] - }, - "host_id": { - "type": ["null", "string"] - }, - "topic": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "integer"] - }, - "status": { - "type": ["null", "string"] - }, - "start_time": { - "format": "date-time", - "type": ["null", "string"] - }, - "duration": { - "type": ["null", "integer"] - }, - "timezone": { - "type": ["null", "string"] - }, - "created_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "agenda": { - "type": ["null", "string"] - }, - "start_url": { - "type": ["null", "string"] - }, - "join_url": { - "type": ["null", "string"] - }, - "password": { - "type": ["null", "string"] - }, - "h323_password": { - "type": ["null", "string"] - }, - "encrypted_Password": { - "type": ["null", "string"] - }, - "pmi": { - "type": ["null", "integer"] - }, - "tracking_fields": { - "items": { - "properties": { - "field": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - }, - "type": ["object"], - "additionalProperties": false - }, - "type": ["null", "array"] - }, - "occurences": { - "items": { - "properties": { - "occurence_id": { - "type": ["null", "string"] - }, - "start_time": { - "format": "date-time", - "type": ["null", "string"] - }, - "duration": { - "type": ["null", "integer"] - }, - "status": { - "type": ["null", "string"] - } - }, - "type": ["object"], - "additionalProperties": false - }, - "type": ["null", "array"] - }, - "settings": { - "properties": { - "host_video": { - "type": ["null", "boolean"] - }, - "participant_video": { - "type": ["null", "boolean"] - }, - "cn_meeting": { - "type": ["null", "boolean"] - }, - "in_meeting": { - "type": ["null", "boolean"] - }, - "join_before_host": { - "type": ["null", "boolean"] - }, - "mute_upon_entry": { - "type": ["null", "boolean"] - }, - "watermark": { - "type": ["null", "boolean"] - }, - "use_pmi": { - "type": ["null", "boolean"] - }, - "approval_type": { - "type": ["null", "integer"] - }, - "registration_type": { - "type": ["null", "integer"] - }, - "audio": { - "type": ["null", "string"] - }, - "auto_recording": { - "type": ["null", "string"] - }, - "enforce_login": { - "type": ["null", "boolean"] - }, - "enforce_login_domains": { - "type": ["null", "string"] - }, - "alternative_hosts": { - "type": ["null", "string"] - }, - "close_registration": { - "type": ["null", "boolean"] - }, - "waiting_room": { - "type": ["null", "boolean"] - }, - "global_dial_in_countries": { - "items": { - "type": ["string"] - }, - "type": ["null", "array"] - }, - "global_dial_in_numbers": { - "items": { - "properties": { - "country": { - "type": ["null", "string"] - }, - "country_name": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "number": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - } - }, - "type": ["object"], - "additionalProperties": false - }, - "type": ["null", "array"] - }, - "contact_name": { - "type": ["null", "boolean"] - }, - "contact_email": { - "type": ["null", "boolean"] - }, - "registrants_confirmation_email": { - "type": ["null", "boolean"] - }, - "registrants_email_notification": { - "type": ["null", "boolean"] - }, - "meeting_authentication": { - "type": ["null", "boolean"] - }, - "authentication_option": { - "type": ["null", "string"] - }, - "authentication_domains": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"], - "additionalProperties": false - }, - "recurrence": { - "properties": { - "type": { - "type": ["null", "integer"] - }, - "repeat_interval": { - "type": ["null", "integer"] - }, - "weekly_days": { - "type": ["null", "integer"] - }, - "monthly_day": { - "type": ["null", "integer"] - }, - "monthly_week": { - "type": ["null", "integer"] - }, - "monthly_week_day": { - "type": ["null", "integer"] - }, - "end_times": { - "type": ["null", "integer"] - }, - "end_date_time": { - "format": "date-time", - "type": ["null", "string"] - } - }, - "type": ["null", "object"], - "additionalProperties": false - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "meeting_questions", - "json_schema": { - "properties": { - "meeting_id": { - "type": ["string"] - }, - "questions": { - "items": { - "properties": { - "field_name": { - "type": ["null", "string"] - }, - "required": { - "type": ["null", "boolean"] - } - }, - "type": ["object"], - "additionalProperties": false - }, - "type": ["null", "array"] - }, - "custom_questions": { - "items": { - "properties": { - "title": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "required": { - "type": ["null", "boolean"] - }, - "answers": { - "items": { - "type": ["string"] - }, - "type": ["null", "array"] - } - }, - "type": ["object"], - "additionalProperties": false - }, - "type": ["null", "array"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "webinar_registrants", - "json_schema": { - "properties": { - "id": { - "type": ["string"] - }, - "webinar_id": { - "type": ["string"] - }, - "email": { - "type": ["null", "string"] - }, - "first_name": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "address": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "county": { - "type": ["null", "string"] - }, - "zip": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "industry": { - "type": ["null", "string"] - }, - "org": { - "type": ["null", "string"] - }, - "job_title": { - "type": ["null", "string"] - }, - "purchasing_time_frame": { - "type": ["null", "string"] - }, - "role_in_purchase_process": { - "type": ["null", "string"] - }, - "no_of_employees": { - "type": ["null", "string"] - }, - "comments": { - "type": ["null", "string"] - }, - "custom_questions": { - "items": { - "properties": { - "title": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - }, - "type": ["object"], - "additionalProperties": false - }, - "type": ["null", "array"] - }, - "status": { - "type": ["null", "string"] - }, - "create_time": { - "format": "date-time", - "type": ["null", "string"] - }, - "join_url": { - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "webinar_panelists", - "json_schema": { - "properties": { - "id": { - "type": ["string"] - }, - "webinar_id": { - "type": ["string"] - }, - "name": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "join_url": { - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "meeting_poll_results", - "json_schema": { - "properties": { - "meeting_uuid": { - "type": ["string"] - }, - "email": { - "type": ["string"] - }, - "name": { - "type": ["null", "string"] - }, - "question_details": { - "items": { - "properties": { - "question": { - "type": ["null", "string"] - }, - "answer": { - "type": ["null", "string"] - } - }, - "type": ["object"], - "additionalProperties": false - }, - "type": ["null", "array"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "webinar_questions", - "json_schema": { - "properties": { - "webinar_id": { - "type": ["string"] - }, - "questions": { - "items": { - "properties": { - "field_name": { - "type": ["null", "string"] - }, - "required": { - "type": ["null", "boolean"] - } - }, - "type": ["object"], - "additionalProperties": false - }, - "type": ["null", "array"] - }, - "custom_questions": { - "items": { - "properties": { - "title": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "required": { - "type": ["null", "boolean"] - }, - "answers": { - "items": { - "type": ["string"] - }, - "type": ["null", "array"] - } - }, - "type": ["object"], - "additionalProperties": false - }, - "type": ["null", "array"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "report_meeting_participants", - "json_schema": { - "properties": { - "id": { - "type": ["string"] - }, - "user_id": { - "type": ["string"] - }, - "meeting_id": { - "type": ["string"] - }, - "name": { - "type": ["null", "string"] - }, - "user_email": { - "type": ["null", "string"] - }, - "join_time": { - "format": "date-time", - "type": ["null", "string"] - }, - "leave_time": { - "format": "date-time", - "type": ["null", "string"] - }, - "duration": { - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "webinars", - "json_schema": { - "properties": { - "uuid": { - "type": ["string"] - }, - "id": { - "type": ["string"] - }, - "host_id": { - "type": ["null", "string"] - }, - "topic": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "integer"] - }, - "start_time": { - "format": "date-time", - "type": ["null", "string"] - }, - "duration": { - "type": ["null", "integer"] - }, - "timezone": { - "type": ["null", "string"] - }, - "agenda": { - "type": ["null", "string"] - }, - "created_at": { - "format": "date-time", - "type": ["null", "string"] - }, - "start_url": { - "type": ["null", "string"] - }, - "join_url": { - "type": ["null", "string"] - }, - "tracking_fields": { - "items": { - "properties": { - "field": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - }, - "type": ["object"], - "additionalProperties": false - }, - "type": ["null", "array"] - }, - "occurences": { - "items": { - "properties": { - "occurence_id": { - "type": ["null", "string"] - }, - "start_time": { - "format": "date-time", - "type": ["null", "string"] - }, - "duration": { - "type": ["null", "integer"] - }, - "status": { - "type": ["null", "string"] - } - }, - "type": ["object"], - "additionalProperties": false - }, - "type": ["null", "array"] - }, - "settings": { - "properties": { - "host_video": { - "type": ["null", "boolean"] - }, - "panelists_video": { - "type": ["null", "boolean"] - }, - "practice_session": { - "type": ["null", "boolean"] - }, - "hd_video": { - "type": ["null", "boolean"] - }, - "approval_type": { - "type": ["null", "integer"] - }, - "registration_type": { - "type": ["null", "integer"] - }, - "audio": { - "type": ["null", "string"] - }, - "auto_recording": { - "type": ["null", "string"] - }, - "enforce_login": { - "type": ["null", "boolean"] - }, - "enforce_login_domains": { - "type": ["null", "string"] - }, - "alternative_hosts": { - "type": ["null", "string"] - }, - "close_registration": { - "type": ["null", "boolean"] - }, - "show_share_button": { - "type": ["null", "boolean"] - }, - "allow_multiple_devices": { - "type": ["null", "boolean"] - }, - "on_demand": { - "type": ["null", "boolean"] - }, - "global_dial_in_countries": { - "items": { - "type": ["string"] - }, - "type": ["null", "array"] - }, - "contact_name": { - "type": ["null", "boolean"] - }, - "contact_email": { - "type": ["null", "boolean"] - }, - "registrants_confirmation_email": { - "type": ["null", "boolean"] - }, - "registrants_restrict_number": { - "type": ["null", "integer"] - }, - "notify_registrants": { - "type": ["null", "boolean"] - }, - "post_webinar_survey": { - "type": ["null", "boolean"] - }, - "survey_url": { - "type": ["null", "string"] - }, - "registrants_email_notification": { - "type": ["null", "boolean"] - }, - "meeting_authentication": { - "type": ["null", "boolean"] - }, - "authentication_option": { - "type": ["null", "string"] - }, - "authentication_domains": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"], - "additionalProperties": false - }, - "recurrence": { - "properties": { - "type": { - "type": ["null", "integer"] - }, - "repeat_interval": { - "type": ["null", "integer"] - }, - "weekly_days": { - "type": ["null", "integer"] - }, - "monthly_day": { - "type": ["null", "integer"] - }, - "monthly_week": { - "type": ["null", "integer"] - }, - "monthly_week_day": { - "type": ["null", "integer"] - }, - "end_times": { - "type": ["null", "integer"] - }, - "end_date_time": { - "format": "date-time", - "type": ["null", "string"] - } - }, - "type": ["null", "object"], - "additionalProperties": false - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "webinar_qna_results", - "json_schema": { - "properties": { - "webinar_uuid": { - "type": ["string"] - }, - "email": { - "type": ["string"] - }, - "name": { - "type": ["null", "string"] - }, - "question_details": { - "items": { - "properties": { - "question": { - "type": ["null", "string"] - }, - "answer": { - "type": ["null", "string"] - } - }, - "type": ["object"], - "additionalProperties": false - }, - "type": ["null", "array"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "webinar_absentees", - "json_schema": { - "properties": { - "id": { - "type": ["string"] - }, - "webinar_uuid": { - "type": ["string"] - }, - "email": { - "type": ["null", "string"] - }, - "first_name": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "address": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "county": { - "type": ["null", "string"] - }, - "zip": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "industry": { - "type": ["null", "string"] - }, - "org": { - "type": ["null", "string"] - }, - "job_title": { - "type": ["null", "string"] - }, - "purchasing_time_frame": { - "type": ["null", "string"] - }, - "role_in_purchase_process": { - "type": ["null", "string"] - }, - "no_of_employees": { - "type": ["null", "string"] - }, - "comments": { - "type": ["null", "string"] - }, - "custom_questions": { - "items": { - "properties": { - "title": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - }, - "type": ["object"], - "additionalProperties": false - }, - "type": ["null", "array"] - }, - "status": { - "type": ["null", "string"] - }, - "create_time": { - "format": "date-time", - "type": ["null", "string"] - }, - "join_url": { - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "webinar_tracking_sources", - "json_schema": { - "properties": { - "id": { - "type": ["string"] - }, - "webinar_id": { - "type": ["string"] - }, - "source_name": { - "type": ["null", "string"] - }, - "tracking_url": { - "type": ["null", "string"] - }, - "registration_count": { - "type": ["null", "integer"] - }, - "visitor_count": { - "type": ["null", "integer"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "report_meetings", - "json_schema": { - "properties": { - "meeting_id": { - "type": ["string"] - }, - "uuid": { - "type": ["string"] - }, - "id": { - "type": ["integer"] - }, - "type": { - "type": ["null", "integer"] - }, - "topic": { - "type": ["null", "string"] - }, - "user_name": { - "type": ["null", "string"] - }, - "user_email": { - "type": ["null", "string"] - }, - "start_time": { - "format": "date-time", - "type": ["null", "string"] - }, - "end_time": { - "format": "date-time", - "type": ["null", "string"] - }, - "duration": { - "type": ["null", "integer"] - }, - "total_minutes": { - "type": ["null", "integer"] - }, - "participants_count": { - "type": ["null", "integer"] - }, - "tracking_fields": { - "items": { - "properties": { - "field": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - }, - "type": ["object"], - "additionalProperties": false - }, - "type": ["null", "array"] - }, - "dept": { - "type": ["null", "integer"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "report_webinars", - "json_schema": { - "properties": { - "webinar_id": { - "type": ["string"] - }, - "uuid": { - "type": ["string"] - }, - "id": { - "type": ["integer"] - }, - "type": { - "type": ["null", "integer"] - }, - "topic": { - "type": ["null", "string"] - }, - "user_name": { - "type": ["null", "string"] - }, - "user_email": { - "type": ["null", "string"] - }, - "start_time": { - "format": "date-time", - "type": ["null", "string"] - }, - "end_time": { - "format": "date-time", - "type": ["null", "string"] - }, - "duration": { - "type": ["null", "integer"] - }, - "total_minutes": { - "type": ["null", "integer"] - }, - "participants_count": { - "type": ["null", "integer"] - }, - "tracking_fields": { - "items": { - "properties": { - "field": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - }, - "type": ["object"], - "additionalProperties": false - }, - "type": ["null", "array"] - }, - "dept": { - "type": ["null", "integer"] - } - }, - "type": "object", - "additionalProperties": false - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-zoom-singer/sample_files/sample_config.json b/airbyte-integrations/connectors/source-zoom-singer/sample_files/sample_config.json deleted file mode 100644 index 2975582744b1..000000000000 --- a/airbyte-integrations/connectors/source-zoom-singer/sample_files/sample_config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "jwt": "" -} diff --git a/airbyte-integrations/connectors/source-zoom-singer/setup.py b/airbyte-integrations/connectors/source-zoom-singer/setup.py deleted file mode 100644 index 6387b921e852..000000000000 --- a/airbyte-integrations/connectors/source-zoom-singer/setup.py +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -setup( - name="source_zoom_singer", - description="Source implementation for Zoom, built on the Singer tap implementation.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=["airbyte-cdk", "tap-zoom==1.0.0", "pytest==6.1.2"], - package_data={"": ["*.json"]}, -) diff --git a/airbyte-integrations/connectors/source-zoom-singer/source_zoom_singer/__init__.py b/airbyte-integrations/connectors/source-zoom-singer/source_zoom_singer/__init__.py deleted file mode 100644 index dee834d0c068..000000000000 --- a/airbyte-integrations/connectors/source-zoom-singer/source_zoom_singer/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .source import SourceZoomSinger - -__all__ = ["SourceZoomSinger"] diff --git a/airbyte-integrations/connectors/source-zoom-singer/source_zoom_singer/source.py b/airbyte-integrations/connectors/source-zoom-singer/source_zoom_singer/source.py deleted file mode 100644 index 3dac31a63e8f..000000000000 --- a/airbyte-integrations/connectors/source-zoom-singer/source_zoom_singer/source.py +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - - -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.sources.singer.source import BaseSingerSource -from requests import HTTPError -from tap_zoom.client import ZoomClient - - -class SourceZoomSinger(BaseSingerSource): - """ - Zoom API Reference: https://marketplace.zoom.us/docs/api-reference/zoom-api - """ - - tap_cmd = "tap-zoom" - tap_name = "Zoom API" - api_error = HTTPError - force_full_refresh = True - - def try_connect(self, logger: AirbyteLogger, config: dict): - client = ZoomClient(config=config, config_path="") - client.get(path="users") diff --git a/airbyte-integrations/connectors/source-zoom-singer/source_zoom_singer/spec.json b/airbyte-integrations/connectors/source-zoom-singer/source_zoom_singer/spec.json deleted file mode 100644 index ccd2b3111b35..000000000000 --- a/airbyte-integrations/connectors/source-zoom-singer/source_zoom_singer/spec.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/zoom", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Source Zoom Singer Spec", - "type": "object", - "required": ["jwt"], - "additionalProperties": false, - "properties": { - "jwt": { - "title": "JWT Token", - "type": "string", - "description": "Zoom JWT Token. See the docs for more information on how to obtain this key.", - "airbyte_secret": true - } - } - } -} diff --git a/airbyte-integrations/connectors/source-zoom/.dockerignore b/airbyte-integrations/connectors/source-zoom/.dockerignore new file mode 100644 index 000000000000..0804d05bd68e --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_zoom +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-zoom/Dockerfile b/airbyte-integrations/connectors/source-zoom/Dockerfile new file mode 100644 index 000000000000..9ad7060d0685 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/Dockerfile @@ -0,0 +1,40 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_zoom ./source_zoom + + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" + +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-zoom diff --git a/airbyte-integrations/connectors/source-zoom/__init__.py b/airbyte-integrations/connectors/source-zoom/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-zoom/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zoom/acceptance-test-config.yml new file mode 100644 index 000000000000..23948f4f26d1 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/acceptance-test-config.yml @@ -0,0 +1,44 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-zoom:dev +tests: + spec: + - spec_path: "source_zoom/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 3600 + empty_streams: + - "meeting_registrants" + - "meeting_polls" + - "meeting_poll_results" + - "meeting_registration_questions" + - "webinars" + - "webinar_panelists" + - "webinar_registrants" + - "webinar_absentees" + - "webinar_polls" + - "webinar_poll_results" + - "webinar_registration_questions" + - "webinar_tracking_sources" + - "webinar_qna_results" + - "report_meetings" + - "report_meeting_participants" + - "report_webinars" + - "report_webinar_participants" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + ignored_fields: + "meetings": + - "start_url" + "webinars": + - "start_url" + timeout_seconds: 3600 diff --git a/airbyte-integrations/connectors/source-zoom/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-zoom/acceptance-test-docker.sh new file mode 100755 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-zoom/build.gradle b/airbyte-integrations/connectors/source-zoom/build.gradle new file mode 100644 index 000000000000..fa9adb45e4e0 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_zoom' +} diff --git a/airbyte-integrations/connectors/source-zoom/integration_tests/__init__.py b/airbyte-integrations/connectors/source-zoom/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-zoom/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-zoom/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..848a6177c4b7 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "users": { + "updated_at": "2222-01-21T00:00:00.000Z" + } +} diff --git a/airbyte-integrations/connectors/source-zoom/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zoom/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-zoom/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-zoom/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..4be548978f15 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/integration_tests/configured_catalog.json @@ -0,0 +1,213 @@ +{ + "streams": [ + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "meetings", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "meeting_registrants", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "meeting_polls", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "meeting_poll_results", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "meeting_registration_questions", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "webinars", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "webinar_panelists", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "webinar_registrants", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "webinar_absentees", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "webinar_polls", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "webinar_poll_results", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "webinar_registration_questions", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "webinar_tracking_sources", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "webinar_qna_results", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "report_meetings", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "report_meeting_participants", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "report_webinars", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "report_webinar_participants", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-zoom/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-zoom/integration_tests/invalid_config.json new file mode 100644 index 000000000000..6a603fda8000 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "jwt_token": "dummy" +} diff --git a/airbyte-integrations/connectors/source-zoom/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-zoom/integration_tests/sample_config.json new file mode 100644 index 000000000000..f875ad8416c6 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "jwt_token": "abcd" +} diff --git a/airbyte-integrations/connectors/source-zoom/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-zoom/integration_tests/sample_state.json new file mode 100644 index 000000000000..0151c6fc660e --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/integration_tests/sample_state.json @@ -0,0 +1,3 @@ +{ + +} diff --git a/airbyte-integrations/connectors/source-zoom/main.py b/airbyte-integrations/connectors/source-zoom/main.py new file mode 100644 index 000000000000..4b6bfd836670 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zoom import SourceZoom + +if __name__ == "__main__": + source = SourceZoom() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zoom/requirements.txt b/airbyte-integrations/connectors/source-zoom/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-zoom/setup.py b/airbyte-integrations/connectors/source-zoom/setup.py new file mode 100644 index 000000000000..e646cdcd338b --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_zoom", + description="Source implementation for Zoom.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/__init__.py b/airbyte-integrations/connectors/source-zoom/source_zoom/__init__.py new file mode 100644 index 000000000000..4fb74e1fc140 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceZoom + +__all__ = ["SourceZoom"] diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/meeting_poll_results.json b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/meeting_poll_results.json new file mode 100644 index 000000000000..0f4b97875223 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/meeting_poll_results.json @@ -0,0 +1,37 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "meeting_uuid": { + "type": "string" + }, + "email": { + "type": "string" + }, + "name": { + "type": "string" + }, + "question_details": { + "type": "array", + "items": { + "type": "object", + "properties": { + "answer": { + "type": "string" + }, + "date_time": { + "type": "string" + }, + "polling_id": { + "type": "string" + }, + "question": { + "type": "string" + } + }, + "required": [] + } + } + }, + "required": [] +} diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/meeting_polls.json b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/meeting_polls.json new file mode 100644 index 000000000000..989fddd5626f --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/meeting_polls.json @@ -0,0 +1,96 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "meeting_id": { + "type": "number" + }, + "status": { + "type": "string" + }, + "anonymous": { + "type": "boolean" + }, + "poll_type": { + "type": "number" + }, + "questions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "answer_max_character": { + "type": "number" + }, + "answer_min_character": { + "type": "number" + }, + "answer_required": { + "type": "boolean" + }, + "answers": { + "type": "array", + "items": { + "type": "string" + } + }, + "case_sensitive": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "prompts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "prompt_question": { + "type": "string" + }, + "prompt_right_answers": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [] + } + }, + "rating_max_label": { + "type": "string" + }, + "rating_max_value": { + "type": "number" + }, + "rating_min_label": { + "type": "string" + }, + "rating_min_value": { + "type": "number" + }, + "right_answers": { + "type": "array", + "items": { + "type": "string" + } + }, + "show_as_dropdown": { + "type": "boolean" + }, + "type": { + "type": "string" + } + }, + "required": [] + } + }, + "title": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/meeting_registrants.json b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/meeting_registrants.json new file mode 100644 index 000000000000..c5dc946fb693 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/meeting_registrants.json @@ -0,0 +1,86 @@ +{ + "$schema":"http://json-schema.org/draft-07/schema#", + "properties": { + "meeting_id": { + "type": "number" + }, + "id": { + "type": "string" + }, + "address": { + "type": "string" + }, + "city": { + "type": "string" + }, + "comments": { + "type": "string" + }, + "country": { + "type": "string" + }, + "custom_questions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + ] + } + }, + "email": { + "type": "string" + }, + "first_name": { + "type": "string" + }, + "industry": { + "type": "string" + }, + "job_title": { + "type": "string" + }, + "last_name": { + "type": "string" + }, + "no_of_employees": { + "type": "string" + }, + "org": { + "type": "string" + }, + "phone": { + "type": "string" + }, + "purchasing_time_frame": { + "type": "string" + }, + "role_in_purchase_process": { + "type": "string" + }, + "state": { + "type": "string" + }, + "status": { + "type": "string" + }, + "zip": { + "type": "string" + }, + "create_time": { + "type": "string" + }, + "join_url": { + "type": "string" + } + }, + "required": [ + ] +} diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/meeting_registration_questions.json b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/meeting_registration_questions.json new file mode 100644 index 000000000000..aa396bf02ef2 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/meeting_registration_questions.json @@ -0,0 +1,50 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "meeting_id": { + "type": [ + "string" + ] + }, + "custom_questions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "answers": { + "type": "array", + "items": { + "type": "string" + } + }, + "required": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": [] + } + }, + "questions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "field_name": { + "type": "string" + }, + "required": { + "type": "boolean" + } + }, + "required": [] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/meetings.json b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/meetings.json new file mode 100644 index 000000000000..06c7d95dc1f1 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/meetings.json @@ -0,0 +1,408 @@ +{ + "$schema": "http://json-schema.org/draft-06/schema#", + "properties": { + "assistant_id": { + "type": "string" + }, + "host_email": { + "type": "string" + }, + "host_id": { + "type": "string" + }, + "id": { + "type": "number" + }, + "uuid": { + "type": "string" + }, + "agenda": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "duration": { + "type": "number" + }, + "encrypted_password": { + "type": "string" + }, + "h323_password": { + "type": "string" + }, + "join_url": { + "type": "string" + }, + "occurrences": { + "type": "array", + "items": { + "type": "object", + "properties": { + "duration": { + "type": "number" + }, + "occurrence_id": { + "type": "string" + }, + "start_time": { + "type": "string" + }, + "status": { + "type": "string" + } + }, + "required": [ + ] + } + }, + "password": { + "type": "string" + }, + "pmi": { + "type": "string" + }, + "pre_schedule": { + "type": "boolean" + }, + "recurrence": { + "type": "object", + "properties": { + "end_date_time": { + "type": "string" + }, + "end_times": { + "type": "number" + }, + "monthly_day": { + "type": "number" + }, + "monthly_week": { + "type": "number" + }, + "monthly_week_day": { + "type": "number" + }, + "repeat_interval": { + "type": "number" + }, + "type": { + "type": "number" + }, + "weekly_days": { + "type": "string" + } + }, + "required": [ + ] + }, + "settings": { + "type": "object", + "properties": { + "allow_multiple_devices": { + "type": "boolean" + }, + "alternative_hosts": { + "type": "string" + }, + "alternative_hosts_email_notification": { + "type": "boolean" + }, + "alternative_host_update_polls": { + "type": "boolean" + }, + "approval_type": { + "type": "number" + }, + "approved_or_denied_countries_or_regions": { + "type": "object", + "properties": { + "approved_list": { + "type": "array", + "items": { + "type": "string" + } + }, + "denied_list": { + "type": "array", + "items": { + "type": "string" + } + }, + "enable": { + "type": "boolean" + }, + "method": { + "type": "string" + } + }, + "required": [ + ] + }, + "audio": { + "type": "string" + }, + "authentication_domains": { + "type": "string" + }, + "authentication_exception": { + "type": "array", + "items": { + "type": "object", + "properties": { + "email": { + "type": "string" + }, + "name": { + "type": "string" + }, + "join_url": { + "type": "string" + } + }, + "required": [ + ] + } + }, + "authentication_name": { + "type": "string" + }, + "authentication_option": { + "type": "string" + }, + "auto_recording": { + "type": "string" + }, + "breakout_room": { + "type": "object", + "properties": { + "enable": { + "type": "boolean" + }, + "rooms": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "participants": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + ] + } + } + }, + "required": [ + ] + }, + "calendar_type": { + "type": "number" + }, + "close_registration": { + "type": "boolean" + }, + "contact_email": { + "type": "string" + }, + "contact_name": { + "type": "string" + }, + "custom_keys": { + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + ] + } + }, + "email_notification": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "focus_mode": { + "type": "boolean" + }, + "global_dial_in_countries": { + "type": "array", + "items": { + "type": "string" + } + }, + "global_dial_in_numbers": { + "type": "array", + "items": { + "type": "object", + "properties": { + "city": { + "type": "string" + }, + "country": { + "type": "string" + }, + "country_name": { + "type": "string" + }, + "number": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": [ + ] + } + }, + "host_video": { + "type": "boolean" + }, + "jbh_time": { + "type": "number" + }, + "join_before_host": { + "type": "boolean" + }, + "language_interpretation": { + "type": "object", + "properties": { + "enable": { + "type": "boolean" + }, + "interpreters": { + "type": "array", + "items": { + "type": "object", + "properties": { + "email": { + "type": "string" + }, + "languages": { + "type": "string" + } + }, + "required": [ + ] + } + } + }, + "required": [ + ] + }, + "meeting_authentication": { + "type": "boolean" + }, + "mute_upon_entry": { + "type": "boolean" + }, + "participant_video": { + "type": "boolean" + }, + "private_meeting": { + "type": "boolean" + }, + "registrants_confirmation_email": { + "type": "boolean" + }, + "registrants_email_notification": { + "type": "boolean" + }, + "registration_type": { + "type": "number" + }, + "show_share_button": { + "type": "boolean" + }, + "use_pmi": { + "type": "boolean" + }, + "waiting_room": { + "type": "boolean" + }, + "waiting_room_options": { + "type": "object", + "properties": { + "enable": { + "type": "boolean" + }, + "admit_type": { + "type": "number" + }, + "auto_admit": { + "type": "number" + }, + "internal_user_auto_admit": { + "type": "number" + } + }, + "required": [ + ] + }, + "watermark": { + "type": "boolean" + }, + "host_save_video_order": { + "type": "boolean" + } + }, + "required": [ + ] + }, + "start_time": { + "type": "string" + }, + "start_url": { + "type": "string" + }, + "status": { + "type": "string" + }, + "timezone": { + "type": "string" + }, + "topic": { + "type": "string" + }, + "tracking_fields": { + "type": "array", + "items": { + "type": "object", + "properties": { + "field": { + "type": "string" + }, + "value": { + "type": "string" + }, + "visible": { + "type": "boolean" + } + }, + "required": [ + ] + } + }, + "type": { + "type": "number" + } + }, + "required": [ + ] + +} diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/report_meeting_participants.json b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/report_meeting_participants.json new file mode 100644 index 000000000000..763392427fc4 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/report_meeting_participants.json @@ -0,0 +1,46 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "meeting_uuid": { + "type": "string" + }, + "customer_key": { + "type": "string" + }, + "duration": { + "type": "number" + }, + "failover": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "join_time": { + "type": "string" + }, + "leave_time": { + "type": "string" + }, + "name": { + "type": "string" + }, + "registrant_id": { + "type": "string" + }, + "user_email": { + "type": "string" + }, + "user_id": { + "type": "string" + }, + "status": { + "type": "string" + }, + "bo_mtg_id": { + "type": "string" + } + }, + "required": [] +} diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/report_meetings.json b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/report_meetings.json new file mode 100644 index 000000000000..e7b31f338a72 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/report_meetings.json @@ -0,0 +1,76 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "meeting_uuid": { + "type": "string" + }, + "custom_keys": { + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [] + } + }, + "dept": { + "type": "string" + }, + "duration": { + "type": "number" + }, + "end_time": { + "type": "string" + }, + "id": { + "type": "number" + }, + "participants_count": { + "type": "number" + }, + "start_time": { + "type": "string" + }, + "topic": { + "type": "string" + }, + "total_minutes": { + "type": "number" + }, + "tracking_fields": { + "type": "array", + "items": { + "type": "object", + "properties": { + "field": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [] + } + }, + "type": { + "type": "number" + }, + "user_email": { + "type": "string" + }, + "user_name": { + "type": "string" + }, + "uuid": { + "type": "string" + } + }, + "required": [] +} diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/report_webinar_participants.json b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/report_webinar_participants.json new file mode 100644 index 000000000000..bfba6ff87d93 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/report_webinar_participants.json @@ -0,0 +1,55 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "webinar_uuid": { + "type": "string" + }, + "customer_key": { + "type": "string" + }, + "duration": { + "type": "number" + }, + "failover": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "join_time": { + "type": "string" + }, + "leave_time": { + "type": "string" + }, + "name": { + "type": "string" + }, + "registrant_id": { + "type": "string" + }, + "status": { + "type": "string" + }, + "user_email": { + "type": "string" + }, + "user_id": { + "type": "string" + } + }, + "required": [ + "customer_key", + "duration", + "failover", + "id", + "join_time", + "leave_time", + "name", + "registrant_id", + "status", + "user_email", + "user_id" + ] +} diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/report_webinars.json b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/report_webinars.json new file mode 100644 index 000000000000..b8ae1ed0ceac --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/report_webinars.json @@ -0,0 +1,76 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "webinar_uuid": { + "type": "string" + }, + "custom_keys": { + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [] + } + }, + "dept": { + "type": "string" + }, + "duration": { + "type": "number" + }, + "end_time": { + "type": "string" + }, + "id": { + "type": "number" + }, + "participants_count": { + "type": "number" + }, + "start_time": { + "type": "string" + }, + "topic": { + "type": "string" + }, + "total_minutes": { + "type": "number" + }, + "tracking_fields": { + "type": "array", + "items": { + "type": "object", + "properties": { + "field": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [] + } + }, + "type": { + "type": "number" + }, + "user_email": { + "type": "string" + }, + "user_name": { + "type": "string" + }, + "uuid": { + "type": "string" + } + }, + "required": [] +} diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/users.json b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/users.json new file mode 100644 index 000000000000..8e2bdef40615 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/users.json @@ -0,0 +1,86 @@ +{ + "$schema": "http://json-schema.org/draft-06/schema#", + "properties": { + "created_at": { + "type": "string" + }, + "custom_attributes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + ] + } + }, + "dept": { + "type": "string" + }, + "email": { + "type": "string" + }, + "employee_unique_id": { + "type": "string" + }, + "first_name": { + "type": "string" + }, + "group_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "id": { + "type": "string" + }, + "im_group_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "last_client_version": { + "type": "string" + }, + "last_login_time": { + "type": "string" + }, + "last_name": { + "type": "string" + }, + "plan_united_type": { + "type": "string" + }, + "pmi": { + "type": "number" + }, + "role_id": { + "type": "string" + }, + "status": { + "type": "string" + }, + "timezone": { + "type": "string" + }, + "type": { + "type": "number" + }, + "verified": { + "type": "number" + } + }, + "required": [ + ] +} diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_absentees.json b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_absentees.json new file mode 100644 index 000000000000..ced32756af1a --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_absentees.json @@ -0,0 +1,85 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "webinar_uuid": { + "type": "string" + }, + "id": { + "type": "string" + }, + "address": { + "type": "string" + }, + "city": { + "type": "string" + }, + "comments": { + "type": "string" + }, + "country": { + "type": "string" + }, + "custom_questions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [] + } + }, + "email": { + "type": "string" + }, + "first_name": { + "type": "string" + }, + "industry": { + "type": "string" + }, + "job_title": { + "type": "string" + }, + "last_name": { + "type": "string" + }, + "no_of_employees": { + "type": "string" + }, + "org": { + "type": "string" + }, + "phone": { + "type": "string" + }, + "purchasing_time_frame": { + "type": "string" + }, + "role_in_purchase_process": { + "type": "string" + }, + "state": { + "type": "string" + }, + "status": { + "type": "string" + }, + "zip": { + "type": "string" + }, + "create_time": { + "type": "string" + }, + "join_url": { + "type": "string" + } + }, + "required": [] +} diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_panelists.json b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_panelists.json new file mode 100644 index 000000000000..53801958fa0e --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_panelists.json @@ -0,0 +1,37 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "webinar_id": { + "type": "number" + }, + "id": { + "type": "string" + }, + "email": { + "type": "string" + }, + "name": { + "type": "string" + }, + "join_url": { + "type": "string" + }, + "virtual_background_id": { + "type": "string" + }, + "name_tag_id": { + "type": "string" + }, + "name_tag_name": { + "type": "string" + }, + "name_tag_pronouns": { + "type": "string" + }, + "name_tag_description": { + "type": "string" + } + }, + "required": [] +} diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_poll_results.json b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_poll_results.json new file mode 100644 index 000000000000..dbb102491ec1 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_poll_results.json @@ -0,0 +1,37 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "webinar_uuid": { + "type": "string" + }, + "email": { + "type": "string" + }, + "name": { + "type": "string" + }, + "question_details": { + "type": "array", + "items": { + "type": "object", + "properties": { + "answer": { + "type": "string" + }, + "date_time": { + "type": "string" + }, + "polling_id": { + "type": "string" + }, + "question": { + "type": "string" + } + }, + "required": [] + } + } + }, + "required": [] +} diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_polls.json b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_polls.json new file mode 100644 index 000000000000..35ed3e392162 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_polls.json @@ -0,0 +1,97 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "webinar_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "status": { + "type": "string" + }, + "anonymous": { + "type": "boolean" + }, + "poll_type": { + "type": "number" + }, + "questions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "answer_max_character": { + "type": "number" + }, + "answer_min_character": { + "type": "number" + }, + "answer_required": { + "type": "boolean" + }, + "answers": { + "type": "array", + "items": { + "type": "string" + } + }, + "case_sensitive": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "prompts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "prompt_question": { + "type": "string" + }, + "prompt_right_answers": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [] + } + }, + "rating_max_label": { + "type": "string" + }, + "rating_max_value": { + "type": "number" + }, + "rating_min_label": { + "type": "string" + }, + "rating_min_value": { + "type": "number" + }, + "right_answers": { + "type": "array", + "items": { + "type": "string" + } + }, + "show_as_dropdown": { + "type": "boolean" + }, + "type": { + "type": "string" + } + }, + "required": [] + } + }, + "title": { + "type": "string" + } + }, + "required": [] +} diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_qna_results.json b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_qna_results.json new file mode 100644 index 000000000000..361b24a5fc56 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_qna_results.json @@ -0,0 +1,31 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "webinar_uuid": { + "type": "string" + }, + "email": { + "type": "string" + }, + "name": { + "type": "string" + }, + "question_details": { + "type": "array", + "items": { + "type": "object", + "properties": { + "answer": { + "type": "string" + }, + "question": { + "type": "string" + } + }, + "required": [] + } + } + }, + "required": [] +} diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_registrants.json b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_registrants.json new file mode 100644 index 000000000000..0b5a7cdaf6c2 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_registrants.json @@ -0,0 +1,85 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "webinar_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "address": { + "type": "string" + }, + "city": { + "type": "string" + }, + "comments": { + "type": "string" + }, + "country": { + "type": "string" + }, + "custom_questions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [] + } + }, + "email": { + "type": "string" + }, + "first_name": { + "type": "string" + }, + "industry": { + "type": "string" + }, + "job_title": { + "type": "string" + }, + "last_name": { + "type": "string" + }, + "no_of_employees": { + "type": "string" + }, + "org": { + "type": "string" + }, + "phone": { + "type": "string" + }, + "purchasing_time_frame": { + "type": "string" + }, + "role_in_purchase_process": { + "type": "string" + }, + "state": { + "type": "string" + }, + "status": { + "type": "string" + }, + "zip": { + "type": "string" + }, + "create_time": { + "type": "string" + }, + "join_url": { + "type": "string" + } + }, + "required": [] +} diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_registration_questions.json b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_registration_questions.json new file mode 100644 index 000000000000..46f0dad22ea0 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_registration_questions.json @@ -0,0 +1,49 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "webinar_id": { + "type": "string" + }, + "custom_questions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "answers": { + "type": "array", + "items": { + "type": "string" + } + }, + "required": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": [] + } + }, + "questions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "field_name": { + "type": "string" + }, + "required": { + "type": "boolean" + } + }, + "required": [] + } + } + }, + "required": [] +} diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_tracking_sources.json b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_tracking_sources.json new file mode 100644 index 000000000000..b7cab1839c57 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinar_tracking_sources.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "webinar_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "registration_count": { + "type": "number" + }, + "source_name": { + "type": "string" + }, + "tracking_url": { + "type": "string" + }, + "visitor_count": { + "type": "number" + } + }, + "required": [] +} diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinars.json b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinars.json new file mode 100644 index 000000000000..850b0c16c0c9 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/schemas/webinars.json @@ -0,0 +1,322 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "host_email": { + "type": "string" + }, + "host_id": { + "type": "string" + }, + "id": { + "type": "number" + }, + "uuid": { + "type": "string" + }, + "agenda": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "duration": { + "type": "number" + }, + "join_url": { + "type": "string" + }, + "occurrences": { + "type": "array", + "items": { + "type": "object", + "properties": { + "duration": { + "type": "number" + }, + "occurrence_id": { + "type": "string" + }, + "start_time": { + "type": "string" + }, + "status": { + "type": "string" + } + }, + "required": [] + } + }, + "password": { + "type": "string" + }, + "recurrence": { + "type": "object", + "properties": { + "end_date_time": { + "type": "string" + }, + "end_times": { + "type": "number" + }, + "monthly_day": { + "type": "number" + }, + "monthly_week": { + "type": "number" + }, + "monthly_week_day": { + "type": "number" + }, + "repeat_interval": { + "type": "number" + }, + "type": { + "type": "number" + }, + "weekly_days": { + "type": "string" + } + }, + "required": [] + }, + "settings": { + "type": "object", + "properties": { + "allow_multiple_devices": { + "type": "boolean" + }, + "alternative_hosts": { + "type": "string" + }, + "alternative_host_update_polls": { + "type": "boolean" + }, + "approval_type": { + "type": "number" + }, + "attendees_and_panelists_reminder_email_notification": { + "type": "object", + "properties": { + "enable": { + "type": "boolean" + }, + "type": { + "type": "number" + } + }, + "required": [] + }, + "audio": { + "type": "string" + }, + "authentication_domains": { + "type": "string" + }, + "authentication_name": { + "type": "string" + }, + "authentication_option": { + "type": "string" + }, + "auto_recording": { + "type": "string" + }, + "close_registration": { + "type": "boolean" + }, + "contact_email": { + "type": "string" + }, + "contact_name": { + "type": "string" + }, + "email_language": { + "type": "string" + }, + "follow_up_absentees_email_notification": { + "type": "object", + "properties": { + "enable": { + "type": "boolean" + }, + "type": { + "type": "number" + } + }, + "required": [] + }, + "follow_up_attendees_email_notification": { + "type": "object", + "properties": { + "enable": { + "type": "boolean" + }, + "type": { + "type": "number" + } + }, + "required": [] + }, + "global_dial_in_countries": { + "type": "array", + "items": { + "type": "string" + } + }, + "hd_video": { + "type": "boolean" + }, + "hd_video_for_attendees": { + "type": "boolean" + }, + "host_video": { + "type": "boolean" + }, + "language_interpretation": { + "type": "object", + "properties": { + "enable": { + "type": "boolean" + }, + "interpreters": { + "type": "array", + "items": { + "type": "object", + "properties": { + "email": { + "type": "string" + }, + "languages": { + "type": "string" + } + }, + "required": [] + } + } + }, + "required": [] + }, + "panelist_authentication": { + "type": "boolean" + }, + "meeting_authentication": { + "type": "boolean" + }, + "add_watermark": { + "type": "boolean" + }, + "add_audio_watermark": { + "type": "boolean" + }, + "notify_registrants": { + "type": "boolean" + }, + "on_demand": { + "type": "boolean" + }, + "panelists_invitation_email_notification": { + "type": "boolean" + }, + "panelists_video": { + "type": "boolean" + }, + "post_webinar_survey": { + "type": "boolean" + }, + "practice_session": { + "type": "boolean" + }, + "question_and_answer": { + "type": "object", + "properties": { + "allow_anonymous_questions": { + "type": "boolean" + }, + "answer_questions": { + "type": "string" + }, + "attendees_can_comment": { + "type": "boolean" + }, + "attendees_can_upvote": { + "type": "boolean" + }, + "allow_auto_reply": { + "type": "boolean" + }, + "auto_reply_text": { + "type": "string" + }, + "enable": { + "type": "boolean" + } + }, + "required": [] + }, + "registrants_confirmation_email": { + "type": "boolean" + }, + "registrants_email_notification": { + "type": "boolean" + }, + "registrants_restrict_number": { + "type": "number" + }, + "registration_type": { + "type": "number" + }, + "send_1080p_video_to_attendees": { + "type": "boolean" + }, + "show_share_button": { + "type": "boolean" + }, + "survey_url": { + "type": "string" + }, + "enable_session_branding": { + "type": "boolean" + } + }, + "required": [] + }, + "start_time": { + "type": "string" + }, + "start_url": { + "type": "string" + }, + "timezone": { + "type": "string" + }, + "topic": { + "type": "string" + }, + "tracking_fields": { + "type": "array", + "items": { + "type": "object", + "properties": { + "field": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [] + } + }, + "type": { + "type": "number" + }, + "is_simulive": { + "type": "boolean" + }, + "record_file_id": { + "type": "string" + } + }, + "required": [] +} diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/source.py b/airbyte-integrations/connectors/source-zoom/source_zoom/source.py new file mode 100644 index 000000000000..f7c16e43a355 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceZoom(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "zoom.yaml"}) diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/spec.yaml b/airbyte-integrations/connectors/source-zoom/source_zoom/spec.yaml new file mode 100644 index 000000000000..a8170e08c3b7 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/spec.yaml @@ -0,0 +1,13 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/zoom +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Zoom Spec + type: object + required: + - jwt_token + additionalProperties: true + properties: + jwt_token: + type: string + description: JWT Token + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/zoom.yaml b/airbyte-integrations/connectors/source-zoom/source_zoom/zoom.yaml new file mode 100644 index 000000000000..77fae4e527d3 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/zoom.yaml @@ -0,0 +1,815 @@ +version: "0.1.0" + +definitions: + requester: + url_base: "https://api.zoom.us/v2/" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['jwt_token'] }}" + + zoom_paginator: + type: DefaultPaginator + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response.next_page_token }}" + stop_condition: "{{ response.next_page_token == '' }}" + page_size: 30 + page_size_option: + field_name: "page_size" + inject_into: "request_parameter" + page_token_option: + field_name: "next_page_token" + inject_into: "request_parameter" + url_base: "*ref(definitions.requester.url_base)" + + + retriever: + requester: + $ref: "*ref(definitions.requester)" + + schema_loader: + type: JsonSchema + file_path: "./source_zoom/schemas/{{ options['name'] }}.json" + + + users_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + retriever: + paginator: + $ref: "*ref(definitions.zoom_paginator)" + record_selector: + extractor: + type: DpathExtractor + field_pointer: ["users"] + $ref: "*ref(definitions.retriever)" + $options: + name: "users" + primary_key: "id" + path: "/users" + + + meetings_list_tmp_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "meetings_list_tmp" + primary_key: "id" + retriever: + paginator: + $ref: "*ref(definitions.zoom_paginator)" + record_selector: + extractor: + type: DpathExtractor + field_pointer: ["meetings"] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/users/{{ stream_slice.parent_id }}/meetings" + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.users_stream)" + parent_key: "id" + stream_slice_field: "parent_id" + + + + meetings_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "meetings" + primary_key: "id" + retriever: + paginator: + type: NoPagination + record_selector: + extractor: + type: DpathExtractor + field_pointer: [] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/meetings/{{ stream_slice.parent_id }}" + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.meetings_list_tmp_stream)" + parent_key: "id" + stream_slice_field: "parent_id" + + + meeting_registrants_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "meeting_registrants" + primary_key: "id" + retriever: + paginator: + $ref: "*ref(definitions.zoom_paginator)" + record_selector: + extractor: + type: DpathExtractor + field_pointer: ["registrants"] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/meetings/{{ stream_slice.parent_id }}/registrants" + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + # Meeting {meetingId} is not found or has expired. This meeting has not set registration as required: {meetingId}. + - predicate: "{{ response.code == 300 }}" + action: IGNORE + - type: DefaultErrorHandler # we're adding this DefaultErrorHandler for 429, 5XX errors etc; + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.meetings_list_tmp_stream)" + parent_key: "id" + stream_slice_field: "parent_id" + transformations: + - type: AddFields + fields: + - path: ["meeting_id"] + value: "{{ stream_slice.parent_id }}" + + + meeting_polls_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "meeting_polls" + primary_key: "id" + retriever: + paginator: + type: NoPagination + record_selector: + extractor: + type: DpathExtractor + field_pointer: ["polls"] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/meetings/{{ stream_slice.parent_id }}/polls" + error_handler: + type: CompositeErrorHandler + # ignore 400 error; We get this error if Meeting poll is not enabled for the meeting, or scheduling capabilities aren't in the account + error_handlers: + - type: DefaultErrorHandler + response_filters: + - http_codes: [400] + action: IGNORE + - type: DefaultErrorHandler + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.meetings_list_tmp_stream)" + parent_key: "id" + stream_slice_field: "parent_id" + transformations: + - type: AddFields + fields: + - path: ["meeting_id"] + value: "{{ stream_slice.parent_id }}" + + + meeting_poll_results_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "meeting_poll_results" + retriever: + paginator: + type: NoPagination + record_selector: + extractor: + type: DpathExtractor + field_pointer: ["questions"] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/past_meetings/{{ stream_slice.parent_id }}/polls" + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + # 400 error is thrown for meetings created an year ago + # 404 error is thrown if the meeting has not enabled polls (from observation, not written in docs) + - http_codes: [400,404] + action: IGNORE + - type: DefaultErrorHandler + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.meetings_list_tmp_stream)" + parent_key: "uuid" + stream_slice_field: "parent_id" + transformations: + - type: AddFields + fields: + - path: ["meeting_uuid"] + value: "{{ stream_slice.parent_id }}" + + + meeting_registration_questions_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "meeting_registration_questions" + retriever: + paginator: + type: NoPagination + record_selector: + extractor: + type: DpathExtractor + field_pointer: [] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/meetings/{{ stream_slice.parent_id }}/registrants/questions" + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + # ignore 400 error; We get this error if Bad Request or Meeting hosting and scheduling capabilities are not allowed for your user account. + - http_codes: [400] + action: IGNORE + - type: DefaultErrorHandler + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.meetings_list_tmp_stream)" + parent_key: "id" + stream_slice_field: "parent_id" + transformations: + - type: AddFields + fields: + - path: ["meeting_id"] + value: "{{ stream_slice.parent_id }}" + + + webinars_list_tmp_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "webinars_list_tmp" + primary_key: "id" + retriever: + paginator: + $ref: "*ref(definitions.zoom_paginator)" + record_selector: + extractor: + type: DpathExtractor + field_pointer: ["webinars"] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/users/{{ stream_slice.parent_id }}/webinars" + error_handler: + type: CompositeErrorHandler + # ignore 400 error; We get this error if Meeting is more than created an year ago + error_handlers: + - type: DefaultErrorHandler + response_filters: + - http_codes: [400] + action: IGNORE + - type: DefaultErrorHandler + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.users_stream)" + parent_key: "id" + stream_slice_field: "parent_id" + + webinars_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "webinars" + primary_key: "id" + retriever: + paginator: + type: NoPagination + record_selector: + extractor: + type: DpathExtractor + field_pointer: [] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/webinars/{{ stream_slice.parent_id }}" + error_handler: + type: CompositeErrorHandler + # ignore 400 error + error_handlers: + - type: DefaultErrorHandler + response_filters: + # When parent stream throws error; then ideally we should have an empty array, and no /webinars/{id} should be called. But somehow we're calling it right now with None. :( + # More context: https://github.com/airbytehq/airbyte/issues/18046 + - http_codes: [400,404] + action: IGNORE + - type: DefaultErrorHandler + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.webinars_list_tmp_stream)" + parent_key: "id" + stream_slice_field: "parent_id" + + webinar_panelists_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "webinar_panelists" + retriever: + paginator: + type: NoPagination + record_selector: + extractor: + type: DpathExtractor + field_pointer: ["panelists"] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/webinars/{{ stream_slice.parent_id }}/panelists" + error_handler: + type: CompositeErrorHandler + # ignore 400 error + error_handlers: + - type: DefaultErrorHandler + response_filters: + # Same problem as "webinars_stream" for 404! and we get 400 error if the account isn't PRO. + - http_codes: [400,404] + action: IGNORE + - type: DefaultErrorHandler + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.webinars_list_tmp_stream)" + parent_key: "id" + stream_slice_field: "parent_id" + transformations: + - type: AddFields + fields: + - path: ["webinar_id"] + value: "{{ stream_slice.parent_id }}" + + + webinar_registrants_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "webinar_registrants" + retriever: + paginator: + $ref: "*ref(definitions.zoom_paginator)" + record_selector: + extractor: + type: DpathExtractor + field_pointer: ["registrants"] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/webinars/{{ stream_slice.parent_id }}/registrants" + error_handler: + type: CompositeErrorHandler + # ignore 400 error + error_handlers: + - type: DefaultErrorHandler + response_filters: + # Same problem as "webinars_stream" for 404! 400 is for non PRO accounts. + - http_codes: [400,404] + action: IGNORE + - type: DefaultErrorHandler + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.webinars_list_tmp_stream)" + parent_key: "id" + stream_slice_field: "parent_id" + transformations: + - type: AddFields + fields: + - path: ["webinar_id"] + value: "{{ stream_slice.parent_id }}" + + + webinar_absentees_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "webinar_absentees" + primary_key: "id" + retriever: + paginator: + $ref: "*ref(definitions.zoom_paginator)" + record_selector: + extractor: + type: DpathExtractor + field_pointer: ["registrants"] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/past_webinars/{{ stream_slice.parent_uuid }}/absentees" + error_handler: + type: CompositeErrorHandler + # ignore 400 error + error_handlers: + - type: DefaultErrorHandler + response_filters: + # Same problem as "webinars_stream" for 404! 400 is for non PRO accounts. + - http_codes: [400,404] + action: IGNORE + - type: DefaultErrorHandler + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.webinars_list_tmp_stream)" + parent_key: "uuid" + stream_slice_field: "parent_uuid" + transformations: + - type: AddFields + fields: + - path: ["webinar_uuid"] + value: "{{ stream_slice.parent_uuid }}" + + + webinar_polls_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "webinar_polls" + retriever: + paginator: + type: NoPagination + record_selector: + extractor: + type: DpathExtractor + field_pointer: ["polls"] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/webinars/{{ stream_slice.parent_id }}/polls" + error_handler: + type: CompositeErrorHandler + # ignore 400 error; We get this error if Webinar poll is disabled + error_handlers: + - type: DefaultErrorHandler + response_filters: + # Same problem as "webinars_stream" for 404! 400 is for non PRO accounts. + - http_codes: [400,404] + action: IGNORE + - type: DefaultErrorHandler + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.webinars_list_tmp_stream)" + parent_key: "id" + stream_slice_field: "parent_id" + transformations: + - type: AddFields + fields: + - path: ["webinar_id"] + value: "{{ stream_slice.parent_id }}" + + + + webinar_poll_results_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "webinar_poll_results" + retriever: + paginator: + type: NoPagination + record_selector: + extractor: + type: DpathExtractor + field_pointer: ["questions"] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/past_webinars/{{ stream_slice.parent_id }}/polls" + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - http_codes: [404] + action: IGNORE + - type: DefaultErrorHandler + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.webinars_list_tmp_stream)" + parent_key: "uuid" + stream_slice_field: "parent_id" + transformations: + - type: AddFields + fields: + - path: ["webinar_uuid"] + value: "{{ stream_slice.parent_id }}" + + + webinar_registration_questions_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "webinar_registration_questions" + retriever: + paginator: + type: NoPagination + record_selector: + extractor: + type: DpathExtractor + field_pointer: [] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/webinars/{{ stream_slice.parent_id }}/registrants/questions" + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + # the docs says 404 code, but that's incorrect (from observation); + - http_codes: [400] + action: IGNORE + - type: DefaultErrorHandler + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.webinars_list_tmp_stream)" + parent_key: "id" + stream_slice_field: "parent_id" + transformations: + - type: AddFields + fields: + - path: ["webinar_id"] + value: "{{ stream_slice.parent_id }}" + + + + webinar_tracking_sources_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "webinar_tracking_sources" + primary_key: "id" + retriever: + paginator: + type: NoPagination + record_selector: + extractor: + type: DpathExtractor + field_pointer: ["tracking_sources"] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/webinars/{{ stream_slice.parent_id }}/tracking_sources" + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - http_codes: [400] + action: IGNORE + - type: DefaultErrorHandler + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.webinars_list_tmp_stream)" + parent_key: "id" + stream_slice_field: "parent_id" + transformations: + - type: AddFields + fields: + - path: ["webinar_id"] + value: "{{ stream_slice.parent_id }}" + + + webinar_qna_results_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "webinar_qna_results" + retriever: + paginator: + type: NoPagination + record_selector: + extractor: + type: DpathExtractor + field_pointer: ["questions"] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/past_webinars/{{ stream_slice.parent_id }}/qa" + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - http_codes: [400,404] + action: IGNORE + - type: DefaultErrorHandler + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.webinars_list_tmp_stream)" + parent_key: "uuid" + stream_slice_field: "parent_id" + transformations: + - type: AddFields + fields: + - path: ["webinar_uuid"] + value: "{{ stream_slice.parent_id }}" + + + + report_meetings_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "report_meetings" + primary_key: "id" + retriever: + paginator: + type: NoPagination + record_selector: + extractor: + type: DpathExtractor + field_pointer: ["tracking_sources"] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/report/meetings/{{ stream_slice.parent_id }}" + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - http_codes: [400] + action: IGNORE + - type: DefaultErrorHandler + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.meetings_list_tmp_stream)" + parent_key: "uuid" + stream_slice_field: "parent_id" + transformations: + - type: AddFields + fields: + - path: ["meeting_uuid"] + value: "{{ stream_slice.parent_id }}" + + + + + + report_meeting_participants_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "report_meeting_participants" + primary_key: "id" + retriever: + paginator: + $ref: "*ref(definitions.zoom_paginator)" + record_selector: + extractor: + type: DpathExtractor + field_pointer: ["participants"] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/report/meetings/{{ stream_slice.parent_id }}/participants" + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - http_codes: [400] + action: IGNORE + - type: DefaultErrorHandler + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.meetings_list_tmp_stream)" + parent_key: "uuid" + stream_slice_field: "parent_id" + transformations: + - type: AddFields + fields: + - path: ["meeting_uuid"] + value: "{{ stream_slice.parent_id }}" + + + report_webinars_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "report_webinars" + retriever: + paginator: + type: NoPagination + record_selector: + extractor: + type: DpathExtractor + field_pointer: [] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/report/webinars/{{ stream_slice.parent_id }}" + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - http_codes: [400] + action: IGNORE + - type: DefaultErrorHandler + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.webinars_list_tmp_stream)" + parent_key: "uuid" + stream_slice_field: "parent_id" + transformations: + - type: AddFields + fields: + - path: ["webinar_uuid"] + value: "{{ stream_slice.parent_id }}" + + + + report_webinar_participants_stream: + schema_loader: + $ref: "*ref(definitions.schema_loader)" + $options: + name: "report_webinar_participants" + retriever: + paginator: + $ref: "*ref(definitions.zoom_paginator)" + record_selector: + extractor: + type: DpathExtractor + field_pointer: ["participants"] + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "/report/webinars/{{ stream_slice.parent_id }}/participants" + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - http_codes: [400] + action: IGNORE + - type: DefaultErrorHandler + stream_slicer: + type: SubstreamSlicer + parent_stream_configs: + - stream: "*ref(definitions.webinars_list_tmp_stream)" + parent_key: "uuid" + stream_slice_field: "parent_id" + transformations: + - type: AddFields + fields: + - path: ["webinar_uuid"] + value: "{{ stream_slice.parent_id }}" + + +streams: + - "*ref(definitions.users_stream)" + - "*ref(definitions.meetings_stream)" + - "*ref(definitions.meeting_registrants_stream)" + - "*ref(definitions.meeting_polls_stream)" + - "*ref(definitions.meeting_poll_results_stream)" + - "*ref(definitions.meeting_registration_questions_stream)" + - "*ref(definitions.webinars_stream)" + - "*ref(definitions.webinar_panelists_stream)" + - "*ref(definitions.webinar_registrants_stream)" + - "*ref(definitions.webinar_absentees_stream)" + - "*ref(definitions.webinar_polls_stream)" + - "*ref(definitions.webinar_poll_results_stream)" + - "*ref(definitions.webinar_registration_questions_stream)" + - "*ref(definitions.webinar_tracking_sources_stream)" + - "*ref(definitions.webinar_qna_results_stream)" + - "*ref(definitions.report_meetings_stream)" + - "*ref(definitions.report_meeting_participants_stream)" + - "*ref(definitions.report_webinars_stream)" + - "*ref(definitions.report_webinar_participants_stream)" + + +check: + stream_names: + - "users" diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java index c636a9d935d6..4fa0cf52e6f6 100644 --- a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java +++ b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java @@ -68,6 +68,11 @@ public static final class Tags { */ public static final String JOB_ROOT_KEY = "job_root"; + /** + * Name of the APM trace tag that holds the process exit value associated with the trace. + */ + public static final String PROCESS_EXIT_VALUE_KEY = "process.exit_value"; + /** * Name of the APM trace tag that holds the source Docker image value associated with the trace. */ diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java index 7a85300ab0c1..82d108dcce52 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java @@ -5,6 +5,7 @@ package io.airbyte.persistence.job; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.version.Version; import io.airbyte.config.ActorDefinitionResourceRequirements; import io.airbyte.config.DestinationConnection; import io.airbyte.config.JobConfig; @@ -52,7 +53,9 @@ public Optional createSyncJob(final SourceConnection source, final DestinationConnection destination, final StandardSync standardSync, final String sourceDockerImageName, + final Version sourceProtocolVersion, final String destinationDockerImageName, + final Version destinationProtocolVersion, final List standardSyncOperations, @Nullable final JsonNode webhookOperationConfigs, @Nullable final ActorDefinitionResourceRequirements sourceResourceReqs, @@ -79,8 +82,10 @@ public Optional createSyncJob(final SourceConnection source, .withNamespaceFormat(standardSync.getNamespaceFormat()) .withPrefix(standardSync.getPrefix()) .withSourceDockerImage(sourceDockerImageName) + .withSourceProtocolVersion(sourceProtocolVersion) .withSourceConfiguration(source.getConfiguration()) .withDestinationDockerImage(destinationDockerImageName) + .withDestinationProtocolVersion(destinationProtocolVersion) .withDestinationConfiguration(destination.getConfiguration()) .withOperationSequence(standardSyncOperations) .withWebhookOperationConfigs(webhookOperationConfigs) @@ -102,23 +107,26 @@ public Optional createSyncJob(final SourceConnection source, public Optional createResetConnectionJob(final DestinationConnection destination, final StandardSync standardSync, final String destinationDockerImage, + final Version destinationProtocolVersion, final List standardSyncOperations, final List streamsToReset) throws IOException { final ConfiguredAirbyteCatalog configuredAirbyteCatalog = standardSync.getCatalog(); configuredAirbyteCatalog.getStreams().forEach(configuredAirbyteStream -> { final StreamDescriptor streamDescriptor = CatalogHelpers.extractDescriptor(configuredAirbyteStream); - configuredAirbyteStream.setSyncMode(SyncMode.FULL_REFRESH); if (streamsToReset.contains(streamDescriptor)) { // The Reset Source will emit no record messages for any streams, so setting the destination sync // mode to OVERWRITE will empty out this stream in the destination. // Note: streams in streamsToReset that are NOT in this configured catalog (i.e. deleted streams) // will still have their state reset by the Reset Source, but will not be modified in the // destination since they are not present in the catalog that is sent to the destination. + configuredAirbyteStream.setSyncMode(SyncMode.FULL_REFRESH); configuredAirbyteStream.setDestinationSyncMode(DestinationSyncMode.OVERWRITE); } else { // Set streams that are not being reset to APPEND so that they are not modified in the destination - configuredAirbyteStream.setDestinationSyncMode(DestinationSyncMode.APPEND); + if (configuredAirbyteStream.getDestinationSyncMode() == DestinationSyncMode.OVERWRITE) { + configuredAirbyteStream.setDestinationSyncMode(DestinationSyncMode.APPEND); + } } }); final JobResetConnectionConfig resetConnectionConfig = new JobResetConnectionConfig() @@ -126,6 +134,7 @@ public Optional createResetConnectionJob(final DestinationConnection desti .withNamespaceFormat(standardSync.getNamespaceFormat()) .withPrefix(standardSync.getPrefix()) .withDestinationDockerImage(destinationDockerImage) + .withDestinationProtocolVersion(destinationProtocolVersion) .withDestinationConfiguration(destination.getConfiguration()) .withOperationSequence(standardSyncOperations) .withConfiguredAirbyteCatalog(configuredAirbyteCatalog) diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/JobCreator.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/JobCreator.java index 16011e80ff9e..687d795dc3f4 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/JobCreator.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/JobCreator.java @@ -5,6 +5,7 @@ package io.airbyte.persistence.job; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.version.Version; import io.airbyte.config.ActorDefinitionResourceRequirements; import io.airbyte.config.DestinationConnection; import io.airbyte.config.SourceConnection; @@ -32,7 +33,9 @@ Optional createSyncJob(SourceConnection source, DestinationConnection destination, StandardSync standardSync, String sourceDockerImage, + Version sourceProtocolVersion, String destinationDockerImage, + Version destinationProtocolVersion, List standardSyncOperations, @Nullable JsonNode webhookOperationConfigs, @Nullable ActorDefinitionResourceRequirements sourceResourceReqs, @@ -51,6 +54,7 @@ Optional createSyncJob(SourceConnection source, Optional createResetConnectionJob(DestinationConnection destination, StandardSync standardSync, String destinationDockerImage, + Version destinationProtocolVersion, List standardSyncOperations, List streamsToReset) throws IOException; diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/factory/DefaultSyncJobFactory.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/factory/DefaultSyncJobFactory.java index d5ed60d9c24a..b83d8f3497dc 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/factory/DefaultSyncJobFactory.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/factory/DefaultSyncJobFactory.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; import io.airbyte.commons.docker.DockerUtils; +import io.airbyte.commons.version.Version; import io.airbyte.config.ActorDefinitionResourceRequirements; import io.airbyte.config.DestinationConnection; import io.airbyte.config.SourceConnection; @@ -91,7 +92,9 @@ public Long create(final UUID connectionId) { destinationConnection, standardSync, sourceImageName, + new Version(sourceDefinition.getProtocolVersion()), destinationImageName, + new Version(destinationDefinition.getProtocolVersion()), standardSyncOperations, workspace.getWebhookOperationConfigs(), sourceResourceRequirements, diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/models/AttemptNormalizationStatus.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/models/AttemptNormalizationStatus.java index 9575bb8e9968..4c106dd654ab 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/models/AttemptNormalizationStatus.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/models/AttemptNormalizationStatus.java @@ -6,4 +6,4 @@ import java.util.Optional; -public record AttemptNormalizationStatus(long attemptNumber, Optional recordsCommitted, boolean normalizationFailed) {} +public record AttemptNormalizationStatus(int attemptNumber, Optional recordsCommitted, boolean normalizationFailed) {} diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java index deeaaf0ce4ef..8fedbae6cd4b 100644 --- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java +++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java @@ -14,6 +14,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.version.Version; import io.airbyte.config.ActorDefinitionResourceRequirements; import io.airbyte.config.DestinationConnection; import io.airbyte.config.JobConfig; @@ -61,7 +62,9 @@ class DefaultJobCreatorTest { private static final StreamDescriptor STREAM2_DESCRIPTOR = new StreamDescriptor().withName(STREAM2_NAME).withNamespace(NAMESPACE); private static final String SOURCE_IMAGE_NAME = "daxtarity/sourceimagename"; + private static final Version SOURCE_PROTOCOL_VERSION = new Version("0.2.2"); private static final String DESTINATION_IMAGE_NAME = "daxtarity/destinationimagename"; + private static final Version DESTINATION_PROTOCOL_VERSION = new Version("0.2.3"); private static final SourceConnection SOURCE_CONNECTION; private static final DestinationConnection DESTINATION_CONNECTION; private static final StandardSync STANDARD_SYNC; @@ -111,11 +114,17 @@ class DefaultJobCreatorTest { final UUID operationId = UUID.randomUUID(); final ConfiguredAirbyteStream stream1 = new ConfiguredAirbyteStream() - .withStream(CatalogHelpers.createAirbyteStream(STREAM1_NAME, Field.of(FIELD_NAME, JsonSchemaType.STRING))); + .withStream(CatalogHelpers.createAirbyteStream(STREAM1_NAME, Field.of(FIELD_NAME, JsonSchemaType.STRING))) + .withSyncMode(SyncMode.FULL_REFRESH) + .withDestinationSyncMode(DestinationSyncMode.APPEND); final ConfiguredAirbyteStream stream2 = new ConfiguredAirbyteStream() - .withStream(CatalogHelpers.createAirbyteStream(STREAM2_NAME, NAMESPACE, Field.of(FIELD_NAME, JsonSchemaType.STRING))); + .withStream(CatalogHelpers.createAirbyteStream(STREAM2_NAME, NAMESPACE, Field.of(FIELD_NAME, JsonSchemaType.STRING))) + .withSyncMode(SyncMode.INCREMENTAL) + .withDestinationSyncMode(DestinationSyncMode.APPEND); final ConfiguredAirbyteStream stream3 = new ConfiguredAirbyteStream() - .withStream(CatalogHelpers.createAirbyteStream(STREAM3_NAME, NAMESPACE, Field.of(FIELD_NAME, JsonSchemaType.STRING))); + .withStream(CatalogHelpers.createAirbyteStream(STREAM3_NAME, NAMESPACE, Field.of(FIELD_NAME, JsonSchemaType.STRING))) + .withSyncMode(SyncMode.FULL_REFRESH) + .withDestinationSyncMode(DestinationSyncMode.OVERWRITE); final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of(stream1, stream2, stream3)); STANDARD_SYNC = new StandardSync() @@ -162,8 +171,10 @@ void testCreateSyncJob() throws IOException { .withPrefix(STANDARD_SYNC.getPrefix()) .withSourceConfiguration(SOURCE_CONNECTION.getConfiguration()) .withSourceDockerImage(SOURCE_IMAGE_NAME) + .withSourceProtocolVersion(SOURCE_PROTOCOL_VERSION) .withDestinationConfiguration(DESTINATION_CONNECTION.getConfiguration()) .withDestinationDockerImage(DESTINATION_IMAGE_NAME) + .withDestinationProtocolVersion(DESTINATION_PROTOCOL_VERSION) .withConfiguredAirbyteCatalog(STANDARD_SYNC.getCatalog()) .withOperationSequence(List.of(STANDARD_SYNC_OPERATION)) .withResourceRequirements(workerResourceRequirements) @@ -183,7 +194,9 @@ void testCreateSyncJob() throws IOException { DESTINATION_CONNECTION, STANDARD_SYNC, SOURCE_IMAGE_NAME, + SOURCE_PROTOCOL_VERSION, DESTINATION_IMAGE_NAME, + DESTINATION_PROTOCOL_VERSION, List.of(STANDARD_SYNC_OPERATION), PERSISTED_WEBHOOK_CONFIGS, null, @@ -199,8 +212,10 @@ void testCreateSyncJobEnsureNoQueuing() throws IOException { .withPrefix(STANDARD_SYNC.getPrefix()) .withSourceConfiguration(SOURCE_CONNECTION.getConfiguration()) .withSourceDockerImage(SOURCE_IMAGE_NAME) + .withDestinationProtocolVersion(SOURCE_PROTOCOL_VERSION) .withDestinationConfiguration(DESTINATION_CONNECTION.getConfiguration()) .withDestinationDockerImage(DESTINATION_IMAGE_NAME) + .withDestinationProtocolVersion(DESTINATION_PROTOCOL_VERSION) .withConfiguredAirbyteCatalog(STANDARD_SYNC.getCatalog()) .withOperationSequence(List.of(STANDARD_SYNC_OPERATION)) .withResourceRequirements(workerResourceRequirements); @@ -217,7 +232,9 @@ void testCreateSyncJobEnsureNoQueuing() throws IOException { DESTINATION_CONNECTION, STANDARD_SYNC, SOURCE_IMAGE_NAME, + SOURCE_PROTOCOL_VERSION, DESTINATION_IMAGE_NAME, + DESTINATION_PROTOCOL_VERSION, List.of(STANDARD_SYNC_OPERATION), null, null, @@ -231,7 +248,9 @@ void testCreateSyncJobDefaultWorkerResourceReqs() throws IOException { DESTINATION_CONNECTION, STANDARD_SYNC, SOURCE_IMAGE_NAME, + SOURCE_PROTOCOL_VERSION, DESTINATION_IMAGE_NAME, + DESTINATION_PROTOCOL_VERSION, List.of(STANDARD_SYNC_OPERATION), null, null, @@ -243,8 +262,10 @@ void testCreateSyncJobDefaultWorkerResourceReqs() throws IOException { .withPrefix(STANDARD_SYNC.getPrefix()) .withSourceConfiguration(SOURCE_CONNECTION.getConfiguration()) .withSourceDockerImage(SOURCE_IMAGE_NAME) + .withSourceProtocolVersion(SOURCE_PROTOCOL_VERSION) .withDestinationConfiguration(DESTINATION_CONNECTION.getConfiguration()) .withDestinationDockerImage(DESTINATION_IMAGE_NAME) + .withDestinationProtocolVersion(DESTINATION_PROTOCOL_VERSION) .withConfiguredAirbyteCatalog(STANDARD_SYNC.getCatalog()) .withOperationSequence(List.of(STANDARD_SYNC_OPERATION)) .withResourceRequirements(workerResourceRequirements) @@ -274,7 +295,9 @@ void testCreateSyncJobConnectionResourceReqs() throws IOException { DESTINATION_CONNECTION, standardSync, SOURCE_IMAGE_NAME, + SOURCE_PROTOCOL_VERSION, DESTINATION_IMAGE_NAME, + DESTINATION_PROTOCOL_VERSION, List.of(STANDARD_SYNC_OPERATION), null, null, @@ -286,8 +309,10 @@ void testCreateSyncJobConnectionResourceReqs() throws IOException { .withPrefix(STANDARD_SYNC.getPrefix()) .withSourceConfiguration(SOURCE_CONNECTION.getConfiguration()) .withSourceDockerImage(SOURCE_IMAGE_NAME) + .withSourceProtocolVersion(SOURCE_PROTOCOL_VERSION) .withDestinationConfiguration(DESTINATION_CONNECTION.getConfiguration()) .withDestinationDockerImage(DESTINATION_IMAGE_NAME) + .withDestinationProtocolVersion(DESTINATION_PROTOCOL_VERSION) .withConfiguredAirbyteCatalog(STANDARD_SYNC.getCatalog()) .withOperationSequence(List.of(STANDARD_SYNC_OPERATION)) .withResourceRequirements(standardSyncResourceRequirements) @@ -321,7 +346,9 @@ void testCreateSyncJobSourceAndDestinationResourceReqs() throws IOException { DESTINATION_CONNECTION, STANDARD_SYNC, SOURCE_IMAGE_NAME, + SOURCE_PROTOCOL_VERSION, DESTINATION_IMAGE_NAME, + DESTINATION_PROTOCOL_VERSION, List.of(STANDARD_SYNC_OPERATION), null, new ActorDefinitionResourceRequirements().withDefault(sourceResourceRequirements), @@ -334,8 +361,10 @@ void testCreateSyncJobSourceAndDestinationResourceReqs() throws IOException { .withPrefix(STANDARD_SYNC.getPrefix()) .withSourceConfiguration(SOURCE_CONNECTION.getConfiguration()) .withSourceDockerImage(SOURCE_IMAGE_NAME) + .withSourceProtocolVersion(SOURCE_PROTOCOL_VERSION) .withDestinationConfiguration(DESTINATION_CONNECTION.getConfiguration()) .withDestinationDockerImage(DESTINATION_IMAGE_NAME) + .withDestinationProtocolVersion(DESTINATION_PROTOCOL_VERSION) .withConfiguredAirbyteCatalog(STANDARD_SYNC.getCatalog()) .withOperationSequence(List.of(STANDARD_SYNC_OPERATION)) .withResourceRequirements(workerResourceRequirements) @@ -379,6 +408,7 @@ void testCreateResetConnectionJob() throws IOException { .withPrefix(STANDARD_SYNC.getPrefix()) .withDestinationConfiguration(DESTINATION_CONNECTION.getConfiguration()) .withDestinationDockerImage(DESTINATION_IMAGE_NAME) + .withDestinationProtocolVersion(DESTINATION_PROTOCOL_VERSION) .withConfiguredAirbyteCatalog(expectedCatalog) .withOperationSequence(List.of(STANDARD_SYNC_OPERATION)) .withResourceRequirements(workerResourceRequirements) @@ -396,6 +426,7 @@ void testCreateResetConnectionJob() throws IOException { DESTINATION_CONNECTION, STANDARD_SYNC, DESTINATION_IMAGE_NAME, + DESTINATION_PROTOCOL_VERSION, List.of(STANDARD_SYNC_OPERATION), streamsToReset); @@ -432,6 +463,7 @@ void testCreateResetConnectionJobEnsureNoQueuing() throws IOException { .withPrefix(STANDARD_SYNC.getPrefix()) .withDestinationConfiguration(DESTINATION_CONNECTION.getConfiguration()) .withDestinationDockerImage(DESTINATION_IMAGE_NAME) + .withDestinationProtocolVersion(DESTINATION_PROTOCOL_VERSION) .withConfiguredAirbyteCatalog(expectedCatalog) .withOperationSequence(List.of(STANDARD_SYNC_OPERATION)) .withResourceRequirements(workerResourceRequirements) @@ -449,6 +481,7 @@ void testCreateResetConnectionJobEnsureNoQueuing() throws IOException { DESTINATION_CONNECTION, STANDARD_SYNC, DESTINATION_IMAGE_NAME, + DESTINATION_PROTOCOL_VERSION, List.of(STANDARD_SYNC_OPERATION), streamsToReset); diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/factory/DefaultSyncJobFactoryTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/factory/DefaultSyncJobFactoryTest.java index 12295be841ac..dc2cfdc9285d 100644 --- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/factory/DefaultSyncJobFactoryTest.java +++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/factory/DefaultSyncJobFactoryTest.java @@ -14,6 +14,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.version.Version; import io.airbyte.config.DestinationConnection; import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; @@ -65,26 +66,29 @@ void createSyncJobFromConnectionId() throws JsonValidationException, ConfigNotFo final String srcDockerRepo = "srcrepo"; final String srcDockerTag = "tag"; final String srcDockerImage = DockerUtils.getTaggedImageName(srcDockerRepo, srcDockerTag); + final Version srcProtocolVersion = new Version("0.3.1"); final String dstDockerRepo = "dstrepo"; final String dstDockerTag = "tag"; final String dstDockerImage = DockerUtils.getTaggedImageName(dstDockerRepo, dstDockerTag); + final Version dstProtocolVersion = new Version("0.3.2"); when(configRepository.getStandardSync(connectionId)).thenReturn(standardSync); when(configRepository.getSourceConnection(sourceId)).thenReturn(sourceConnection); when(configRepository.getDestinationConnection(destinationId)).thenReturn(destinationConnection); when(configRepository.getStandardSyncOperation(operationId)).thenReturn(operation); when( - jobCreator.createSyncJob(sourceConnection, destinationConnection, standardSync, srcDockerImage, dstDockerImage, operations, + jobCreator.createSyncJob(sourceConnection, destinationConnection, standardSync, srcDockerImage, srcProtocolVersion, dstDockerImage, + dstProtocolVersion, operations, persistedWebhookConfigs, null, null)) .thenReturn(Optional.of(jobId)); when(configRepository.getStandardSourceDefinition(sourceDefinitionId)) .thenReturn(new StandardSourceDefinition().withSourceDefinitionId(sourceDefinitionId).withDockerRepository(srcDockerRepo) - .withDockerImageTag(srcDockerTag)); + .withDockerImageTag(srcDockerTag).withProtocolVersion(srcProtocolVersion.serialize())); when(configRepository.getStandardDestinationDefinition(destinationDefinitionId)) .thenReturn(new StandardDestinationDefinition().withDestinationDefinitionId(destinationDefinitionId).withDockerRepository(dstDockerRepo) - .withDockerImageTag(dstDockerTag)); + .withDockerImageTag(dstDockerTag).withProtocolVersion(dstProtocolVersion.serialize())); when(configRepository.getStandardWorkspaceNoSecrets(any(), eq(true))).thenReturn( new StandardWorkspace().withWebhookOperationConfigs(persistedWebhookConfigs)); @@ -94,7 +98,8 @@ void createSyncJobFromConnectionId() throws JsonValidationException, ConfigNotFo assertEquals(jobId, actualJobId); verify(jobCreator) - .createSyncJob(sourceConnection, destinationConnection, standardSync, srcDockerImage, dstDockerImage, operations, persistedWebhookConfigs, + .createSyncJob(sourceConnection, destinationConnection, standardSync, srcDockerImage, srcProtocolVersion, dstDockerImage, dstProtocolVersion, + operations, persistedWebhookConfigs, null, null); } diff --git a/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index 731ab00a332d..9965bde95825 100644 --- a/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -4,7 +4,7 @@ title: AirbyteProtocol type: object description: AirbyteProtocol structs -version: 0.3.0 +version: 0.3.1 properties: airbyte_message: "$ref": "#/definitions/AirbyteMessage" @@ -28,6 +28,7 @@ definitions: - CONNECTION_STATUS - CATALOG - TRACE + - CONTROL log: description: "log message: any kind of logging you want the platform to know about." "$ref": "#/definitions/AirbyteLogMessage" @@ -48,6 +49,9 @@ definitions: trace: description: "trace message: a message to communicate information about the status and performance of a connector" "$ref": "#/definitions/AirbyteTraceMessage" + control: + description: "connector config message: a message to communicate an updated configuration from a connector that should be persisted" + "$ref": "#/definitions/AirbyteControlMessage" AirbyteRecordMessage: type: object additionalProperties: true @@ -197,6 +201,35 @@ definitions: enum: - system_error - config_error + AirbyteControlMessage: + type: object + additionalProperties: true + required: + - type + - emitted_at + properties: + type: + title: orchestrator type + description: "the type of orchestrator message" + type: string + enum: + - CONNECTOR_CONFIG + emitted_at: + description: "the time in ms that the message was emitted" + type: number + connectorConfig: + description: "connector config orchestrator message: the updated config for the platform to store for this connector" + "$ref": "#/definitions/AirbyteControlConnectorConfigMessage" + AirbyteControlConnectorConfigMessage: + type: object + additionalProperties: true + required: + - config + properties: + config: + description: "the config items from this connector's spec to update" + type: object + additionalProperties: true AirbyteConnectionStatus: type: object description: Airbyte connection status diff --git a/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/well_known_types.yaml b/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/well_known_types.yaml new file mode 100644 index 000000000000..7cb9828063b7 --- /dev/null +++ b/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/well_known_types.yaml @@ -0,0 +1,75 @@ +# Regexes are purely illustrative and need to be workshopped: BC dates shouldn't require 4-digit years; years may have >=5 digits; etc +definitions: + String: + type: string + description: Arbitrary text + BinaryData: + type: string + description: > + Arbitrary binary data. Represented as base64-encoded strings in the JSON transport. + In the future, if we support other transports, may be encoded differently. + # All credit to https://stackoverflow.com/a/475217 for this pattern + pattern: ^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$ + Date: + type: string + # Examples: + # 2022-01-23 + # 2022-01-23 BC + # format: date is a superset of what we want, so we cannot use it here (e.g. it accepts 2-digit years) + pattern: ^\d{4}-\d{2}-\d{2}( BC)?$ + description: RFC 3339§5.6's full-date format, extended with BC era support + TimestampWithTimezone: + type: string + # Examples: + # 2022-01-23T01:23:45Z + # 2022-01-23T01:23:45.678-11:30 BC + # format: date-time is a superset of what we want, so we cannot use it here (e.g. it accepts 2-digit years) + pattern: ^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+\-]\d{1,2}:\d{2})( BC)?$ + description: > + An instant in time. Frequently simply referred to as just a timestamp, or timestamptz. + Uses RFC 3339§5.6's date-time format, requiring a "T" separator, and extended with BC era support. + Note that we do _not_ accept Unix epochs here. + TimestampWithoutTimezone: + type: string + # Examples: + # 2022-01-23T01:23:45 + # 2022-01-23T01:23:45.678 BC + # 2022-01-23T01:23:45.678 + pattern: ^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?( BC)?$ + description: > + Also known as a localdatetime, or just datetime. + Under RFC 3339§5.6, this would be represented as `full-date "T" partial-time`, extended with BC era support. + TimeWithTimezone: + type: string + # Examples: + # 01:23:45Z + # 01:23:45.678-11:30 + pattern: ^\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+\-]\d{1,2}:\d{2})$ + description: An RFC 3339§5.6 full-time + TimeWithoutTimezone: + type: string + # Examples: + # 01:23:45 + # 01:23:45.678 + pattern: ^\d{2}:\d{2}:\d{2}(\.\d+)?$ + description: An RFC 3339§5.6 partial-time + Number: + type: string + oneOf: + - pattern: -?(0|[0-9]\d*)(\.\d+)? + - enum: + - Infinity + - -Infinity + - NaN + description: Note the mix of regex validation for normal numbers, and enum validation for special values. + Integer: + type: string + oneOf: + - pattern: -?(0|[0-9]\d*) + - enum: + - Infinity + - -Infinity + - NaN + Boolean: + type: boolean + description: Note the direct usage of a primitive boolean rather than string. Unlike Numbers and Integers, we don't expect unusual values here. diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index b38e879fcb5b..19094e22d126 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -27,7 +27,7 @@ dependencies { implementation libs.flyway.core implementation 'com.github.slugify:slugify:2.4' implementation 'commons-cli:commons-cli:1.4' - implementation 'io.temporal:temporal-sdk:1.8.1' + implementation libs.temporal.sdk implementation 'org.apache.cxf:cxf-core:3.4.2' implementation 'org.eclipse.jetty:jetty-server:9.4.31.v20200723' implementation 'org.eclipse.jetty:jetty-servlet:9.4.31.v20200723' diff --git a/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java index 4ed1af386612..297718c74fd9 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java @@ -33,8 +33,6 @@ public class ConfigurationApiFactory implements Factory { private static SynchronousSchedulerClient synchronousSchedulerClient; private static StatePersistence statePersistence; private static Map mdc; - private static Database configsDatabase; - private static Database jobsDatabase; private static TrackingClient trackingClient; private static WorkerEnvironment workerEnvironment; private static LogConfigs logConfigs; @@ -42,8 +40,6 @@ public class ConfigurationApiFactory implements Factory { private static AirbyteVersion airbyteVersion; private static HttpClient httpClient; private static EventRunner eventRunner; - private static Flyway configsFlyway; - private static Flyway jobsFlyway; public static void setValues( final ConfigRepository configRepository, @@ -70,8 +66,6 @@ public static void setValues( ConfigurationApiFactory.secretsRepositoryWriter = secretsRepositoryWriter; ConfigurationApiFactory.synchronousSchedulerClient = synchronousSchedulerClient; ConfigurationApiFactory.mdc = mdc; - ConfigurationApiFactory.configsDatabase = configsDatabase; - ConfigurationApiFactory.jobsDatabase = jobsDatabase; ConfigurationApiFactory.trackingClient = trackingClient; ConfigurationApiFactory.workerEnvironment = workerEnvironment; ConfigurationApiFactory.logConfigs = logConfigs; @@ -79,8 +73,6 @@ public static void setValues( ConfigurationApiFactory.airbyteVersion = airbyteVersion; ConfigurationApiFactory.httpClient = httpClient; ConfigurationApiFactory.eventRunner = eventRunner; - ConfigurationApiFactory.configsFlyway = configsFlyway; - ConfigurationApiFactory.jobsFlyway = jobsFlyway; ConfigurationApiFactory.statePersistence = statePersistence; } @@ -94,8 +86,6 @@ public ConfigurationApi provide() { ConfigurationApiFactory.secretsRepositoryReader, ConfigurationApiFactory.secretsRepositoryWriter, ConfigurationApiFactory.synchronousSchedulerClient, - ConfigurationApiFactory.configsDatabase, - ConfigurationApiFactory.jobsDatabase, ConfigurationApiFactory.statePersistence, ConfigurationApiFactory.trackingClient, ConfigurationApiFactory.workerEnvironment, @@ -103,9 +93,7 @@ public ConfigurationApi provide() { ConfigurationApiFactory.airbyteVersion, ConfigurationApiFactory.workspaceRoot, ConfigurationApiFactory.httpClient, - ConfigurationApiFactory.eventRunner, - ConfigurationApiFactory.configsFlyway, - ConfigurationApiFactory.jobsFlyway); + ConfigurationApiFactory.eventRunner); } @Override diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index ee59c365d776..8ec8ac60b824 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -42,6 +42,7 @@ import io.airbyte.persistence.job.DefaultJobPersistence; import io.airbyte.persistence.job.JobPersistence; import io.airbyte.persistence.job.WebUrlHelper; +import io.airbyte.persistence.job.WorkspaceHelper; import io.airbyte.persistence.job.errorreporter.JobErrorReporter; import io.airbyte.persistence.job.errorreporter.JobErrorReportingClient; import io.airbyte.persistence.job.errorreporter.JobErrorReportingClientFactory; @@ -53,10 +54,18 @@ import io.airbyte.server.errors.KnownExceptionMapper; import io.airbyte.server.errors.NotFoundExceptionMapper; import io.airbyte.server.errors.UncaughtExceptionMapper; +import io.airbyte.server.handlers.AttemptHandler; +import io.airbyte.server.handlers.ConnectionsHandler; import io.airbyte.server.handlers.DbMigrationHandler; +import io.airbyte.server.handlers.DestinationDefinitionsHandler; +import io.airbyte.server.handlers.DestinationHandler; +import io.airbyte.server.handlers.HealthCheckHandler; +import io.airbyte.server.handlers.OperationsHandler; +import io.airbyte.server.handlers.SchedulerHandler; import io.airbyte.server.scheduler.DefaultSynchronousSchedulerClient; import io.airbyte.server.scheduler.EventRunner; import io.airbyte.server.scheduler.TemporalEventRunner; +import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.normalization.NormalizationRunnerFactory; import io.temporal.serviceclient.WorkflowServiceStubs; @@ -255,6 +264,44 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, // "major" version bump as it will no longer be needed. migrateExistingConnectionsToTemporalScheduler(configRepository, jobPersistence, eventRunner); + final WorkspaceHelper workspaceHelper = new WorkspaceHelper(configRepository, jobPersistence); + + final JsonSchemaValidator schemaValidator = new JsonSchemaValidator(); + + final AttemptHandler attemptHandler = new AttemptHandler(jobPersistence); + + final ConnectionsHandler connectionsHandler = new ConnectionsHandler( + configRepository, + workspaceHelper, + trackingClient, + eventRunner); + + final DestinationHandler destinationHandler = new DestinationHandler( + configRepository, + secretsRepositoryReader, + secretsRepositoryWriter, + schemaValidator, + connectionsHandler); + + final OperationsHandler operationsHandler = new OperationsHandler(configRepository); + + final SchedulerHandler schedulerHandler = new SchedulerHandler( + configRepository, + secretsRepositoryReader, + secretsRepositoryWriter, + syncSchedulerClient, + jobPersistence, + configs.getWorkerEnvironment(), + configs.getLogConfigs(), + eventRunner); + + final DbMigrationHandler dbMigrationHandler = new DbMigrationHandler(configsDatabase, configsFlyway, jobsDatabase, jobsFlyway); + + final DestinationDefinitionsHandler destinationDefinitionsHandler = new DestinationDefinitionsHandler(configRepository, syncSchedulerClient, + destinationHandler); + + final HealthCheckHandler healthCheckHandler = new HealthCheckHandler(configRepository); + LOGGER.info("Starting server..."); return apiFactory.create( @@ -273,7 +320,15 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, httpClient, eventRunner, configsFlyway, - jobsFlyway); + jobsFlyway, + attemptHandler, + connectionsHandler, + dbMigrationHandler, + destinationDefinitionsHandler, + destinationHandler, + healthCheckHandler, + operationsHandler, + schedulerHandler); } @VisibleForTesting diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index a86486d53961..42a18c32ddc6 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -14,33 +14,71 @@ import io.airbyte.config.persistence.StatePersistence; import io.airbyte.db.Database; import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.server.apis.AttemptApiController; import io.airbyte.server.apis.ConfigurationApi; +import io.airbyte.server.apis.ConnectionApiController; +import io.airbyte.server.apis.DbMigrationApiController; +import io.airbyte.server.apis.DestinationApiController; +import io.airbyte.server.apis.DestinationDefinitionApiController; +import io.airbyte.server.apis.DestinationDefinitionSpecificationApiController; +import io.airbyte.server.apis.HealthApiController; +import io.airbyte.server.apis.binders.AttemptApiBinder; +import io.airbyte.server.apis.binders.ConnectionApiBinder; +import io.airbyte.server.apis.binders.DbMigrationBinder; +import io.airbyte.server.apis.binders.DestinationApiBinder; +import io.airbyte.server.apis.binders.DestinationDefinitionApiBinder; +import io.airbyte.server.apis.binders.DestinationDefinitionSpecificationApiBinder; +import io.airbyte.server.apis.binders.HealthApiBinder; +import io.airbyte.server.apis.factories.AttemptApiFactory; +import io.airbyte.server.apis.factories.ConnectionApiFactory; +import io.airbyte.server.apis.factories.DbMigrationApiFactory; +import io.airbyte.server.apis.factories.DestinationApiFactory; +import io.airbyte.server.apis.factories.DestinationDefinitionApiFactory; +import io.airbyte.server.apis.factories.DestinationDefinitionSpecificationApiFactory; +import io.airbyte.server.apis.factories.HealthApiFactory; +import io.airbyte.server.handlers.AttemptHandler; +import io.airbyte.server.handlers.ConnectionsHandler; +import io.airbyte.server.handlers.DbMigrationHandler; +import io.airbyte.server.handlers.DestinationDefinitionsHandler; +import io.airbyte.server.handlers.DestinationHandler; +import io.airbyte.server.handlers.HealthCheckHandler; +import io.airbyte.server.handlers.OperationsHandler; +import io.airbyte.server.handlers.SchedulerHandler; import io.airbyte.server.scheduler.EventRunner; import io.airbyte.server.scheduler.SynchronousSchedulerClient; import java.net.http.HttpClient; import java.nio.file.Path; +import java.util.Map; import java.util.Set; import org.flywaydb.core.Flyway; import org.slf4j.MDC; public interface ServerFactory { - ServerRunnable create(SynchronousSchedulerClient cachingSchedulerClient, - ConfigRepository configRepository, - SecretsRepositoryReader secretsRepositoryReader, - SecretsRepositoryWriter secretsRepositoryWriter, - JobPersistence jobPersistence, - Database configsDatabase, - Database jobsDatabase, - TrackingClient trackingClient, - WorkerEnvironment workerEnvironment, - LogConfigs logConfigs, - AirbyteVersion airbyteVersion, - Path workspaceRoot, - HttpClient httpClient, - EventRunner eventRunner, - Flyway configsFlyway, - Flyway jobsFlyway); + ServerRunnable create(final SynchronousSchedulerClient synchronousSchedulerClient, + final ConfigRepository configRepository, + final SecretsRepositoryReader secretsRepositoryReader, + final SecretsRepositoryWriter secretsRepositoryWriter, + final JobPersistence jobPersistence, + final Database configsDatabase, + final Database jobsDatabase, + final TrackingClient trackingClient, + final WorkerEnvironment workerEnvironment, + final LogConfigs logConfigs, + final AirbyteVersion airbyteVersion, + final Path workspaceRoot, + final HttpClient httpClient, + final EventRunner eventRunner, + final Flyway configsFlyway, + final Flyway jobsFlyway, + final AttemptHandler attemptHandler, + final ConnectionsHandler connectionsHandler, + final DbMigrationHandler dbMigrationHandler, + final DestinationDefinitionsHandler destinationDefinitionsHandler, + final DestinationHandler destinationApiHandler, + final HealthCheckHandler healthCheckHandler, + final OperationsHandler operationsHandler, + final SchedulerHandler schedulerHandler); class Api implements ServerFactory { @@ -60,7 +98,17 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul final HttpClient httpClient, final EventRunner eventRunner, final Flyway configsFlyway, - final Flyway jobsFlyway) { + final Flyway jobsFlyway, + final AttemptHandler attemptHandler, + final ConnectionsHandler connectionsHandler, + final DbMigrationHandler dbMigrationHandler, + final DestinationDefinitionsHandler destinationDefinitionsHandler, + final DestinationHandler destinationApiHandler, + final HealthCheckHandler healthCheckHandler, + final OperationsHandler operationsHandler, + final SchedulerHandler schedulerHandler) { + final Map mdc = MDC.getCopyOfContextMap(); + // set static values for factory ConfigurationApiFactory.setValues( configRepository, @@ -69,7 +117,7 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul jobPersistence, synchronousSchedulerClient, new StatePersistence(configsDatabase), - MDC.getCopyOfContextMap(), + mdc, configsDatabase, jobsDatabase, trackingClient, @@ -82,9 +130,45 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul configsFlyway, jobsFlyway); + AttemptApiFactory.setValues(attemptHandler, mdc); + + ConnectionApiFactory.setValues( + connectionsHandler, + operationsHandler, + schedulerHandler, + mdc); + + DbMigrationApiFactory.setValues(dbMigrationHandler, mdc); + + DestinationApiFactory.setValues(destinationApiHandler, schedulerHandler, mdc); + + DestinationDefinitionApiFactory.setValues(destinationDefinitionsHandler); + + DestinationDefinitionSpecificationApiFactory.setValues(schedulerHandler); + + HealthApiFactory.setValues(healthCheckHandler); + // server configurations - final Set> componentClasses = Set.of(ConfigurationApi.class); - final Set components = Set.of(new CorsFilter(), new ConfigurationApiBinder()); + final Set> componentClasses = Set.of( + ConfigurationApi.class, + AttemptApiController.class, + ConnectionApiController.class, + DbMigrationApiController.class, + DestinationApiController.class, + DestinationDefinitionApiController.class, + DestinationDefinitionSpecificationApiController.class, + HealthApiController.class); + + final Set components = Set.of( + new CorsFilter(), + new ConfigurationApiBinder(), + new AttemptApiBinder(), + new ConnectionApiBinder(), + new DbMigrationBinder(), + new DestinationApiBinder(), + new DestinationDefinitionApiBinder(), + new DestinationDefinitionSpecificationApiBinder(), + new HealthApiBinder()); // construct server return new ServerApp(airbyteVersion, componentClasses, components); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java new file mode 100644 index 000000000000..71274154cca4 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis; + +import io.airbyte.api.generated.AttemptApi; +import io.airbyte.api.model.generated.InternalOperationResult; +import io.airbyte.api.model.generated.SetWorkflowInAttemptRequestBody; +import io.airbyte.server.handlers.AttemptHandler; +import javax.ws.rs.Path; + +@Path("/v1/attempt/set_workflow_in_attempt") +public class AttemptApiController implements AttemptApi { + + private final AttemptHandler attemptHandler; + + public AttemptApiController(final AttemptHandler attemptHandler) { + this.attemptHandler = attemptHandler; + } + + @Override + public InternalOperationResult setWorkflowInAttempt(final SetWorkflowInAttemptRequestBody requestBody) { + return ConfigurationApi.execute(() -> attemptHandler.setWorkflowInAttempt(requestBody)); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index ae575aaa2572..6bc240c92a41 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -5,6 +5,7 @@ package io.airbyte.server.apis; import io.airbyte.analytics.TrackingClient; +import io.airbyte.api.model.generated.AttemptNormalizationStatusReadList; import io.airbyte.api.model.generated.CheckConnectionRead; import io.airbyte.api.model.generated.CheckOperationRead; import io.airbyte.api.model.generated.CompleteDestinationOAuthRequest; @@ -108,17 +109,13 @@ import io.airbyte.config.persistence.SecretsRepositoryReader; import io.airbyte.config.persistence.SecretsRepositoryWriter; import io.airbyte.config.persistence.StatePersistence; -import io.airbyte.db.Database; import io.airbyte.persistence.job.JobPersistence; import io.airbyte.persistence.job.WorkspaceHelper; import io.airbyte.server.errors.BadObjectSchemaKnownException; import io.airbyte.server.errors.IdNotFoundKnownException; -import io.airbyte.server.handlers.AttemptHandler; import io.airbyte.server.handlers.ConnectionsHandler; -import io.airbyte.server.handlers.DbMigrationHandler; import io.airbyte.server.handlers.DestinationDefinitionsHandler; import io.airbyte.server.handlers.DestinationHandler; -import io.airbyte.server.handlers.HealthCheckHandler; import io.airbyte.server.handlers.JobHistoryHandler; import io.airbyte.server.handlers.LogsHandler; import io.airbyte.server.handlers.OAuthHandler; @@ -141,7 +138,7 @@ import java.nio.file.Path; import java.util.Map; import lombok.extern.slf4j.Slf4j; -import org.flywaydb.core.Flyway; +import org.apache.commons.lang3.NotImplementedException; @javax.ws.rs.Path("/v1") @Slf4j @@ -159,12 +156,9 @@ public class ConfigurationApi implements io.airbyte.api.generated.V1Api { private final JobHistoryHandler jobHistoryHandler; private final WebBackendConnectionsHandler webBackendConnectionsHandler; private final WebBackendGeographiesHandler webBackendGeographiesHandler; - private final HealthCheckHandler healthCheckHandler; private final LogsHandler logsHandler; private final OpenApiConfigHandler openApiConfigHandler; - private final DbMigrationHandler dbMigrationHandler; private final OAuthHandler oAuthHandler; - private final AttemptHandler attemptHandler; private final WorkerEnvironment workerEnvironment; private final LogConfigs logConfigs; private final Path workspaceRoot; @@ -174,8 +168,6 @@ public ConfigurationApi(final ConfigRepository configRepository, final SecretsRepositoryReader secretsRepositoryReader, final SecretsRepositoryWriter secretsRepositoryWriter, final SynchronousSchedulerClient synchronousSchedulerClient, - final Database configsDatabase, - final Database jobsDatabase, final StatePersistence statePersistence, final TrackingClient trackingClient, final WorkerEnvironment workerEnvironment, @@ -183,9 +175,7 @@ public ConfigurationApi(final ConfigRepository configRepository, final AirbyteVersion airbyteVersion, final Path workspaceRoot, final HttpClient httpClient, - final EventRunner eventRunner, - final Flyway configsFlyway, - final Flyway jobsFlyway) { + final EventRunner eventRunner) { this.workerEnvironment = workerEnvironment; this.logConfigs = logConfigs; this.workspaceRoot = workspaceRoot; @@ -245,11 +235,8 @@ public ConfigurationApi(final ConfigRepository configRepository, eventRunner, configRepository); webBackendGeographiesHandler = new WebBackendGeographiesHandler(); - healthCheckHandler = new HealthCheckHandler(configRepository); logsHandler = new LogsHandler(); openApiConfigHandler = new OpenApiConfigHandler(); - dbMigrationHandler = new DbMigrationHandler(configsDatabase, configsFlyway, jobsDatabase, jobsFlyway); - attemptHandler = new AttemptHandler(jobPersistence); } // WORKSPACE @@ -433,6 +420,15 @@ public void setInstancewideSourceOauthParams(final SetInstancewideSourceOauthPar }); } + /** + * This implementation has been moved to {@link AttemptApiController}. Since the path of + * {@link AttemptApiController} is more granular, it will override this implementation + */ + @Override + public InternalOperationResult setWorkflowInAttempt(final SetWorkflowInAttemptRequestBody setWorkflowInAttemptRequestBody) { + throw new NotImplementedException(); + } + // SOURCE IMPLEMENTATION @Override @@ -490,208 +486,346 @@ public SourceDiscoverSchemaRead discoverSchemaForSource(final SourceDiscoverSche // DB MIGRATION + /** + * This implementation has been moved to {@link DbMigrationApiController}. Since the path of + * {@link DbMigrationApiController} is more granular, it will override this implementation + */ @Override public DbMigrationReadList listMigrations(final DbMigrationRequestBody request) { - return execute(() -> dbMigrationHandler.list(request)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DbMigrationApiController}. Since the path of + * {@link DbMigrationApiController} is more granular, it will override this implementation + */ @Override public DbMigrationExecutionRead executeMigrations(final DbMigrationRequestBody request) { - return execute(() -> dbMigrationHandler.migrate(request)); + throw new NotImplementedException(); } // DESTINATION + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation + */ @Override public DestinationDefinitionReadList listDestinationDefinitions() { - return execute(destinationDefinitionsHandler::listDestinationDefinitions); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation + */ @Override public DestinationDefinitionReadList listDestinationDefinitionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { - return execute(() -> destinationDefinitionsHandler.listDestinationDefinitionsForWorkspace(workspaceIdRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation + */ @Override public DestinationDefinitionReadList listLatestDestinationDefinitions() { - return execute(destinationDefinitionsHandler::listLatestDestinationDefinitions); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation + */ @Override public PrivateDestinationDefinitionReadList listPrivateDestinationDefinitions(final WorkspaceIdRequestBody workspaceIdRequestBody) { - return execute(() -> destinationDefinitionsHandler.listPrivateDestinationDefinitions(workspaceIdRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation + */ @Override public DestinationDefinitionRead getDestinationDefinition(final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) { - return execute(() -> destinationDefinitionsHandler.getDestinationDefinition(destinationDefinitionIdRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation + */ @Override public DestinationDefinitionRead getDestinationDefinitionForWorkspace( final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { - return execute(() -> destinationDefinitionsHandler.getDestinationDefinitionForWorkspace(destinationDefinitionIdWithWorkspaceId)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation + */ // TODO: Deprecate this route in favor of createCustomDestinationDefinition // since all connector definitions created through the API are custom @Override public DestinationDefinitionRead createDestinationDefinition(final DestinationDefinitionCreate destinationDefinitionCreate) { - return execute(() -> destinationDefinitionsHandler.createPrivateDestinationDefinition(destinationDefinitionCreate)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation + */ @Override public DestinationDefinitionRead createCustomDestinationDefinition(final CustomDestinationDefinitionCreate customDestinationDefinitionCreate) { - return execute(() -> destinationDefinitionsHandler.createCustomDestinationDefinition(customDestinationDefinitionCreate)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation + */ @Override public DestinationDefinitionRead updateDestinationDefinition(final DestinationDefinitionUpdate destinationDefinitionUpdate) { - return execute(() -> destinationDefinitionsHandler.updateDestinationDefinition(destinationDefinitionUpdate)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation + */ @Override public DestinationDefinitionRead updateCustomDestinationDefinition(final CustomDestinationDefinitionUpdate customDestinationDefinitionUpdate) { - return execute(() -> destinationDefinitionsHandler.updateCustomDestinationDefinition(customDestinationDefinitionUpdate)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation + */ @Override public void deleteDestinationDefinition(final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) { - execute(() -> { - destinationDefinitionsHandler.deleteDestinationDefinition(destinationDefinitionIdRequestBody); - return null; - }); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation + */ @Override public void deleteCustomDestinationDefinition(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { - execute(() -> { - destinationDefinitionsHandler.deleteCustomDestinationDefinition(destinationDefinitionIdWithWorkspaceId); - return null; - }); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation + */ @Override public PrivateDestinationDefinitionRead grantDestinationDefinitionToWorkspace( final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { - return execute(() -> destinationDefinitionsHandler.grantDestinationDefinitionToWorkspace(destinationDefinitionIdWithWorkspaceId)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationDefinitionApiController}. Since the path + * of {@link DestinationDefinitionApiController} is more granular, it will override this + * implementation + */ @Override public void revokeDestinationDefinitionFromWorkspace(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { - execute(() -> { - destinationDefinitionsHandler.revokeDestinationDefinitionFromWorkspace(destinationDefinitionIdWithWorkspaceId); - return null; - }); + throw new NotImplementedException(); } // DESTINATION SPECIFICATION - + /** + * This implementation has been moved to {@link DestinationDefinitionSpecificationApiController}. + * Since the path of {@link DestinationDefinitionSpecificationApiController} is more granular, it + * will override this implementation + */ @Override public DestinationDefinitionSpecificationRead getDestinationDefinitionSpecification( final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { - return execute(() -> schedulerHandler.getDestinationSpecification(destinationDefinitionIdWithWorkspaceId)); + throw new NotImplementedException(); } // DESTINATION IMPLEMENTATION + /** + * This implementation has been moved to {@link DestinationApiController}. Since the path of + * {@link DestinationApiController} is more granular, it will override this implementation + */ @Override public DestinationRead createDestination(final DestinationCreate destinationCreate) { - return execute(() -> destinationHandler.createDestination(destinationCreate)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationApiController}. Since the path of + * {@link DestinationApiController} is more granular, it will override this implementation + */ @Override public void deleteDestination(final DestinationIdRequestBody destinationIdRequestBody) { - execute(() -> { - destinationHandler.deleteDestination(destinationIdRequestBody); - return null; - }); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationApiController}. Since the path of + * {@link DestinationApiController} is more granular, it will override this implementation + */ @Override public DestinationRead updateDestination(final DestinationUpdate destinationUpdate) { - return execute(() -> destinationHandler.updateDestination(destinationUpdate)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationApiController}. Since the path of + * {@link DestinationApiController} is more granular, it will override this implementation + */ @Override public DestinationReadList listDestinationsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { - return execute(() -> destinationHandler.listDestinationsForWorkspace(workspaceIdRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationApiController}. Since the path of + * {@link DestinationApiController} is more granular, it will override this implementation + */ @Override public DestinationReadList searchDestinations(final DestinationSearch destinationSearch) { - return execute(() -> destinationHandler.searchDestinations(destinationSearch)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationApiController}. Since the path of + * {@link DestinationApiController} is more granular, it will override this implementation + */ @Override public DestinationRead getDestination(final DestinationIdRequestBody destinationIdRequestBody) { - return execute(() -> destinationHandler.getDestination(destinationIdRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationApiController}. Since the path of + * {@link DestinationApiController} is more granular, it will override this implementation + */ @Override public DestinationRead cloneDestination(final DestinationCloneRequestBody destinationCloneRequestBody) { - return execute(() -> destinationHandler.cloneDestination(destinationCloneRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationApiController}. Since the path of + * {@link DestinationApiController} is more granular, it will override this implementation + */ @Override public CheckConnectionRead checkConnectionToDestination(final DestinationIdRequestBody destinationIdRequestBody) { - return execute(() -> schedulerHandler.checkDestinationConnectionFromDestinationId(destinationIdRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link DestinationApiController}. Since the path of + * {@link DestinationApiController} is more granular, it will override this implementation + */ @Override public CheckConnectionRead checkConnectionToDestinationForUpdate(final DestinationUpdate destinationUpdate) { - return execute(() -> schedulerHandler.checkDestinationConnectionFromDestinationIdForUpdate(destinationUpdate)); + throw new NotImplementedException(); } // CONNECTION + /** + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation + */ @Override public ConnectionRead createConnection(final ConnectionCreate connectionCreate) { - return execute(() -> connectionsHandler.createConnection(connectionCreate)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation + */ @Override public ConnectionRead updateConnection(final ConnectionUpdate connectionUpdate) { - return execute(() -> connectionsHandler.updateConnection(connectionUpdate)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation + */ @Override public ConnectionReadList listConnectionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { - return execute(() -> connectionsHandler.listConnectionsForWorkspace(workspaceIdRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation + */ @Override public ConnectionReadList listAllConnectionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { - return execute(() -> connectionsHandler.listAllConnectionsForWorkspace(workspaceIdRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation + */ @Override public ConnectionReadList searchConnections(final ConnectionSearch connectionSearch) { - return execute(() -> connectionsHandler.searchConnections(connectionSearch)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation + */ @Override public ConnectionRead getConnection(final ConnectionIdRequestBody connectionIdRequestBody) { - return execute(() -> connectionsHandler.getConnection(connectionIdRequestBody.getConnectionId())); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation + */ @Override public void deleteConnection(final ConnectionIdRequestBody connectionIdRequestBody) { - execute(() -> { - operationsHandler.deleteOperationsForConnection(connectionIdRequestBody); - connectionsHandler.deleteConnection(connectionIdRequestBody.getConnectionId()); - return null; - }); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation + */ @Override public JobInfoRead syncConnection(final ConnectionIdRequestBody connectionIdRequestBody) { - return execute(() -> schedulerHandler.syncConnection(connectionIdRequestBody)); + throw new NotImplementedException(); } + /** + * This implementation has been moved to {@link ConnectionApiController}. Since the path of + * {@link ConnectionApiController} is more granular, it will override this implementation + */ @Override public JobInfoRead resetConnection(final ConnectionIdRequestBody connectionIdRequestBody) { - return execute(() -> schedulerHandler.resetConnection(connectionIdRequestBody)); + throw new NotImplementedException(); } // Operations @@ -708,7 +842,7 @@ public OperationRead createOperation(final OperationCreate operationCreate) { @Override public ConnectionState createOrUpdateState(final ConnectionStateCreateOrUpdate connectionStateCreateOrUpdate) { - return execute(() -> stateHandler.createOrUpdateState(connectionStateCreateOrUpdate)); + return ConfigurationApi.execute(() -> stateHandler.createOrUpdateState(connectionStateCreateOrUpdate)); } @Override @@ -736,7 +870,7 @@ public OperationRead updateOperation(final OperationUpdate operationUpdate) { @Override public ConnectionState getState(final ConnectionIdRequestBody connectionIdRequestBody) { - return execute(() -> stateHandler.getState(connectionIdRequestBody)); + return ConfigurationApi.execute(() -> stateHandler.getState(connectionIdRequestBody)); } // SCHEDULER @@ -782,6 +916,11 @@ public JobDebugInfoRead getJobDebugInfo(final JobIdRequestBody jobIdRequestBody) return execute(() -> jobHistoryHandler.getJobDebugInfo(jobIdRequestBody)); } + @Override + public AttemptNormalizationStatusReadList getAttemptNormalizationStatusesForJob(final JobIdRequestBody jobIdRequestBody) { + return execute(() -> jobHistoryHandler.getAttemptNormalizationStatuses(jobIdRequestBody)); + } + @Override public File getLogs(final LogsRequestBody logsRequestBody) { return execute(() -> logsHandler.getLogs(workspaceRoot, workerEnvironment, logConfigs, logsRequestBody)); @@ -793,9 +932,13 @@ public File getOpenApiSpec() { } // HEALTH + /** + * This implementation has been moved to {@link HealthApiController}. Since the path of + * {@link HealthApiController} is more granular, it will override this implementation + */ @Override public HealthCheckRead getHealthCheck() { - return healthCheckHandler.health(); + throw new NotImplementedException(); } // WEB BACKEND @@ -827,7 +970,7 @@ public WebBackendConnectionRead webBackendUpdateConnection(final WebBackendConne @Override public ConnectionStateType getStateType(final ConnectionIdRequestBody connectionIdRequestBody) { - return execute(() -> webBackendConnectionsHandler.getStateType(connectionIdRequestBody)); + return ConfigurationApi.execute(() -> webBackendConnectionsHandler.getStateType(connectionIdRequestBody)); } @Override @@ -835,12 +978,8 @@ public WebBackendWorkspaceStateResult webBackendGetWorkspaceState(final WebBacke return execute(() -> webBackendConnectionsHandler.getWorkspaceState(webBackendWorkspaceState)); } - @Override - public InternalOperationResult setWorkflowInAttempt(final SetWorkflowInAttemptRequestBody requestBody) { - return execute(() -> attemptHandler.setWorkflowInAttempt(requestBody)); - } - - private static T execute(final HandlerCall call) { + // TODO: Move to common when all the api are moved + static T execute(final HandlerCall call) { try { return call.call(); } catch (final ConfigNotFoundException e) { @@ -854,7 +993,7 @@ private static T execute(final HandlerCall call) { } } - private interface HandlerCall { + interface HandlerCall { T call() throws ConfigNotFoundException, IOException, JsonValidationException; diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java new file mode 100644 index 000000000000..21d80c04e84c --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis; + +import io.airbyte.api.generated.ConnectionApi; +import io.airbyte.api.model.generated.ConnectionCreate; +import io.airbyte.api.model.generated.ConnectionIdRequestBody; +import io.airbyte.api.model.generated.ConnectionRead; +import io.airbyte.api.model.generated.ConnectionReadList; +import io.airbyte.api.model.generated.ConnectionSearch; +import io.airbyte.api.model.generated.ConnectionUpdate; +import io.airbyte.api.model.generated.JobInfoRead; +import io.airbyte.api.model.generated.WorkspaceIdRequestBody; +import io.airbyte.server.handlers.ConnectionsHandler; +import io.airbyte.server.handlers.OperationsHandler; +import io.airbyte.server.handlers.SchedulerHandler; +import javax.ws.rs.Path; + +@Path("/v1/connections") +public class ConnectionApiController implements ConnectionApi { + + private final ConnectionsHandler connectionsHandler; + private final OperationsHandler operationsHandler; + private final SchedulerHandler schedulerHandler; + + public ConnectionApiController(final ConnectionsHandler connectionsHandler, + final OperationsHandler operationsHandler, + final SchedulerHandler schedulerHandler) { + this.connectionsHandler = connectionsHandler; + this.operationsHandler = operationsHandler; + this.schedulerHandler = schedulerHandler; + } + + @Override + public ConnectionRead createConnection(final ConnectionCreate connectionCreate) { + return ConfigurationApi.execute(() -> connectionsHandler.createConnection(connectionCreate)); + } + + @Override + public ConnectionRead updateConnection(final ConnectionUpdate connectionUpdate) { + return ConfigurationApi.execute(() -> connectionsHandler.updateConnection(connectionUpdate)); + } + + @Override + public ConnectionReadList listConnectionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { + return ConfigurationApi.execute(() -> connectionsHandler.listConnectionsForWorkspace(workspaceIdRequestBody)); + } + + @Override + public ConnectionReadList listAllConnectionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { + return ConfigurationApi.execute(() -> connectionsHandler.listAllConnectionsForWorkspace(workspaceIdRequestBody)); + } + + @Override + public ConnectionReadList searchConnections(final ConnectionSearch connectionSearch) { + return ConfigurationApi.execute(() -> connectionsHandler.searchConnections(connectionSearch)); + } + + @Override + public ConnectionRead getConnection(final ConnectionIdRequestBody connectionIdRequestBody) { + return ConfigurationApi.execute(() -> connectionsHandler.getConnection(connectionIdRequestBody.getConnectionId())); + } + + @Override + public void deleteConnection(final ConnectionIdRequestBody connectionIdRequestBody) { + ConfigurationApi.execute(() -> { + operationsHandler.deleteOperationsForConnection(connectionIdRequestBody); + connectionsHandler.deleteConnection(connectionIdRequestBody.getConnectionId()); + return null; + }); + } + + @Override + public JobInfoRead syncConnection(final ConnectionIdRequestBody connectionIdRequestBody) { + return ConfigurationApi.execute(() -> schedulerHandler.syncConnection(connectionIdRequestBody)); + } + + @Override + public JobInfoRead resetConnection(final ConnectionIdRequestBody connectionIdRequestBody) { + return ConfigurationApi.execute(() -> schedulerHandler.resetConnection(connectionIdRequestBody)); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DbMigrationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DbMigrationApiController.java new file mode 100644 index 000000000000..93cb85dd33a7 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DbMigrationApiController.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis; + +import io.airbyte.api.generated.DbMigrationApi; +import io.airbyte.api.model.generated.DbMigrationExecutionRead; +import io.airbyte.api.model.generated.DbMigrationReadList; +import io.airbyte.api.model.generated.DbMigrationRequestBody; +import io.airbyte.server.handlers.DbMigrationHandler; +import javax.ws.rs.Path; + +@Path("/v1/db_migrations") +public class DbMigrationApiController implements DbMigrationApi { + + private final DbMigrationHandler dbMigrationHandler; + + public DbMigrationApiController(final DbMigrationHandler dbMigrationHandler) { + this.dbMigrationHandler = dbMigrationHandler; + } + + @Override + public DbMigrationExecutionRead executeMigrations(final DbMigrationRequestBody dbMigrationRequestBody) { + return ConfigurationApi.execute(() -> dbMigrationHandler.migrate(dbMigrationRequestBody)); + } + + @Override + public DbMigrationReadList listMigrations(final DbMigrationRequestBody dbMigrationRequestBody) { + return ConfigurationApi.execute(() -> dbMigrationHandler.list(dbMigrationRequestBody)); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationApiController.java new file mode 100644 index 000000000000..99148027a59e --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationApiController.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis; + +import io.airbyte.api.generated.DestinationApi; +import io.airbyte.api.model.generated.CheckConnectionRead; +import io.airbyte.api.model.generated.DestinationCloneRequestBody; +import io.airbyte.api.model.generated.DestinationCreate; +import io.airbyte.api.model.generated.DestinationIdRequestBody; +import io.airbyte.api.model.generated.DestinationRead; +import io.airbyte.api.model.generated.DestinationReadList; +import io.airbyte.api.model.generated.DestinationSearch; +import io.airbyte.api.model.generated.DestinationUpdate; +import io.airbyte.api.model.generated.WorkspaceIdRequestBody; +import io.airbyte.server.handlers.DestinationHandler; +import io.airbyte.server.handlers.SchedulerHandler; +import javax.ws.rs.Path; +import lombok.AllArgsConstructor; + +@Path("/v1/destinations") +@AllArgsConstructor +public class DestinationApiController implements DestinationApi { + + private final DestinationHandler destinationHandler; + private final SchedulerHandler schedulerHandler; + + @Override + public CheckConnectionRead checkConnectionToDestination(final DestinationIdRequestBody destinationIdRequestBody) { + return ConfigurationApi.execute(() -> schedulerHandler.checkDestinationConnectionFromDestinationId(destinationIdRequestBody)); + } + + @Override + public CheckConnectionRead checkConnectionToDestinationForUpdate(final DestinationUpdate destinationUpdate) { + return ConfigurationApi.execute(() -> schedulerHandler.checkDestinationConnectionFromDestinationIdForUpdate(destinationUpdate)); + } + + @Override + public DestinationRead cloneDestination(final DestinationCloneRequestBody destinationCloneRequestBody) { + return ConfigurationApi.execute(() -> destinationHandler.cloneDestination(destinationCloneRequestBody)); + } + + @Override + public DestinationRead createDestination(final DestinationCreate destinationCreate) { + return ConfigurationApi.execute(() -> destinationHandler.createDestination(destinationCreate)); + } + + @Override + public void deleteDestination(final DestinationIdRequestBody destinationIdRequestBody) { + ConfigurationApi.execute(() -> { + destinationHandler.deleteDestination(destinationIdRequestBody); + return null; + }); + } + + @Override + public DestinationRead getDestination(final DestinationIdRequestBody destinationIdRequestBody) { + return ConfigurationApi.execute(() -> destinationHandler.getDestination(destinationIdRequestBody)); + } + + @Override + public DestinationReadList listDestinationsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { + return ConfigurationApi.execute(() -> destinationHandler.listDestinationsForWorkspace(workspaceIdRequestBody)); + } + + @Override + public DestinationReadList searchDestinations(final DestinationSearch destinationSearch) { + return ConfigurationApi.execute(() -> destinationHandler.searchDestinations(destinationSearch)); + } + + @Override + public DestinationRead updateDestination(final DestinationUpdate destinationUpdate) { + return ConfigurationApi.execute(() -> destinationHandler.updateDestination(destinationUpdate)); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java new file mode 100644 index 000000000000..209a1f1c3eb0 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionApiController.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis; + +import io.airbyte.api.generated.DestinationDefinitionApi; +import io.airbyte.api.model.generated.CustomDestinationDefinitionCreate; +import io.airbyte.api.model.generated.CustomDestinationDefinitionUpdate; +import io.airbyte.api.model.generated.DestinationDefinitionCreate; +import io.airbyte.api.model.generated.DestinationDefinitionIdRequestBody; +import io.airbyte.api.model.generated.DestinationDefinitionIdWithWorkspaceId; +import io.airbyte.api.model.generated.DestinationDefinitionRead; +import io.airbyte.api.model.generated.DestinationDefinitionReadList; +import io.airbyte.api.model.generated.DestinationDefinitionUpdate; +import io.airbyte.api.model.generated.PrivateDestinationDefinitionRead; +import io.airbyte.api.model.generated.PrivateDestinationDefinitionReadList; +import io.airbyte.api.model.generated.WorkspaceIdRequestBody; +import io.airbyte.server.handlers.DestinationDefinitionsHandler; +import javax.ws.rs.Path; +import lombok.AllArgsConstructor; + +@Path("/v1/destination_definitions") +@AllArgsConstructor +public class DestinationDefinitionApiController implements DestinationDefinitionApi { + + private final DestinationDefinitionsHandler destinationDefinitionsHandler; + + @Override + public DestinationDefinitionRead createCustomDestinationDefinition(final CustomDestinationDefinitionCreate customDestinationDefinitionCreate) { + return ConfigurationApi.execute(() -> destinationDefinitionsHandler.createCustomDestinationDefinition(customDestinationDefinitionCreate)); + } + + // TODO: Deprecate this route in favor of createCustomDestinationDefinition + // since all connector definitions created through the API are custom + @Override + public DestinationDefinitionRead createDestinationDefinition(final DestinationDefinitionCreate destinationDefinitionCreate) { + return ConfigurationApi.execute(() -> destinationDefinitionsHandler.createPrivateDestinationDefinition(destinationDefinitionCreate)); + } + + @Override + public void deleteCustomDestinationDefinition(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { + ConfigurationApi.execute(() -> { + destinationDefinitionsHandler.deleteCustomDestinationDefinition(destinationDefinitionIdWithWorkspaceId); + return null; + }); + } + + @Override + public void deleteDestinationDefinition(final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) { + ConfigurationApi.execute(() -> { + destinationDefinitionsHandler.deleteDestinationDefinition(destinationDefinitionIdRequestBody); + return null; + }); + } + + @Override + public DestinationDefinitionRead getDestinationDefinition(final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) { + return ConfigurationApi.execute(() -> destinationDefinitionsHandler.getDestinationDefinition(destinationDefinitionIdRequestBody)); + } + + @Override + public DestinationDefinitionRead getDestinationDefinitionForWorkspace(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { + return ConfigurationApi.execute(() -> destinationDefinitionsHandler.getDestinationDefinitionForWorkspace(destinationDefinitionIdWithWorkspaceId)); + } + + @Override + public PrivateDestinationDefinitionRead grantDestinationDefinitionToWorkspace(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { + return ConfigurationApi + .execute(() -> destinationDefinitionsHandler.grantDestinationDefinitionToWorkspace(destinationDefinitionIdWithWorkspaceId)); + } + + @Override + public DestinationDefinitionReadList listDestinationDefinitions() { + return ConfigurationApi.execute(destinationDefinitionsHandler::listDestinationDefinitions); + } + + @Override + public DestinationDefinitionReadList listDestinationDefinitionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { + return ConfigurationApi.execute(() -> destinationDefinitionsHandler.listDestinationDefinitionsForWorkspace(workspaceIdRequestBody)); + } + + @Override + public DestinationDefinitionReadList listLatestDestinationDefinitions() { + return ConfigurationApi.execute(destinationDefinitionsHandler::listLatestDestinationDefinitions); + } + + @Override + public PrivateDestinationDefinitionReadList listPrivateDestinationDefinitions(final WorkspaceIdRequestBody workspaceIdRequestBody) { + return ConfigurationApi.execute(() -> destinationDefinitionsHandler.listPrivateDestinationDefinitions(workspaceIdRequestBody)); + } + + @Override + public void revokeDestinationDefinitionFromWorkspace(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { + ConfigurationApi.execute(() -> { + destinationDefinitionsHandler.revokeDestinationDefinitionFromWorkspace(destinationDefinitionIdWithWorkspaceId); + return null; + }); + } + + @Override + public DestinationDefinitionRead updateCustomDestinationDefinition(final CustomDestinationDefinitionUpdate customDestinationDefinitionUpdate) { + return ConfigurationApi.execute(() -> destinationDefinitionsHandler.updateCustomDestinationDefinition(customDestinationDefinitionUpdate)); + } + + @Override + public DestinationDefinitionRead updateDestinationDefinition(final DestinationDefinitionUpdate destinationDefinitionUpdate) { + return ConfigurationApi.execute(() -> destinationDefinitionsHandler.updateDestinationDefinition(destinationDefinitionUpdate)); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java new file mode 100644 index 000000000000..184059356746 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis; + +import io.airbyte.api.generated.DestinationDefinitionSpecificationApi; +import io.airbyte.api.model.generated.DestinationDefinitionIdWithWorkspaceId; +import io.airbyte.api.model.generated.DestinationDefinitionSpecificationRead; +import io.airbyte.server.handlers.SchedulerHandler; +import javax.ws.rs.Path; +import lombok.AllArgsConstructor; + +@Path("/v1/destination_definition_specifications/get") +@AllArgsConstructor +public class DestinationDefinitionSpecificationApiController implements DestinationDefinitionSpecificationApi { + + private final SchedulerHandler schedulerHandler; + + @Override + public DestinationDefinitionSpecificationRead getDestinationDefinitionSpecification(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { + return ConfigurationApi.execute(() -> schedulerHandler.getDestinationSpecification(destinationDefinitionIdWithWorkspaceId)); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java new file mode 100644 index 000000000000..75c10ca4852b --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis; + +import io.airbyte.api.generated.HealthApi; +import io.airbyte.api.model.generated.HealthCheckRead; +import io.airbyte.server.handlers.HealthCheckHandler; +import javax.ws.rs.Path; +import lombok.AllArgsConstructor; + +@Path("/v1/health") +@AllArgsConstructor +public class HealthApiController implements HealthApi { + + private final HealthCheckHandler healthCheckHandler; + + @Override + public HealthCheckRead getHealthCheck() { + return healthCheckHandler.health(); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/AttemptApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/AttemptApiBinder.java new file mode 100644 index 000000000000..2eb09dddaf02 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/AttemptApiBinder.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.binders; + +import io.airbyte.server.apis.AttemptApiController; +import io.airbyte.server.apis.factories.AttemptApiFactory; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.process.internal.RequestScoped; + +public class AttemptApiBinder extends AbstractBinder { + + @Override + protected void configure() { + bindFactory(AttemptApiFactory.class) + .to(AttemptApiController.class) + .in(RequestScoped.class); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/ConnectionApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/ConnectionApiBinder.java new file mode 100644 index 000000000000..d95fe3385514 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/ConnectionApiBinder.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.binders; + +import io.airbyte.server.apis.ConnectionApiController; +import io.airbyte.server.apis.factories.ConnectionApiFactory; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.process.internal.RequestScoped; + +public class ConnectionApiBinder extends AbstractBinder { + + @Override + protected void configure() { + bindFactory(ConnectionApiFactory.class) + .to(ConnectionApiController.class) + .in(RequestScoped.class); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DbMigrationBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DbMigrationBinder.java new file mode 100644 index 000000000000..78471be218d4 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DbMigrationBinder.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.binders; + +import io.airbyte.server.apis.DbMigrationApiController; +import io.airbyte.server.apis.factories.DbMigrationApiFactory; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.process.internal.RequestScoped; + +public class DbMigrationBinder extends AbstractBinder { + + @Override + protected void configure() { + bindFactory(DbMigrationApiFactory.class) + .to(DbMigrationApiController.class) + .in(RequestScoped.class); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationApiBinder.java new file mode 100644 index 000000000000..a1ae3cd38e8a --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationApiBinder.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.binders; + +import io.airbyte.server.apis.DestinationApiController; +import io.airbyte.server.apis.factories.DestinationApiFactory; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.process.internal.RequestScoped; + +public class DestinationApiBinder extends AbstractBinder { + + @Override + protected void configure() { + bindFactory(DestinationApiFactory.class) + .to(DestinationApiController.class) + .in(RequestScoped.class); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionApiBinder.java new file mode 100644 index 000000000000..1d9279119796 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionApiBinder.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.binders; + +import io.airbyte.server.apis.DestinationDefinitionApiController; +import io.airbyte.server.apis.factories.DestinationDefinitionApiFactory; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.process.internal.RequestScoped; + +public class DestinationDefinitionApiBinder extends AbstractBinder { + + @Override + protected void configure() { + bindFactory(DestinationDefinitionApiFactory.class) + .to(DestinationDefinitionApiController.class) + .in(RequestScoped.class); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionSpecificationApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionSpecificationApiBinder.java new file mode 100644 index 000000000000..3d166bd139af --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/DestinationDefinitionSpecificationApiBinder.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.binders; + +import io.airbyte.server.apis.DestinationDefinitionSpecificationApiController; +import io.airbyte.server.apis.factories.DestinationDefinitionSpecificationApiFactory; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.process.internal.RequestScoped; + +public class DestinationDefinitionSpecificationApiBinder extends AbstractBinder { + + @Override + protected void configure() { + bindFactory(DestinationDefinitionSpecificationApiFactory.class) + .to(DestinationDefinitionSpecificationApiController.class) + .in(RequestScoped.class); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/binders/HealthApiBinder.java b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/HealthApiBinder.java new file mode 100644 index 000000000000..bfe6161529f8 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/binders/HealthApiBinder.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.binders; + +import io.airbyte.server.apis.HealthApiController; +import io.airbyte.server.apis.factories.HealthApiFactory; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.process.internal.RequestScoped; + +public class HealthApiBinder extends AbstractBinder { + + @Override + protected void configure() { + bindFactory(HealthApiFactory.class) + .to(HealthApiController.class) + .in(RequestScoped.class); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/AttemptApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/AttemptApiFactory.java new file mode 100644 index 000000000000..27fb62696b41 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/AttemptApiFactory.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.factories; + +import io.airbyte.server.apis.AttemptApiController; +import io.airbyte.server.handlers.AttemptHandler; +import java.util.Map; +import org.glassfish.hk2.api.Factory; +import org.slf4j.MDC; + +public class AttemptApiFactory implements Factory { + + private static AttemptHandler attemptHandler; + private static Map mdc; + + public static void setValues(final AttemptHandler attemptHandler, final Map mdc) { + AttemptApiFactory.attemptHandler = attemptHandler; + AttemptApiFactory.mdc = mdc; + } + + @Override + public AttemptApiController provide() { + MDC.setContextMap(AttemptApiFactory.mdc); + + return new AttemptApiController(attemptHandler); + } + + @Override + public void dispose(final AttemptApiController instance) { + /* no op */ + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/ConnectionApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/ConnectionApiFactory.java new file mode 100644 index 000000000000..7378d342b65f --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/ConnectionApiFactory.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.factories; + +import io.airbyte.server.apis.ConnectionApiController; +import io.airbyte.server.handlers.ConnectionsHandler; +import io.airbyte.server.handlers.OperationsHandler; +import io.airbyte.server.handlers.SchedulerHandler; +import java.util.Map; +import org.glassfish.hk2.api.Factory; +import org.slf4j.MDC; + +public class ConnectionApiFactory implements Factory { + + private static ConnectionsHandler connectionsHandler; + private static OperationsHandler operationsHandler; + private static SchedulerHandler schedulerHandler; + private static Map mdc; + + public static void setValues(final ConnectionsHandler connectionsHandler, + final OperationsHandler operationsHandler, + final SchedulerHandler schedulerHandler, + final Map mdc) { + ConnectionApiFactory.connectionsHandler = connectionsHandler; + ConnectionApiFactory.operationsHandler = operationsHandler; + ConnectionApiFactory.schedulerHandler = schedulerHandler; + ConnectionApiFactory.mdc = mdc; + } + + @Override + public ConnectionApiController provide() { + MDC.setContextMap(ConnectionApiFactory.mdc); + + return new ConnectionApiController(connectionsHandler, operationsHandler, schedulerHandler); + } + + @Override + public void dispose(final ConnectionApiController instance) { + /* no op */ + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DbMigrationApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DbMigrationApiFactory.java new file mode 100644 index 000000000000..ee4430f48e94 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DbMigrationApiFactory.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.factories; + +import io.airbyte.server.apis.DbMigrationApiController; +import io.airbyte.server.handlers.DbMigrationHandler; +import java.util.Map; +import org.glassfish.hk2.api.Factory; +import org.slf4j.MDC; + +public class DbMigrationApiFactory implements Factory { + + private static DbMigrationHandler dbMigrationHandler; + private static Map mdc; + + public static void setValues(final DbMigrationHandler dbMigrationHandler, final Map mdc) { + DbMigrationApiFactory.dbMigrationHandler = dbMigrationHandler; + DbMigrationApiFactory.mdc = mdc; + } + + @Override + public DbMigrationApiController provide() { + MDC.setContextMap(DbMigrationApiFactory.mdc); + + return new DbMigrationApiController(dbMigrationHandler); + } + + @Override + public void dispose(final DbMigrationApiController instance) { + /* no op */ + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationApiFactory.java new file mode 100644 index 000000000000..2dcd6aa62714 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationApiFactory.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.factories; + +import io.airbyte.server.apis.DestinationApiController; +import io.airbyte.server.handlers.DestinationHandler; +import io.airbyte.server.handlers.SchedulerHandler; +import java.util.Map; +import org.glassfish.hk2.api.Factory; +import org.slf4j.MDC; + +public class DestinationApiFactory implements Factory { + + private static DestinationHandler destinationHandler; + private static SchedulerHandler schedulerHandler; + private static Map mdc; + + public static void setValues(final DestinationHandler destinationHandler, + final SchedulerHandler schedulerHandler, + final Map mdc) { + DestinationApiFactory.destinationHandler = destinationHandler; + DestinationApiFactory.schedulerHandler = schedulerHandler; + DestinationApiFactory.mdc = mdc; + } + + @Override + public DestinationApiController provide() { + MDC.setContextMap(DestinationApiFactory.mdc); + + return new DestinationApiController(destinationHandler, schedulerHandler); + } + + @Override + public void dispose(final DestinationApiController instance) { + /* no op */ + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionApiFactory.java new file mode 100644 index 000000000000..f76f631145ce --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionApiFactory.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.factories; + +import io.airbyte.server.apis.DestinationDefinitionApiController; +import io.airbyte.server.handlers.DestinationDefinitionsHandler; +import org.glassfish.hk2.api.Factory; + +public class DestinationDefinitionApiFactory implements Factory { + + private static DestinationDefinitionsHandler destinationDefinitionsHandler; + + public static void setValues(final DestinationDefinitionsHandler destinationDefinitionsHandler) { + DestinationDefinitionApiFactory.destinationDefinitionsHandler = destinationDefinitionsHandler; + } + + @Override + public DestinationDefinitionApiController provide() { + return new DestinationDefinitionApiController(destinationDefinitionsHandler); + } + + @Override + public void dispose(final DestinationDefinitionApiController instance) { + /* no op */ + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionSpecificationApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionSpecificationApiFactory.java new file mode 100644 index 000000000000..d7dcbfac229d --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/DestinationDefinitionSpecificationApiFactory.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.factories; + +import io.airbyte.server.apis.DestinationDefinitionSpecificationApiController; +import io.airbyte.server.handlers.SchedulerHandler; +import org.glassfish.hk2.api.Factory; + +public class DestinationDefinitionSpecificationApiFactory implements Factory { + + private static SchedulerHandler schedulerHandler; + + public static void setValues(final SchedulerHandler schedulerHandler) { + DestinationDefinitionSpecificationApiFactory.schedulerHandler = schedulerHandler; + } + + @Override + public DestinationDefinitionSpecificationApiController provide() { + return new DestinationDefinitionSpecificationApiController(DestinationDefinitionSpecificationApiFactory.schedulerHandler); + } + + @Override + public void dispose(final DestinationDefinitionSpecificationApiController instance) { + + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/factories/HealthApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/HealthApiFactory.java new file mode 100644 index 000000000000..b13c17d5ebb2 --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/factories/HealthApiFactory.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis.factories; + +import io.airbyte.server.apis.HealthApiController; +import io.airbyte.server.handlers.HealthCheckHandler; +import org.glassfish.hk2.api.Factory; + +public class HealthApiFactory implements Factory { + + private static HealthCheckHandler healthCheckHandler; + + public static void setValues(final HealthCheckHandler healthCheckHandler) { + HealthApiFactory.healthCheckHandler = healthCheckHandler; + } + + @Override + public HealthApiController provide() { + return new HealthApiController(healthCheckHandler); + } + + @Override + public void dispose(final HealthApiController instance) { + /* no op */ + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java b/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java index 614344230532..52c28f3640f1 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java +++ b/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java @@ -9,6 +9,7 @@ import io.airbyte.api.model.generated.AttemptFailureSummary; import io.airbyte.api.model.generated.AttemptFailureType; import io.airbyte.api.model.generated.AttemptInfoRead; +import io.airbyte.api.model.generated.AttemptNormalizationStatusRead; import io.airbyte.api.model.generated.AttemptRead; import io.airbyte.api.model.generated.AttemptStats; import io.airbyte.api.model.generated.AttemptStatus; @@ -38,6 +39,7 @@ import io.airbyte.config.helpers.LogClientSingleton; import io.airbyte.config.helpers.LogConfigs; import io.airbyte.persistence.job.models.Attempt; +import io.airbyte.persistence.job.models.AttemptNormalizationStatus; import io.airbyte.persistence.job.models.Job; import io.airbyte.server.scheduler.SynchronousJobMetadata; import io.airbyte.server.scheduler.SynchronousResponse; @@ -240,4 +242,13 @@ public SynchronousJobRead getSynchronousJobRead(final SynchronousJobMetadata met .logs(getLogRead(metadata.getLogPath())); } + public static AttemptNormalizationStatusRead convertAttemptNormalizationStatus( + AttemptNormalizationStatus databaseStatus) { + return new AttemptNormalizationStatusRead() + .attemptNumber(databaseStatus.attemptNumber()) + .hasRecordsCommitted(!databaseStatus.recordsCommitted().isEmpty()) + .recordsCommitted(databaseStatus.recordsCommitted().orElse(0L)) + .hasNormalizationFailed(databaseStatus.normalizationFailed()); + } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java index 3e79d8bcb33b..df0850e2c97d 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java @@ -339,6 +339,10 @@ private static void applyPatchToStandardSync(final StandardSync sync, final Conn if (patch.getResourceRequirements() != null) { sync.setResourceRequirements(ApiPojoConverters.resourceRequirementsToInternal(patch.getResourceRequirements())); } + + if (patch.getGeography() != null) { + sync.setGeography(ApiPojoConverters.toPersistenceGeography(patch.getGeography())); + } } private void validateConnectionPatch(final WorkspaceHelper workspaceHelper, final StandardSync persistedSync, final ConnectionUpdate patch) { diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationHandler.java index 27ec317b83c5..a828448c77a9 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationHandler.java @@ -100,7 +100,7 @@ public DestinationRead createDestination(final DestinationCreate destinationCrea false); // read configuration from db - return buildDestinationRead(destinationId, spec); + return buildDestinationRead(configRepository.getDestinationConnection(destinationId), spec); } public void deleteDestination(final DestinationIdRequestBody destinationIdRequestBody) @@ -157,7 +157,8 @@ public DestinationRead updateDestination(final DestinationUpdate destinationUpda updatedDestination.getTombstone()); // read configuration from db - return buildDestinationRead(destinationUpdate.getDestinationId(), spec); + return buildDestinationRead( + configRepository.getDestinationConnection(destinationUpdate.getDestinationId()), spec); } public DestinationRead getDestination(final DestinationIdRequestBody destinationIdRequestBody) @@ -195,20 +196,11 @@ public DestinationRead cloneDestination(final DestinationCloneRequestBody destin public DestinationReadList listDestinationsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) throws ConfigNotFoundException, IOException, JsonValidationException { - final List reads = Lists.newArrayList(); - - for (final DestinationConnection dci : configRepository.listDestinationConnection()) { - if (!dci.getWorkspaceId().equals(workspaceIdRequestBody.getWorkspaceId())) { - continue; - } - if (dci.getTombstone()) { - continue; - } - - reads.add(buildDestinationRead(dci.getDestinationId())); + final List reads = Lists.newArrayList(); + for (final DestinationConnection dci : configRepository.listWorkspaceDestinationConnection(workspaceIdRequestBody.getWorkspaceId())) { + reads.add(buildDestinationRead(dci)); } - return new DestinationReadList().destinations(reads); } @@ -216,15 +208,9 @@ public DestinationReadList listDestinationsForDestinationDefinition(final Destin throws JsonValidationException, IOException, ConfigNotFoundException { final List reads = Lists.newArrayList(); - for (final DestinationConnection destinationConnection : configRepository.listDestinationConnection()) { - if (!destinationConnection.getDestinationDefinitionId().equals(destinationDefinitionIdRequestBody.getDestinationDefinitionId())) { - continue; - } - if (destinationConnection.getTombstone() != null && destinationConnection.getTombstone()) { - continue; - } - - reads.add(buildDestinationRead(destinationConnection.getDestinationId())); + for (final DestinationConnection destinationConnection : configRepository + .listDestinationsForDefinition(destinationDefinitionIdRequestBody.getDestinationDefinitionId())) { + reads.add(buildDestinationRead(destinationConnection)); } return new DestinationReadList().destinations(reads); @@ -236,7 +222,7 @@ public DestinationReadList searchDestinations(final DestinationSearch destinatio for (final DestinationConnection dci : configRepository.listDestinationConnection()) { if (!dci.getTombstone()) { - final DestinationRead destinationRead = buildDestinationRead(dci.getDestinationId()); + final DestinationRead destinationRead = buildDestinationRead(dci); if (connectionsHandler.matchSearch(destinationSearch, destinationRead)) { reads.add(destinationRead); } @@ -273,15 +259,20 @@ private void persistDestinationConnection(final String name, } private DestinationRead buildDestinationRead(final UUID destinationId) throws JsonValidationException, IOException, ConfigNotFoundException { - final ConnectorSpecification spec = getSpec(configRepository.getDestinationConnection(destinationId).getDestinationDefinitionId()); - return buildDestinationRead(destinationId, spec); + return buildDestinationRead(configRepository.getDestinationConnection(destinationId)); + } + + private DestinationRead buildDestinationRead(final DestinationConnection destinationConnection) + throws JsonValidationException, IOException, ConfigNotFoundException { + final ConnectorSpecification spec = getSpec(destinationConnection.getDestinationDefinitionId()); + return buildDestinationRead(destinationConnection, spec); } - private DestinationRead buildDestinationRead(final UUID destinationId, final ConnectorSpecification spec) + private DestinationRead buildDestinationRead(final DestinationConnection destinationConnection, final ConnectorSpecification spec) throws ConfigNotFoundException, IOException, JsonValidationException { // remove secrets from config before returning the read - final DestinationConnection dci = Jsons.clone(configRepository.getDestinationConnection(destinationId)); + final DestinationConnection dci = Jsons.clone(destinationConnection); dci.setConfiguration(secretsProcessor.prepareSecretsForOutput(dci.getConfiguration(), spec.getConnectionSpecification())); final StandardDestinationDefinition standardDestinationDefinition = diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/JobHistoryHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/JobHistoryHandler.java index bb4f7bbb551f..e25bee37f04b 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/JobHistoryHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/JobHistoryHandler.java @@ -5,6 +5,7 @@ package io.airbyte.server.handlers; import com.google.common.base.Preconditions; +import io.airbyte.api.model.generated.AttemptNormalizationStatusReadList; import io.airbyte.api.model.generated.ConnectionRead; import io.airbyte.api.model.generated.DestinationDefinitionIdRequestBody; import io.airbyte.api.model.generated.DestinationDefinitionRead; @@ -146,6 +147,12 @@ public List getLatestSyncJobsForConnections(final List connection .collect(Collectors.toList()); } + public AttemptNormalizationStatusReadList getAttemptNormalizationStatuses(final JobIdRequestBody jobIdRequestBody) throws IOException { + return new AttemptNormalizationStatusReadList() + .attemptNormalizationStatuses(jobPersistence.getAttemptNormalizationStatusesForJob(jobIdRequestBody.getId()).stream() + .map(JobConverter::convertAttemptNormalizationStatus).collect(Collectors.toList())); + } + public List getRunningSyncJobForConnections(final List connectionIds) throws IOException { return jobPersistence.getRunningSyncJobForConnections(connectionIds).stream() .map(JobConverter::getJobRead) diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceHandler.java index 743c4b9e4f74..5751d271900c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceHandler.java @@ -17,7 +17,6 @@ import io.airbyte.api.model.generated.SourceSearch; import io.airbyte.api.model.generated.SourceUpdate; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; -import io.airbyte.commons.lang.MoreBooleans; import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.persistence.ConfigNotFoundException; @@ -100,16 +99,17 @@ public SourceRead createSource(final SourceCreate sourceCreate) spec); // read configuration from db - return buildSourceRead(sourceId, spec); + return buildSourceRead(configRepository.getSourceConnection(sourceId), spec); } public SourceRead updateSource(final SourceUpdate sourceUpdate) throws ConfigNotFoundException, IOException, JsonValidationException { + final UUID sourceId = sourceUpdate.getSourceId(); final SourceConnection updatedSource = configurationUpdate - .source(sourceUpdate.getSourceId(), sourceUpdate.getName(), + .source(sourceId, sourceUpdate.getName(), sourceUpdate.getConnectionConfiguration()); - final ConnectorSpecification spec = getSpecFromSourceId(updatedSource.getSourceId()); + final ConnectorSpecification spec = getSpecFromSourceId(sourceId); validateSource(spec, sourceUpdate.getConnectionConfiguration()); // persist @@ -123,7 +123,7 @@ public SourceRead updateSource(final SourceUpdate sourceUpdate) spec); // read configuration from db - return buildSourceRead(sourceUpdate.getSourceId(), spec); + return buildSourceRead(configRepository.getSourceConnection(sourceId), spec); } public SourceRead getSource(final SourceIdRequestBody sourceIdRequestBody) @@ -166,7 +166,7 @@ public SourceReadList listSourcesForWorkspace(final WorkspaceIdRequestBody works final List reads = Lists.newArrayList(); for (final SourceConnection sc : sourceConnections) { - reads.add(buildSourceRead(sc.getSourceId())); + reads.add(buildSourceRead(sc)); } return new SourceReadList().sources(reads); @@ -175,15 +175,9 @@ public SourceReadList listSourcesForWorkspace(final WorkspaceIdRequestBody works public SourceReadList listSourcesForSourceDefinition(final SourceDefinitionIdRequestBody sourceDefinitionIdRequestBody) throws JsonValidationException, IOException, ConfigNotFoundException { - final List sourceConnections = configRepository.listSourceConnection() - .stream() - .filter(sc -> sc.getSourceDefinitionId().equals(sourceDefinitionIdRequestBody.getSourceDefinitionId()) - && !MoreBooleans.isTruthy(sc.getTombstone())) - .toList(); - final List reads = Lists.newArrayList(); - for (final SourceConnection sourceConnection : sourceConnections) { - reads.add(buildSourceRead(sourceConnection.getSourceId())); + for (final SourceConnection sourceConnection : configRepository.listSourcesForDefinition(sourceDefinitionIdRequestBody.getSourceDefinitionId())) { + reads.add(buildSourceRead(sourceConnection)); } return new SourceReadList().sources(reads); @@ -195,7 +189,7 @@ public SourceReadList searchSources(final SourceSearch sourceSearch) for (final SourceConnection sci : configRepository.listSourceConnection()) { if (!sci.getTombstone()) { - final SourceRead sourceRead = buildSourceRead(sci.getSourceId()); + final SourceRead sourceRead = buildSourceRead(sci); if (connectionsHandler.matchSearch(sourceSearch, sourceRead)) { reads.add(sourceRead); } @@ -242,15 +236,20 @@ public void deleteSource(final SourceRead source) private SourceRead buildSourceRead(final UUID sourceId) throws ConfigNotFoundException, IOException, JsonValidationException { // read configuration from db - final StandardSourceDefinition sourceDef = configRepository.getSourceDefinitionFromSource(sourceId); + final SourceConnection sourceConnection = configRepository.getSourceConnection(sourceId); + return buildSourceRead(sourceConnection); + } + + private SourceRead buildSourceRead(final SourceConnection sourceConnection) + throws ConfigNotFoundException, IOException, JsonValidationException { + final StandardSourceDefinition sourceDef = configRepository.getSourceDefinitionFromSource(sourceConnection.getSourceId()); final ConnectorSpecification spec = sourceDef.getSpec(); - return buildSourceRead(sourceId, spec); + return buildSourceRead(sourceConnection, spec); } - private SourceRead buildSourceRead(final UUID sourceId, final ConnectorSpecification spec) + private SourceRead buildSourceRead(final SourceConnection sourceConnection, final ConnectorSpecification spec) throws ConfigNotFoundException, IOException, JsonValidationException { // read configuration from db - final SourceConnection sourceConnection = configRepository.getSourceConnection(sourceId); final StandardSourceDefinition standardSourceDefinition = configRepository .getStandardSourceDefinition(sourceConnection.getSourceDefinitionId()); final JsonNode sanitizedConfig = secretsProcessor.prepareSecretsForOutput(sourceConnection.getConfiguration(), spec.getConnectionSpecification()); diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java index 3743d8382bbb..f6658c94be20 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java @@ -30,6 +30,7 @@ import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.SecretsRepositoryWriter; import io.airbyte.notification.NotificationClient; +import io.airbyte.server.converters.ApiPojoConverters; import io.airbyte.server.converters.NotificationConverter; import io.airbyte.server.converters.WorkspaceWebhookConfigsConverter; import io.airbyte.server.errors.IdNotFoundKnownException; @@ -301,8 +302,7 @@ private void applyPatchToStandardWorkspace(final StandardWorkspace workspace, fi workspace.setNotifications(NotificationConverter.toConfigList(workspacePatch.getNotifications())); } if (workspacePatch.getDefaultGeography() != null) { - workspace.setDefaultGeography( - Enums.convertTo(workspacePatch.getDefaultGeography(), io.airbyte.config.Geography.class)); + workspace.setDefaultGeography(ApiPojoConverters.toPersistenceGeography(workspacePatch.getDefaultGeography())); } if (workspacePatch.getWebhookConfigs() != null) { workspace.setWebhookOperationConfigs(WorkspaceWebhookConfigsConverter.toPersistenceWrite(workspacePatch.getWebhookConfigs(), uuidSupplier)); diff --git a/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java b/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java deleted file mode 100644 index 0bba057a25aa..000000000000 --- a/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.server.apis; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import io.airbyte.analytics.TrackingClient; -import io.airbyte.commons.version.AirbyteVersion; -import io.airbyte.config.Configs; -import io.airbyte.config.Configs.WorkerEnvironment; -import io.airbyte.config.helpers.LogConfigs; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.config.persistence.SecretsRepositoryReader; -import io.airbyte.config.persistence.SecretsRepositoryWriter; -import io.airbyte.config.persistence.StatePersistence; -import io.airbyte.db.Database; -import io.airbyte.persistence.job.JobPersistence; -import io.airbyte.server.scheduler.EventRunner; -import io.airbyte.server.scheduler.SynchronousSchedulerClient; -import java.net.http.HttpClient; -import java.nio.file.Path; -import org.flywaydb.core.Flyway; -import org.junit.jupiter.api.Test; - -class ConfigurationApiTest { - - @Test - void testImportDefinitions() { - final Configs configs = mock(Configs.class); - when(configs.getAirbyteVersion()).thenReturn(new AirbyteVersion("0.1.0-alpha")); - when(configs.getWebappUrl()).thenReturn("http://localhost"); - - final ConfigurationApi configurationApi = new ConfigurationApi( - mock(ConfigRepository.class), - mock(JobPersistence.class), - mock(SecretsRepositoryReader.class), - mock(SecretsRepositoryWriter.class), - mock(SynchronousSchedulerClient.class), - mock(Database.class), - mock(Database.class), - mock(StatePersistence.class), - mock(TrackingClient.class), - WorkerEnvironment.DOCKER, - LogConfigs.EMPTY, - new AirbyteVersion("0.1.0-alpha"), - Path.of(""), - mock(HttpClient.class), - mock(EventRunner.class), - mock(Flyway.class), - mock(Flyway.class)); - - assertFalse(configurationApi.getHealthCheck().getAvailable()); - } - -} diff --git a/airbyte-server/src/test/java/io/airbyte/server/apis/HealthCheckApiTest.java b/airbyte-server/src/test/java/io/airbyte/server/apis/HealthCheckApiTest.java new file mode 100644 index 000000000000..cc4f5d814e35 --- /dev/null +++ b/airbyte-server/src/test/java/io/airbyte/server/apis/HealthCheckApiTest.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.server.apis; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.airbyte.api.model.generated.HealthCheckRead; +import io.airbyte.server.handlers.HealthCheckHandler; +import org.junit.jupiter.api.Test; + +class HealthCheckApiTest { + + @Test + void testImportDefinitions() { + final HealthCheckHandler healthCheckHandler = mock(HealthCheckHandler.class); + when(healthCheckHandler.health()) + .thenReturn(new HealthCheckRead().available( + false)); + + final HealthApiController configurationApi = new HealthApiController(healthCheckHandler); + + assertFalse(configurationApi.getHealthCheck().getAvailable()); + } + +} diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java index 778cc821ed49..08086c7e3bae 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java @@ -586,7 +586,8 @@ void testUpdateConnectionPatchingSeveralFieldsAndReplaceAStream() throws JsonVal .syncCatalog(catalogForUpdate) .resourceRequirements(resourceRequirements) .sourceCatalogId(newSourceCatalogId) - .operationIds(List.of(operationId, otherOperationId)); + .operationIds(List.of(operationId, otherOperationId)) + .geography(io.airbyte.api.model.generated.Geography.EU); final ConfiguredAirbyteCatalog expectedPersistedCatalog = ConnectionHelpers.generateBasicConfiguredAirbyteCatalog(); expectedPersistedCatalog.getStreams().get(0).getStream().withName(AZKABAN_USERS); @@ -600,7 +601,8 @@ void testUpdateConnectionPatchingSeveralFieldsAndReplaceAStream() throws JsonVal .withCatalog(expectedPersistedCatalog) .withResourceRequirements(ApiPojoConverters.resourceRequirementsToInternal(resourceRequirements)) .withSourceCatalogId(newSourceCatalogId) - .withOperationIds(List.of(operationId, otherOperationId)); + .withOperationIds(List.of(operationId, otherOperationId)) + .withGeography(Geography.EU); when(configRepository.getStandardSync(standardSync.getConnectionId())).thenReturn(standardSync); diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationHandlerTest.java index 6dc096c862e1..28fb3868bb2b 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationHandlerTest.java @@ -230,7 +230,8 @@ void testListDestinationForWorkspace() throws JsonValidationException, ConfigNot final WorkspaceIdRequestBody workspaceIdRequestBody = new WorkspaceIdRequestBody().workspaceId(destinationConnection.getWorkspaceId()); when(configRepository.getDestinationConnection(destinationConnection.getDestinationId())).thenReturn(destinationConnection); - when(configRepository.listDestinationConnection()).thenReturn(Lists.newArrayList(destinationConnection)); + when(configRepository.listWorkspaceDestinationConnection(destinationConnection.getWorkspaceId())) + .thenReturn(Lists.newArrayList(destinationConnection)); when(configRepository.getStandardDestinationDefinition(standardDestinationDefinition.getDestinationDefinitionId())) .thenReturn(standardDestinationDefinition); when(secretsProcessor.prepareSecretsForOutput(destinationConnection.getConfiguration(), diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/JobHistoryHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/JobHistoryHandlerTest.java index 3d4f6e942401..d53044e95596 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/JobHistoryHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/JobHistoryHandlerTest.java @@ -20,6 +20,7 @@ import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.persistence.job.JobPersistence; import io.airbyte.persistence.job.models.Attempt; +import io.airbyte.persistence.job.models.AttemptNormalizationStatus; import io.airbyte.persistence.job.models.AttemptStatus; import io.airbyte.persistence.job.models.Job; import io.airbyte.persistence.job.models.JobStatus; @@ -376,4 +377,19 @@ void testEnumConversion() { assertTrue(Enums.isCompatible(JobConfig.ConfigType.class, JobConfigType.class)); } + @Test + @DisplayName("Should return attempt normalization info for the job") + void testGetAttemptNormalizationStatuses() throws IOException { + + AttemptNormalizationStatus databaseReadResult = new AttemptNormalizationStatus(1, Optional.of(10L), /* hasNormalizationFailed= */ false); + + when(jobPersistence.getAttemptNormalizationStatusesForJob(JOB_ID)).thenReturn(List.of(databaseReadResult)); + + AttemptNormalizationStatusReadList expectedStatus = new AttemptNormalizationStatusReadList().attemptNormalizationStatuses( + List.of(new AttemptNormalizationStatusRead().attemptNumber(1).hasRecordsCommitted(true).hasNormalizationFailed(false).recordsCommitted(10L))); + + assertEquals(expectedStatus, jobHistoryHandler.getAttemptNormalizationStatuses(new JobIdRequestBody().id(JOB_ID))); + + } + } diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceHandlerTest.java index 49b0651fbdb6..9da285912971 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceHandlerTest.java @@ -283,7 +283,7 @@ void testListSourcesForSourceDefinition() throws JsonValidationException, Config new SourceDefinitionIdRequestBody().sourceDefinitionId(sourceConnection.getSourceDefinitionId()); when(configRepository.getSourceConnection(sourceConnection.getSourceId())).thenReturn(sourceConnection); - when(configRepository.listSourceConnection()).thenReturn(Lists.newArrayList(sourceConnection)); + when(configRepository.listSourcesForDefinition(sourceConnection.getSourceDefinitionId())).thenReturn(Lists.newArrayList(sourceConnection)); when(configRepository.getStandardSourceDefinition(sourceDefinitionSpecificationRead.getSourceDefinitionId())) .thenReturn(standardSourceDefinition); when(configRepository.getSourceDefinitionFromSource(sourceConnection.getSourceId())).thenReturn(standardSourceDefinition); diff --git a/airbyte-test-utils/build.gradle b/airbyte-test-utils/build.gradle index 39a3aa7eb46a..aaa2980c6dca 100644 --- a/airbyte-test-utils/build.gradle +++ b/airbyte-test-utils/build.gradle @@ -14,7 +14,7 @@ dependencies { implementation project(':airbyte-commons-worker') implementation 'io.fabric8:kubernetes-client:5.12.2' - implementation 'io.temporal:temporal-sdk:1.8.1' + implementation libs.temporal.sdk api libs.junit.jupiter.api diff --git a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AirbyteAcceptanceTestHarness.java b/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AirbyteAcceptanceTestHarness.java index 761ef8f63a83..433f7e82ab4e 100644 --- a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AirbyteAcceptanceTestHarness.java +++ b/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AirbyteAcceptanceTestHarness.java @@ -797,11 +797,11 @@ public static JobRead waitWhileJobHasStatus(final JobsApi jobsApi, @SuppressWarnings("BusyWait") public static ConnectionState waitForConnectionState(final AirbyteApiClient apiClient, final UUID connectionId) throws ApiException, InterruptedException { - ConnectionState connectionState = apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId)); + ConnectionState connectionState = apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId)); int count = 0; while (count < 60 && (connectionState.getState() == null || connectionState.getState().isNull())) { LOGGER.info("fetching connection state. attempt: {}", count++); - connectionState = apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId)); + connectionState = apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId)); sleep(1000); } return connectionState; diff --git a/airbyte-tests/build.gradle b/airbyte-tests/build.gradle index da79354c3a49..038267ab977f 100644 --- a/airbyte-tests/build.gradle +++ b/airbyte-tests/build.gradle @@ -53,7 +53,7 @@ dependencies { acceptanceTestsImplementation 'com.fasterxml.jackson.core:jackson-databind' acceptanceTestsImplementation 'io.github.cdimascio:java-dotenv:3.0.0' - acceptanceTestsImplementation 'io.temporal:temporal-sdk:1.8.1' + acceptanceTestsImplementation libs.temporal.sdk acceptanceTestsImplementation 'org.apache.commons:commons-csv:1.4' acceptanceTestsImplementation libs.platform.testcontainers.postgresql acceptanceTestsImplementation libs.postgresql diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java index b1584e3f2c0b..4f9baa0bc91b 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java @@ -611,7 +611,7 @@ void testIncrementalSync() throws Exception { final JobInfoRead connectionSyncRead1 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead1.getJob()); - LOGGER.info(STATE_AFTER_SYNC_ONE, apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info(STATE_AFTER_SYNC_ONE, apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); testHarness.assertSourceAndDestinationDbInSync(WITHOUT_SCD_TABLE); @@ -631,7 +631,7 @@ void testIncrementalSync() throws Exception { final JobInfoRead connectionSyncRead2 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead2.getJob()); - LOGGER.info(STATE_AFTER_SYNC_TWO, apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info(STATE_AFTER_SYNC_TWO, apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); testHarness.assertRawDestinationContains(expectedRecords, new SchemaTableNamePair(PUBLIC, STREAM_NAME)); @@ -642,7 +642,7 @@ void testIncrementalSync() throws Exception { waitWhileJobHasStatus(apiClient.getJobsApi(), jobInfoRead.getJob(), Sets.newHashSet(JobStatus.PENDING, JobStatus.RUNNING, JobStatus.INCOMPLETE, JobStatus.FAILED)); - LOGGER.info("state after reset: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info("state after reset: {}", apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); testHarness.assertRawDestinationContains(Collections.emptyList(), new SchemaTableNamePair(PUBLIC, STREAM_NAME)); @@ -652,7 +652,7 @@ void testIncrementalSync() throws Exception { final JobInfoRead connectionSyncRead3 = apiClient.getConnectionApi().syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead3.getJob()); - LOGGER.info("state after sync 3: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info("state after sync 3: {}", apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); testHarness.assertSourceAndDestinationDbInSync(WITHOUT_SCD_TABLE); @@ -906,7 +906,7 @@ void testSyncAfterUpgradeToPerStreamState(final TestInfo testInfo) throws Except final JobInfoRead connectionSyncRead1 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead1.getJob()); - LOGGER.info(STATE_AFTER_SYNC_ONE, apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info(STATE_AFTER_SYNC_ONE, apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); testHarness.assertSourceAndDestinationDbInSync(WITHOUT_SCD_TABLE); @@ -930,7 +930,7 @@ void testSyncAfterUpgradeToPerStreamState(final TestInfo testInfo) throws Except final JobInfoRead connectionSyncRead2 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead2.getJob()); - LOGGER.info(STATE_AFTER_SYNC_TWO, apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info(STATE_AFTER_SYNC_TWO, apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); testHarness.assertRawDestinationContains(expectedRecords, new SchemaTableNamePair(PUBLIC, STREAM_NAME)); @@ -940,7 +940,7 @@ void testSyncAfterUpgradeToPerStreamState(final TestInfo testInfo) throws Except waitWhileJobHasStatus(apiClient.getJobsApi(), jobInfoRead.getJob(), Sets.newHashSet(JobStatus.PENDING, JobStatus.RUNNING, JobStatus.INCOMPLETE, JobStatus.FAILED)); - LOGGER.info("state after reset: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info("state after reset: {}", apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); testHarness.assertRawDestinationContains(Collections.emptyList(), new SchemaTableNamePair(PUBLIC, STREAM_NAME)); @@ -952,7 +952,7 @@ void testSyncAfterUpgradeToPerStreamState(final TestInfo testInfo) throws Except final JobInfoRead connectionSyncRead3 = apiClient.getConnectionApi().syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead3.getJob()); - final ConnectionState state = apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId)); + final ConnectionState state = apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId)); LOGGER.info("state after sync 3: {}", state); testHarness.assertSourceAndDestinationDbInSync(WITHOUT_SCD_TABLE); @@ -995,7 +995,7 @@ void testSyncAfterUpgradeToPerStreamStateWithNoNewData(final TestInfo testInfo) final JobInfoRead connectionSyncRead1 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead1.getJob()); - LOGGER.info(STATE_AFTER_SYNC_ONE, apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info(STATE_AFTER_SYNC_ONE, apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); testHarness.assertSourceAndDestinationDbInSync(WITHOUT_SCD_TABLE); @@ -1008,7 +1008,7 @@ void testSyncAfterUpgradeToPerStreamStateWithNoNewData(final TestInfo testInfo) final JobInfoRead connectionSyncRead2 = apiClient.getConnectionApi().syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead2.getJob()); - LOGGER.info(STATE_AFTER_SYNC_TWO, apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info(STATE_AFTER_SYNC_TWO, apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); final JobInfoRead syncJob = apiClient.getJobsApi().getJobInfo(new JobIdRequestBody().id(connectionSyncRead2.getJob().getId())); final Optional result = syncJob.getAttempts().stream() @@ -1086,7 +1086,7 @@ void testResetAllWhenSchemaIsModifiedForLegacySource() throws Exception { return null; }); final ConnectionState initSyncState = - apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connection.getConnectionId())); + apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connection.getConnectionId())); LOGGER.info("ConnectionState after the initial sync: " + initSyncState.toString()); testHarness.assertSourceAndDestinationDbInSync(false); @@ -1127,7 +1127,7 @@ void testResetAllWhenSchemaIsModifiedForLegacySource() throws Exception { return null; }); final ConnectionState postResetState = - apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connection.getConnectionId())); + apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connection.getConnectionId())); LOGGER.info("ConnectionState after the update request: {}", postResetState.toString()); // Wait until the sync from the UpdateConnection is finished @@ -1136,7 +1136,7 @@ void testResetAllWhenSchemaIsModifiedForLegacySource() throws Exception { waitForSuccessfulJob(apiClient.getJobsApi(), syncFromTheUpdate); final ConnectionState postUpdateState = - apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connection.getConnectionId())); + apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connection.getConnectionId())); LOGGER.info("ConnectionState after the final sync: {}", postUpdateState.toString()); LOGGER.info("Inspecting DBs After the final sync"); @@ -1202,7 +1202,7 @@ void testIncrementalSyncMultipleStreams() throws Exception { final JobInfoRead connectionSyncRead1 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead1.getJob()); - LOGGER.info(STATE_AFTER_SYNC_ONE, apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info(STATE_AFTER_SYNC_ONE, apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); testHarness.assertSourceAndDestinationDbInSync(WITHOUT_SCD_TABLE); @@ -1232,7 +1232,7 @@ void testIncrementalSyncMultipleStreams() throws Exception { final JobInfoRead connectionSyncRead2 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead2.getJob()); - LOGGER.info(STATE_AFTER_SYNC_TWO, apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info(STATE_AFTER_SYNC_TWO, apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); testHarness.assertRawDestinationContains(expectedRecordsIdAndName, new SchemaTableNamePair(PUBLIC_SCHEMA_NAME, STREAM_NAME)); testHarness.assertRawDestinationContains(expectedRecordsCoolEmployees, new SchemaTableNamePair(STAGING_SCHEMA_NAME, COOL_EMPLOYEES_TABLE_NAME)); @@ -1245,7 +1245,7 @@ void testIncrementalSyncMultipleStreams() throws Exception { waitWhileJobHasStatus(apiClient.getJobsApi(), jobInfoRead.getJob(), Sets.newHashSet(JobStatus.PENDING, JobStatus.RUNNING, JobStatus.INCOMPLETE, JobStatus.FAILED)); - LOGGER.info("state after reset: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info("state after reset: {}", apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); testHarness.assertRawDestinationContains(Collections.emptyList(), new SchemaTableNamePair(PUBLIC, STREAM_NAME)); @@ -1255,7 +1255,7 @@ void testIncrementalSyncMultipleStreams() throws Exception { final JobInfoRead connectionSyncRead3 = apiClient.getConnectionApi().syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead3.getJob()); - LOGGER.info("state after sync 3: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info("state after sync 3: {}", apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); testHarness.assertSourceAndDestinationDbInSync(WITHOUT_SCD_TABLE); @@ -1407,7 +1407,7 @@ void testPartialResetResetAllWhenSchemaIsModified(final TestInfo testInfo) throw } private void assertStreamStateContainsStream(final UUID connectionId, final List expectedStreamDescriptors) throws ApiException { - final ConnectionState state = apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId)); + final ConnectionState state = apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId)); final List streamDescriptors = state.getStreamState().stream().map(StreamState::getStreamDescriptor).toList(); Assertions.assertTrue(streamDescriptors.containsAll(expectedStreamDescriptors) && expectedStreamDescriptors.containsAll(streamDescriptors)); @@ -1441,14 +1441,14 @@ private JobRead waitUntilTheNextJobIsStarted(final UUID connectionId) throws Exc * @param maxRetries the number of times to retry * @throws InterruptedException */ - private void waitForSuccessfulJobWithRetries(final UUID connectionId, int maxRetries) throws InterruptedException { + private void waitForSuccessfulJobWithRetries(final UUID connectionId, final int maxRetries) throws InterruptedException { int i; for (i = 0; i < maxRetries; i++) { try { final JobRead jobInfo = testHarness.getMostRecentSyncJobId(connectionId); waitForSuccessfulJob(apiClient.getJobsApi(), jobInfo); break; - } catch (Exception e) { + } catch (final Exception e) { LOGGER.info("Something went wrong querying jobs API, retrying..."); } sleep(Duration.ofSeconds(30).toMillis()); diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java index 745e2e5c27e0..807d6a7c73a0 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java @@ -168,7 +168,7 @@ void testIncrementalCdcSync(final TestInfo testInfo) throws Exception { final JobInfoRead connectionSyncRead1 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead1.getJob()); - LOGGER.info("state after sync 1: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info("state after sync 1: {}", apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); final Database source = testHarness.getSourceDatabase(); @@ -218,7 +218,7 @@ void testIncrementalCdcSync(final TestInfo testInfo) throws Exception { final JobInfoRead connectionSyncRead2 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead2.getJob()); - LOGGER.info("state after sync 2: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info("state after sync 2: {}", apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); assertDestinationMatches(ID_AND_NAME_TABLE, expectedIdAndNameRecords); assertDestinationMatches(COLOR_PALETTE_TABLE, expectedColorPaletteRecords); @@ -230,7 +230,7 @@ void testIncrementalCdcSync(final TestInfo testInfo) throws Exception { final JobInfoRead jobInfoRead = apiClient.getConnectionApi().resetConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), jobInfoRead.getJob()); - LOGGER.info("state after reset: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info("state after reset: {}", apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); assertDestinationMatches(ID_AND_NAME_TABLE, Collections.emptyList()); assertDestinationMatches(COLOR_PALETTE_TABLE, Collections.emptyList()); @@ -241,7 +241,7 @@ void testIncrementalCdcSync(final TestInfo testInfo) throws Exception { final JobInfoRead connectionSyncRead3 = apiClient.getConnectionApi().syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead3.getJob()); - LOGGER.info("state after sync 3: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info("state after sync 3: {}", apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); expectedIdAndNameRecords = getCdcRecordMatchersFromSource(source, ID_AND_NAME_TABLE); assertDestinationMatches(ID_AND_NAME_TABLE, expectedIdAndNameRecords); @@ -285,7 +285,7 @@ void testDeleteRecordCdcSync(final TestInfo testInfo) throws Exception { final JobInfoRead connectionSyncRead1 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead1.getJob()); - LOGGER.info("state after sync 1: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info("state after sync 1: {}", apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); final Database source = testHarness.getSourceDatabase(); final List expectedIdAndNameRecords = getCdcRecordMatchersFromSource(source, ID_AND_NAME_TABLE); @@ -309,7 +309,7 @@ void testDeleteRecordCdcSync(final TestInfo testInfo) throws Exception { final JobInfoRead connectionSyncRead2 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead2.getJob()); - LOGGER.info("state after sync 2: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info("state after sync 2: {}", apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); assertDestinationMatches(ID_AND_NAME_TABLE, expectedIdAndNameRecords); } @@ -559,7 +559,7 @@ private void assertDestinationMatches(final String streamName, final List expectedStreams) throws ApiException { - final ConnectionState state = apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId)); + final ConnectionState state = apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId)); LOGGER.info("state: {}", state); assertEquals(ConnectionStateType.GLOBAL, state.getStateType()); final List stateStreams = state.getGlobalState().getStreamStates().stream().map(StreamState::getStreamDescriptor).toList(); @@ -569,7 +569,7 @@ private void assertGlobalStateContainsStreams(final UUID connectionId, final Lis } private void assertNoState(final UUID connectionId) throws ApiException { - final ConnectionState state = apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId)); + final ConnectionState state = apiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId)); assertEquals(ConnectionStateType.NOT_SET, state.getStateType()); assertNull(state.getState()); assertNull(state.getStreamState()); diff --git a/airbyte-webapp/package-lock.json b/airbyte-webapp/package-lock.json index 47a41ec05ca5..d6a677da4c03 100644 --- a/airbyte-webapp/package-lock.json +++ b/airbyte-webapp/package-lock.json @@ -26,6 +26,7 @@ "flat": "^5.0.2", "formik": "^2.2.9", "framer-motion": "^6.3.11", + "js-yaml": "^4.1.0", "launchdarkly-js-client-sdk": "^2.22.1", "lodash": "^4.17.21", "mdast": "^3.0.0", @@ -73,6 +74,7 @@ "@testing-library/user-event": "^13.5.0", "@types/flat": "^5.0.2", "@types/jest": "^27.4.1", + "@types/js-yaml": "^4.0.5", "@types/json-schema": "^7.0.11", "@types/lodash": "^4.14.182", "@types/node": "^17.0.40", @@ -132,6 +134,28 @@ "js-yaml": "^3.13.1" } }, + "node_modules/@apidevtools/json-schema-ref-parser/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/@apidevtools/json-schema-ref-parser/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/@apidevtools/openapi-schemas": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@apidevtools/openapi-schemas/-/openapi-schemas-2.1.0.tgz", @@ -2706,12 +2730,6 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, - "node_modules/@eslint/eslintrc/node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true - }, "node_modules/@eslint/eslintrc/node_modules/globals": { "version": "13.12.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.12.0.tgz", @@ -2736,18 +2754,6 @@ "node": ">= 4" } }, - "node_modules/@eslint/eslintrc/node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/@eslint/eslintrc/node_modules/type-fest": { "version": "0.20.2", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", @@ -3711,6 +3717,15 @@ "node": ">=8" } }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, "node_modules/@istanbuljs/load-nyc-config/node_modules/camelcase": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", @@ -3720,6 +3735,19 @@ "node": ">=6" } }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/@istanbuljs/load-nyc-config/node_modules/resolve-from": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", @@ -14429,6 +14457,12 @@ "resolved": "https://registry.npmjs.org/@types/js-cookie/-/js-cookie-2.2.6.tgz", "integrity": "sha512-+oY0FDTO2GYKEV0YPvSshGq9t7YozVkgvXLty7zogQNuCxBhT9/3INX9Q7H1aRZ4SUDRXAKlJuA4EA5nTt7SNw==" }, + "node_modules/@types/js-yaml": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.5.tgz", + "integrity": "sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA==", + "dev": true + }, "node_modules/@types/json-schema": { "version": "7.0.11", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz", @@ -16094,13 +16128,9 @@ "dev": true }, "node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, - "dependencies": { - "sprintf-js": "~1.0.2" - } + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" }, "node_modules/aria-query": { "version": "4.2.2", @@ -21765,12 +21795,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/eslint/node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true - }, "node_modules/eslint/node_modules/chalk": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", @@ -21884,18 +21908,6 @@ "node": ">=8" } }, - "node_modules/eslint/node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/eslint/node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -25024,6 +25036,15 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/ibm-openapi-validator/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, "node_modules/ibm-openapi-validator/node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -25085,6 +25106,19 @@ "node": ">=8" } }, + "node_modules/ibm-openapi-validator/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/ibm-openapi-validator/node_modules/locate-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", @@ -29775,13 +29809,11 @@ "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" }, "node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", - "dev": true, + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" + "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" @@ -29909,6 +29941,15 @@ "ono": "^4.0.6" } }, + "node_modules/json-schema-ref-parser/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, "node_modules/json-schema-ref-parser/node_modules/debug": { "version": "3.2.7", "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", @@ -29918,6 +29959,19 @@ "ms": "^2.1.1" } }, + "node_modules/json-schema-ref-parser/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -42941,7 +42995,7 @@ "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dev": true }, "node_modules/spy-on-component": { @@ -44167,6 +44221,28 @@ "node": ">=4.0.0" } }, + "node_modules/svgo/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/svgo/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/swagger2openapi": { "version": "7.0.8", "resolved": "https://registry.npmjs.org/swagger2openapi/-/swagger2openapi-7.0.8.tgz", @@ -47728,6 +47804,27 @@ "@jsdevtools/ono": "^7.1.3", "call-me-maybe": "^1.0.1", "js-yaml": "^3.13.1" + }, + "dependencies": { + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "~1.0.2" + } + }, + "js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + } } }, "@apidevtools/openapi-schemas": { @@ -49547,12 +49644,6 @@ "strip-json-comments": "^3.1.1" }, "dependencies": { - "argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true - }, "globals": { "version": "13.12.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.12.0.tgz", @@ -49568,15 +49659,6 @@ "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", "dev": true }, - "js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, - "requires": { - "argparse": "^2.0.1" - } - }, "type-fest": { "version": "0.20.2", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", @@ -50466,12 +50548,31 @@ "resolve-from": "^5.0.0" }, "dependencies": { + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "~1.0.2" + } + }, "camelcase": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", "dev": true }, + "js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, "resolve-from": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", @@ -58587,6 +58688,12 @@ "resolved": "https://registry.npmjs.org/@types/js-cookie/-/js-cookie-2.2.6.tgz", "integrity": "sha512-+oY0FDTO2GYKEV0YPvSshGq9t7YozVkgvXLty7zogQNuCxBhT9/3INX9Q7H1aRZ4SUDRXAKlJuA4EA5nTt7SNw==" }, + "@types/js-yaml": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.5.tgz", + "integrity": "sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA==", + "dev": true + }, "@types/json-schema": { "version": "7.0.11", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz", @@ -59926,13 +60033,9 @@ "dev": true }, "argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, - "requires": { - "sprintf-js": "~1.0.2" - } + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" }, "aria-query": { "version": "4.2.2", @@ -64022,12 +64125,6 @@ "color-convert": "^2.0.1" } }, - "argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true - }, "chalk": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", @@ -64105,15 +64202,6 @@ "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true }, - "js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, - "requires": { - "argparse": "^2.0.1" - } - }, "supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -66931,6 +67019,15 @@ "color-convert": "^2.0.1" } }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "~1.0.2" + } + }, "chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -66977,6 +67074,16 @@ "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true }, + "js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, "locate-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", @@ -70546,13 +70653,11 @@ "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" }, "js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", - "dev": true, + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "requires": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" + "argparse": "^2.0.1" } }, "jsdom": { @@ -70653,6 +70758,15 @@ "ono": "^4.0.6" }, "dependencies": { + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "~1.0.2" + } + }, "debug": { "version": "3.2.7", "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", @@ -70661,6 +70775,16 @@ "requires": { "ms": "^2.1.1" } + }, + "js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } } } }, @@ -80221,7 +80345,7 @@ "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dev": true }, "spy-on-component": { @@ -81139,6 +81263,27 @@ "stable": "^0.1.8", "unquote": "~1.1.1", "util.promisify": "~1.0.0" + }, + "dependencies": { + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "~1.0.2" + } + }, + "js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + } } }, "swagger2openapi": { diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index e71d58b91a68..53e0150b8a8e 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -42,6 +42,7 @@ "flat": "^5.0.2", "formik": "^2.2.9", "framer-motion": "^6.3.11", + "js-yaml": "^4.1.0", "launchdarkly-js-client-sdk": "^2.22.1", "lodash": "^4.17.21", "mdast": "^3.0.0", @@ -89,6 +90,7 @@ "@testing-library/user-event": "^13.5.0", "@types/flat": "^5.0.2", "@types/jest": "^27.4.1", + "@types/js-yaml": "^4.0.5", "@types/json-schema": "^7.0.11", "@types/lodash": "^4.14.182", "@types/node": "^17.0.40", diff --git a/airbyte-webapp/src/components/ConnectorBlocks/TableItemTitle.tsx b/airbyte-webapp/src/components/ConnectorBlocks/TableItemTitle.tsx index a946d7c1cd0c..5329c9d98c52 100644 --- a/airbyte-webapp/src/components/ConnectorBlocks/TableItemTitle.tsx +++ b/airbyte-webapp/src/components/ConnectorBlocks/TableItemTitle.tsx @@ -4,6 +4,7 @@ import { FormattedMessage, useIntl } from "react-intl"; import { ReleaseStageBadge } from "components/ReleaseStageBadge"; import { Button } from "components/ui/Button"; import { DropDownOptionDataItem } from "components/ui/DropDown"; +import { Heading } from "components/ui/Heading"; import { Popout } from "components/ui/Popout"; import { Text } from "components/ui/Text"; @@ -47,19 +48,17 @@ const TableItemTitle: React.FC = ({
{entityIcon &&
{entityIcon}
}
- - {entityName} - - + {entityName} + {entity}
- + - + { const renderResult = await render(); expect(renderResult).toMatchSnapshot(); }); + + describe("cron expression validation", () => { + const INVALID_CRON_EXPRESSION = "invalid cron expression"; + const CRON_EXPRESSION_EVERY_MINUTE = "* * * * * * ?"; + + it("should display an error for an invalid cron expression", async () => { + jest.spyOn(sourceHook, "useDiscoverSchema").mockImplementationOnce(() => baseUseDiscoverSchema); + + const container = tlr( + + + + ); + + await selectEvent.select(container.getByTestId("scheduleData"), /cron/i); + + const cronExpressionInput = container.getByTestId("cronExpression"); + + userEvent.clear(cronExpressionInput); + await userEvent.type(cronExpressionInput, INVALID_CRON_EXPRESSION, { delay: 1 }); + + const errorMessage = container.getByText("Invalid cron expression"); + + expect(errorMessage).toBeInTheDocument(); + }); + + it("should allow cron expressions under one hour when feature enabled", async () => { + jest.spyOn(sourceHook, "useDiscoverSchema").mockImplementationOnce(() => baseUseDiscoverSchema); + + const container = tlr( + + + + ); + + await selectEvent.select(container.getByTestId("scheduleData"), /cron/i); + + const cronExpressionField = container.getByTestId("cronExpression"); + + await userEvent.type(cronExpressionField, `{selectall}${CRON_EXPRESSION_EVERY_MINUTE}`, { delay: 1 }); + + const errorMessage = container.queryByTestId("cronExpressionError"); + + expect(errorMessage).not.toBeInTheDocument(); + }); + + it("should not allow cron expressions under one hour when feature not enabled", async () => { + jest.spyOn(sourceHook, "useDiscoverSchema").mockImplementationOnce(() => baseUseDiscoverSchema); + + const featuresToInject = defaultFeatures.filter((f) => f !== FeatureItem.AllowSyncSubOneHourCronExpressions); + + const container = tlr( + + + + ); + + await selectEvent.select(container.getByTestId("scheduleData"), /cron/i); + + const cronExpressionField = container.getByTestId("cronExpression"); + + await userEvent.type(cronExpressionField, `{selectall}${CRON_EXPRESSION_EVERY_MINUTE}`, { delay: 1 }); + + const errorMessage = container.getByTestId("cronExpressionError"); + + expect(errorMessage).toBeInTheDocument(); + }); + }); }); diff --git a/airbyte-webapp/src/components/CreateConnection/CreateConnectionForm.tsx b/airbyte-webapp/src/components/CreateConnection/CreateConnectionForm.tsx index 01c12a7fed1e..6851a2ef20d4 100644 --- a/airbyte-webapp/src/components/CreateConnection/CreateConnectionForm.tsx +++ b/airbyte-webapp/src/components/CreateConnection/CreateConnectionForm.tsx @@ -10,6 +10,7 @@ import { tidyConnectionFormValues, useConnectionFormService, } from "hooks/services/ConnectionForm/ConnectionFormService"; +import { FeatureItem, useFeature } from "hooks/services/Feature"; import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; import { useCreateConnection } from "hooks/services/useConnectionHook"; import { SchemaError as SchemaErrorType, useDiscoverSchema } from "hooks/services/useSourceHook"; @@ -17,7 +18,10 @@ import { useCurrentWorkspaceId } from "services/workspaces/WorkspacesService"; import CreateControls from "views/Connection/ConnectionForm/components/CreateControls"; import { OperationsSection } from "views/Connection/ConnectionForm/components/OperationsSection"; import { ConnectionFormFields } from "views/Connection/ConnectionForm/ConnectionFormFields"; -import { connectionValidationSchema, FormikConnectionFormValues } from "views/Connection/ConnectionForm/formConfig"; +import { + createConnectionValidationSchema, + FormikConnectionFormValues, +} from "views/Connection/ConnectionForm/formConfig"; import styles from "./CreateConnectionForm.module.scss"; import { CreateConnectionNameField } from "./CreateConnectionNameField"; @@ -44,10 +48,11 @@ const CreateConnectionFormInner: React.FC = ({ schem const { connection, initialValues, mode, formId, getErrorMessage, setSubmitError } = useConnectionFormService(); const [editingTransformation, setEditingTransformation] = useState(false); + const allowSubOneHourCronExpressions = useFeature(FeatureItem.AllowSyncSubOneHourCronExpressions); const onFormSubmit = useCallback( async (formValues: FormikConnectionFormValues, formikHelpers: FormikHelpers) => { - const values = tidyConnectionFormValues(formValues, workspaceId, mode); + const values = tidyConnectionFormValues(formValues, workspaceId, mode, allowSubOneHourCronExpressions); try { const createdConnection = await createConnection({ @@ -89,6 +94,7 @@ const CreateConnectionFormInner: React.FC = ({ schem afterSubmitConnection, navigate, setSubmitError, + allowSubOneHourCronExpressions, ] ); @@ -101,9 +107,8 @@ const CreateConnectionFormInner: React.FC = ({ schem
{({ values, isSubmitting, isValid, dirty }) => (
diff --git a/airbyte-webapp/src/components/CreateConnection/CreateConnectionNameField.tsx b/airbyte-webapp/src/components/CreateConnection/CreateConnectionNameField.tsx index 2510ee642e71..e0ea3f271ff1 100644 --- a/airbyte-webapp/src/components/CreateConnection/CreateConnectionNameField.tsx +++ b/airbyte-webapp/src/components/CreateConnection/CreateConnectionNameField.tsx @@ -2,8 +2,8 @@ import { Field, FieldProps } from "formik"; import { FormattedMessage, useIntl } from "react-intl"; import { ControlLabels } from "components/LabeledControl"; +import { Heading } from "components/ui/Heading"; import { Input } from "components/ui/Input"; -import { Text } from "components/ui/Text"; import { Section } from "views/Connection/ConnectionForm/components/Section"; @@ -23,9 +23,9 @@ export const CreateConnectionNameField = () => { nextLine error={!!meta.error && meta.touched} label={ - + - + } message={formatMessage({ id: "form.connectionName.message", diff --git a/airbyte-webapp/src/components/CreateConnection/TryAfterErrorBlock.tsx b/airbyte-webapp/src/components/CreateConnection/TryAfterErrorBlock.tsx index 4bb6a7a091bf..4ebcd84f906e 100644 --- a/airbyte-webapp/src/components/CreateConnection/TryAfterErrorBlock.tsx +++ b/airbyte-webapp/src/components/CreateConnection/TryAfterErrorBlock.tsx @@ -16,11 +16,11 @@ export const TryAfterErrorBlock: React.FC = ({ message, return (
- + {message && ( - + )} diff --git a/airbyte-webapp/src/components/EmptyResourceListView/EmptyResourceListView.tsx b/airbyte-webapp/src/components/EmptyResourceListView/EmptyResourceListView.tsx index f87234468aa3..400b8f807a80 100644 --- a/airbyte-webapp/src/components/EmptyResourceListView/EmptyResourceListView.tsx +++ b/airbyte-webapp/src/components/EmptyResourceListView/EmptyResourceListView.tsx @@ -3,7 +3,7 @@ import { useMemo } from "react"; import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; -import { Text } from "components/ui/Text"; +import { Heading } from "components/ui/Heading"; import { ReactComponent as BowtieHalf } from "./bowtie-half.svg"; import styles from "./EmptyResourceListView.module.scss"; @@ -30,9 +30,9 @@ export const EmptyResourceListView: React.FC = ({ return (
- + - +
{resourceType !== "destinations" && (
+ ); +}; diff --git a/airbyte-webapp/src/components/StreamTestingPanel/index.tsx b/airbyte-webapp/src/components/StreamTestingPanel/index.tsx new file mode 100644 index 000000000000..056c7823f21b --- /dev/null +++ b/airbyte-webapp/src/components/StreamTestingPanel/index.tsx @@ -0,0 +1 @@ +export * from "./StreamTestingPanel"; diff --git a/airbyte-webapp/src/components/YamlEditor/YamlEditor.module.scss b/airbyte-webapp/src/components/YamlEditor/YamlEditor.module.scss index 6552c514205a..55db3db92add 100644 --- a/airbyte-webapp/src/components/YamlEditor/YamlEditor.module.scss +++ b/airbyte-webapp/src/components/YamlEditor/YamlEditor.module.scss @@ -1,4 +1,5 @@ @use "scss/colors"; +@use "scss/variables"; .container { display: flex; @@ -10,25 +11,11 @@ flex: 0 0 auto; background-color: colors.$dark-blue; display: flex; - padding: 10px; + padding: variables.$spacing-md; + flex-direction: row; + justify-content: flex-end; } .editorContainer { flex: 1 1 0; - background-image: linear-gradient(colors.$dark-blue-900, colors.$dark-blue-1000); -} - -.downloadButton { - margin-left: auto; -} - -// Export colors to be used in monaco editor -:export { - // Monaco editor requires 6-character hex values for theme colors - /* stylelint-disable-next-line color-hex-length, color-no-hex */ - tokenString: colors.$white; - tokenType: colors.$blue-300; - tokenNumber: colors.$orange-300; - tokenDelimiter: colors.$yellow-300; - tokenKeyword: colors.$green-300; } diff --git a/airbyte-webapp/src/components/YamlEditor/YamlEditor.tsx b/airbyte-webapp/src/components/YamlEditor/YamlEditor.tsx index 8129d65dc3c4..6aa53a83b9ce 100644 --- a/airbyte-webapp/src/components/YamlEditor/YamlEditor.tsx +++ b/airbyte-webapp/src/components/YamlEditor/YamlEditor.tsx @@ -1,65 +1,25 @@ -import Editor, { Monaco } from "@monaco-editor/react"; -import { useState } from "react"; -import { useDebounce, useLocalStorage } from "react-use"; +import { CodeEditor } from "components/ui/CodeEditor"; + +import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; import { DownloadYamlButton } from "./DownloadYamlButton"; import styles from "./YamlEditor.module.scss"; -import { template } from "./YamlTemplate"; export const YamlEditor: React.FC = () => { - const [locallyStoredEditorValue, setLocallyStoredEditorValue] = useLocalStorage( - "connectorBuilderEditorContent", - template - ); - const [editorValue, setEditorValue] = useState(locallyStoredEditorValue ?? ""); - useDebounce(() => setLocallyStoredEditorValue(editorValue), 500, [editorValue]); - - const handleEditorChange = (value: string | undefined) => { - setEditorValue(value ?? ""); - }; - - const setEditorTheme = (monaco: Monaco) => { - monaco.editor.defineTheme("airbyte", { - base: "vs-dark", - inherit: true, - rules: [ - { token: "string", foreground: styles.tokenString }, - { token: "type", foreground: styles.tokenType }, - { token: "number", foreground: styles.tokenNumber }, - { token: "delimiter", foreground: styles.tokenDelimiter }, - { token: "keyword", foreground: styles.tokenKeyword }, - ], - colors: { - "editor.background": "#00000000", // transparent, so that parent background is shown instead - }, - }); - - monaco.editor.setTheme("airbyte"); - }; + const { yamlManifest, setYamlManifest } = useConnectorBuilderState(); return (
- +
- setYamlManifest(value ?? "")} + lineNumberCharacterWidth={6} />
diff --git a/airbyte-webapp/src/components/base/Titles/Titles.tsx b/airbyte-webapp/src/components/base/Titles/Titles.tsx index 46471d7c4060..9d2a5ee0fd9d 100644 --- a/airbyte-webapp/src/components/base/Titles/Titles.tsx +++ b/airbyte-webapp/src/components/base/Titles/Titles.tsx @@ -19,13 +19,13 @@ const H1 = styled.h1` margin: 0; `; -/** @deprecated Use `` */ +/** @deprecated Use `` */ export const H3 = styled(H1).attrs({ as: "h3" })` font-size: 20px; line-height: 24px; `; -/** @deprecated Use `` */ +/** @deprecated Use `` */ export const H5 = styled(H1).attrs({ as: "h5" })` font-size: ${({ theme }) => theme.h5?.fontSize || "16px"}; line-height: ${({ theme }) => theme.h5?.lineHeight || "28px"}; diff --git a/airbyte-webapp/src/components/connection/CatalogTree/CatalogSection.tsx b/airbyte-webapp/src/components/connection/CatalogTree/CatalogSection.tsx index 2e1c03fdeba5..5306123c3085 100644 --- a/airbyte-webapp/src/components/connection/CatalogTree/CatalogSection.tsx +++ b/airbyte-webapp/src/components/connection/CatalogTree/CatalogSection.tsx @@ -18,6 +18,8 @@ import { equal, naturalComparatorBy } from "utils/objects"; import { ConnectionFormValues, SUPPORTED_MODES } from "views/Connection/ConnectionForm/formConfig"; import styles from "./CatalogSection.module.scss"; +import { CatalogTreeTableRow } from "./next/CatalogTreeTableRow"; +import { StreamDetailsPanel } from "./next/StreamDetailsPanel"; import { StreamFieldTable } from "./StreamFieldTable"; import { StreamHeader } from "./StreamHeader"; import { flatten, getPathType } from "./utils"; @@ -41,6 +43,8 @@ const CatalogSectionInner: React.FC = ({ errors, changedSelected, }) => { + const isNewStreamsTableEnabled = process.env.REACT_APP_NEW_STREAMS_TABLE ?? false; + const { destDefinition: { supportedDestinationSyncModes }, } = useConnectionFormService(); @@ -135,9 +139,11 @@ const CatalogSectionInner: React.FC = ({ const hasFields = fields?.length > 0; const disabled = mode === "readonly"; + const StreamComponent = isNewStreamsTableEnabled ? CatalogTreeTableRow : StreamHeader; + return (
- = ({ hasError={hasError} disabled={disabled} /> - {isRowExpanded && hasFields && ( -
- -
- )} + ) : ( +
+ +
+ ))}
); }; diff --git a/airbyte-webapp/src/components/connection/CatalogTree/CatalogTree.tsx b/airbyte-webapp/src/components/connection/CatalogTree/CatalogTree.tsx index b00a71c0cf9e..932a78b16d5f 100644 --- a/airbyte-webapp/src/components/connection/CatalogTree/CatalogTree.tsx +++ b/airbyte-webapp/src/components/connection/CatalogTree/CatalogTree.tsx @@ -12,6 +12,9 @@ import { CatalogTreeBody } from "./CatalogTreeBody"; import { CatalogTreeHeader } from "./CatalogTreeHeader"; import { CatalogTreeSearch } from "./CatalogTreeSearch"; import { CatalogTreeSubheader } from "./CatalogTreeSubheader"; +import { BulkEditPanel } from "./next/BulkEditPanel"; +import { CatalogTreeTableHeader } from "./next/CatalogTreeTableHeader"; +import { StreamConnectionHeader } from "./next/StreamConnectionHeader"; interface CatalogTreeProps { streams: SyncSchemaStream[]; @@ -24,6 +27,7 @@ const CatalogTreeComponent: React.FC> onStreamsChanged, isLoading, }) => { + const isNewStreamsTableEnabled = process.env.REACT_APP_NEW_STREAMS_TABLE ?? false; const { mode } = useConnectionFormService(); const [searchString, setSearchString] = useState(""); @@ -53,11 +57,21 @@ const CatalogTreeComponent: React.FC> {mode !== "readonly" && } - - - + {isNewStreamsTableEnabled ? ( + <> + + + + ) : ( + <> + + + + + )} + {isNewStreamsTableEnabled && } ); }; diff --git a/airbyte-webapp/src/components/connection/CatalogTree/PathPopoutButton.module.scss b/airbyte-webapp/src/components/connection/CatalogTree/PathPopoutButton.module.scss index 0b1615bbf20e..097557198a45 100644 --- a/airbyte-webapp/src/components/connection/CatalogTree/PathPopoutButton.module.scss +++ b/airbyte-webapp/src/components/connection/CatalogTree/PathPopoutButton.module.scss @@ -12,6 +12,7 @@ flex-direction: row; justify-content: space-between; gap: variables.$spacing-sm; + max-width: 100%; padding: 8px; border-radius: variables.$border-radius-xs; background-color: colors.$grey-100; diff --git a/airbyte-webapp/src/components/connection/CatalogTree/PathPopoutButton.tsx b/airbyte-webapp/src/components/connection/CatalogTree/PathPopoutButton.tsx index 91513e04d88f..b7c685c5870b 100644 --- a/airbyte-webapp/src/components/connection/CatalogTree/PathPopoutButton.tsx +++ b/airbyte-webapp/src/components/connection/CatalogTree/PathPopoutButton.tsx @@ -1,4 +1,4 @@ -import { faSortDown } from "@fortawesome/free-solid-svg-icons"; +import { faCaretDown } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React from "react"; @@ -23,7 +23,7 @@ export const PathPopoutButton: React.FC {children} - + } placement="bottom-start" diff --git a/airbyte-webapp/src/components/connection/CatalogTree/StreamHeader.module.scss b/airbyte-webapp/src/components/connection/CatalogTree/StreamHeader.module.scss index 2bd4fa8812e8..f7a7b2fed34a 100644 --- a/airbyte-webapp/src/components/connection/CatalogTree/StreamHeader.module.scss +++ b/airbyte-webapp/src/components/connection/CatalogTree/StreamHeader.module.scss @@ -1,6 +1,6 @@ @use "scss/colors"; @use "scss/variables"; -@forward "./CatalogTreeBody.module.scss"; +@forward "./CatalogTreeHeader.module.scss"; .icon { margin-right: 7px; diff --git a/airbyte-webapp/src/components/connection/CatalogTree/SyncSettingsDropdown.tsx b/airbyte-webapp/src/components/connection/CatalogTree/SyncSettingsDropdown.tsx index 9dee9dd87178..d54b049e5c4d 100644 --- a/airbyte-webapp/src/components/connection/CatalogTree/SyncSettingsDropdown.tsx +++ b/airbyte-webapp/src/components/connection/CatalogTree/SyncSettingsDropdown.tsx @@ -53,7 +53,8 @@ const Title = styled.span` const OptionContent = styled(OptionView)` justify-content: left; - font-size: 12px; + font-size: 11px; + padding: 10px; `; const DropdownControl = styled(components.Control)>` @@ -74,7 +75,7 @@ const Mode: React.FC<{ }> = (props) => { return ( <> - {props.title}: + {props.title}:
{props.label}
{props.separator ? {props.separator} : null} diff --git a/airbyte-webapp/src/components/connection/CatalogTree/next/BulkEditPanel.tsx b/airbyte-webapp/src/components/connection/CatalogTree/next/BulkEditPanel.tsx new file mode 100644 index 000000000000..7855697333de --- /dev/null +++ b/airbyte-webapp/src/components/connection/CatalogTree/next/BulkEditPanel.tsx @@ -0,0 +1,180 @@ +import { faArrowRight } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import intersection from "lodash/intersection"; +import React, { useMemo } from "react"; +import { createPortal } from "react-dom"; +import { FormattedMessage } from "react-intl"; +import styled from "styled-components"; + +import { Cell, Header } from "components"; +import { Button } from "components/ui/Button"; +import { Switch } from "components/ui/Switch"; + +import { SyncSchemaField, SyncSchemaFieldObject, SyncSchemaStream, traverseSchemaToField } from "core/domain/catalog"; +import { DestinationSyncMode, SyncMode } from "core/request/AirbyteClient"; +import { useBulkEditService } from "hooks/services/BulkEdit/BulkEditService"; +import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; +import { SUPPORTED_MODES } from "views/Connection/ConnectionForm/formConfig"; + +import { pathDisplayName, PathPopout } from "../PathPopout"; +import { HeaderCell } from "../styles"; +import { SyncSettingsDropdown } from "../SyncSettingsDropdown"; +import { flatten, getPathType } from "../utils"; + +const SchemaHeader = styled(Header)` + position: fixed; + bottom: 0; + left: 122px; + z-index: 1000; + width: calc(100% - 152px); + bottom: 0; + height: unset; + background: ${({ theme }) => theme.primaryColor}; + border-radius: 8px 8px 0 0; +`; + +function calculateSharedFields(selectedBatchNodes: SyncSchemaStream[]) { + const primitiveFieldsByStream = selectedBatchNodes.map(({ stream }) => { + const traversedFields = traverseSchemaToField(stream?.jsonSchema, stream?.name); + const flattenedFields = flatten(traversedFields); + + return flattenedFields.filter(SyncSchemaFieldObject.isPrimitive); + }); + + const pathMap = new Map(); + + // calculate intersection of primitive fields across all selected streams + primitiveFieldsByStream.forEach((fields, index) => { + if (index === 0) { + fields.forEach((field) => pathMap.set(pathDisplayName(field.path), field)); + } else { + const fieldMap = new Set(fields.map((f) => pathDisplayName(f.path))); + pathMap.forEach((_, k) => (!fieldMap.has(k) ? pathMap.delete(k) : null)); + } + }); + + return Array.from(pathMap.values()); +} + +export const BulkEditPanel: React.FC = () => { + const { + destDefinition: { supportedDestinationSyncModes }, + } = useConnectionFormService(); + const { selectedBatchNodes, options, onChangeOption, onApply, isActive, onCancel } = useBulkEditService(); + + const availableSyncModes = useMemo( + () => + SUPPORTED_MODES.filter(([syncMode, destinationSyncMode]) => { + const supportableModes = intersection(selectedBatchNodes.flatMap((n) => n.stream?.supportedSyncModes)); + return supportableModes.includes(syncMode) && supportedDestinationSyncModes?.includes(destinationSyncMode); + }).map(([syncMode, destinationSyncMode]) => ({ + value: { syncMode, destinationSyncMode }, + })), + [selectedBatchNodes, supportedDestinationSyncModes] + ); + + const primitiveFields: SyncSchemaField[] = useMemo( + () => calculateSharedFields(selectedBatchNodes), + [selectedBatchNodes] + ); + + if (!isActive) { + return null; + } + + const pkRequired = options.destinationSyncMode === DestinationSyncMode.append_dedup; + const shouldDefinePk = selectedBatchNodes.every((n) => n.stream?.sourceDefinedPrimaryKey?.length === 0) && pkRequired; + const cursorRequired = options.syncMode === SyncMode.incremental; + const shouldDefineCursor = selectedBatchNodes.every((n) => !n.stream?.sourceDefinedCursor) && cursorRequired; + const numStreamsSelected = selectedBatchNodes.length; + + const pkType = getPathType(pkRequired, shouldDefinePk); + const cursorType = getPathType(cursorRequired, shouldDefineCursor); + + const paths = primitiveFields.map((f) => f.path); + + return createPortal( + + +
{numStreamsSelected}
+ + + +
+ +
+ onChangeOption({ selected: !options.selected })} /> + + +
+ +
+ onChangeOption({ ...value })} + /> + + + {cursorType && ( + <> +
+ +
+ onChangeOption({ cursorField: path })} + pathType={cursorType} + paths={paths} + path={options.cursorField} + /> + + )} + + + {pkType && ( + <> +
+ +
+ onChangeOption({ primaryKey: path })} + pathType={pkType} + paths={paths} + path={options.primaryKey} + /> + + )} + + + + + +
+ +
+ onChangeOption({ ...value })} + /> + + + + + + , + document.body + ); +}; diff --git a/airbyte-webapp/src/components/connection/CatalogTree/next/CatalogTreeTableHeader.tsx b/airbyte-webapp/src/components/connection/CatalogTree/next/CatalogTreeTableHeader.tsx new file mode 100644 index 000000000000..2d3f11682aab --- /dev/null +++ b/airbyte-webapp/src/components/connection/CatalogTree/next/CatalogTreeTableHeader.tsx @@ -0,0 +1,70 @@ +import { FormattedMessage } from "react-intl"; + +import { Cell, Header } from "components/SimpleTableComponents"; +import { CheckBox } from "components/ui/CheckBox"; +import { InfoTooltip, TooltipLearnMoreLink } from "components/ui/Tooltip"; + +import { useBulkEditService } from "hooks/services/BulkEdit/BulkEditService"; +import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; +import { links } from "utils/links"; + +export const CatalogTreeTableHeader: React.FC = () => { + const { mode } = useConnectionFormService(); + const { onCheckAll, selectedBatchNodeIds, allChecked } = useBulkEditService(); + + return ( +
+ + {mode !== "readonly" && ( + 0 && !allChecked} + checked={allChecked} + /> + )} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ ); +}; diff --git a/airbyte-webapp/src/components/connection/CatalogTree/next/CatalogTreeTableRow.module.scss b/airbyte-webapp/src/components/connection/CatalogTree/next/CatalogTreeTableRow.module.scss new file mode 100644 index 000000000000..557907278af4 --- /dev/null +++ b/airbyte-webapp/src/components/connection/CatalogTree/next/CatalogTreeTableRow.module.scss @@ -0,0 +1,48 @@ +@use "scss/colors"; +@use "scss/variables"; + +.icon { + margin-right: 7px; + margin-top: -1px; + + &.plus { + color: colors.$green; + } + + &.minus { + color: colors.$red; + } +} + +.streamHeaderContent { + background: colors.$white; + border-bottom: 1px solid colors.$grey-50; + + &:hover { + background: colors.$grey-30; + cursor: pointer; + } + + &.disabledChange { + background-color: colors.$red-50; + } + + &.enabledChange { + background-color: colors.$green-50; + } + + &.error { + border: 1px solid colors.$red; + } + + &.selected { + background-color: colors.$blue-transparent; + } +} + +.streamRowCheckboxCell { + display: flex; + flex-direction: row; + justify-content: flex-end; + padding-left: 27px; +} diff --git a/airbyte-webapp/src/components/connection/CatalogTree/next/CatalogTreeTableRow.tsx b/airbyte-webapp/src/components/connection/CatalogTree/next/CatalogTreeTableRow.tsx new file mode 100644 index 000000000000..c651777b389f --- /dev/null +++ b/airbyte-webapp/src/components/connection/CatalogTree/next/CatalogTreeTableRow.tsx @@ -0,0 +1,147 @@ +import { faArrowRight, faMinus, faPlus } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import classnames from "classnames"; +import { useMemo } from "react"; +import { FormattedMessage } from "react-intl"; + +import { Cell, Row } from "components/SimpleTableComponents"; +import { CheckBox } from "components/ui/CheckBox"; +import { Switch } from "components/ui/Switch"; + +import { useBulkEditSelect } from "hooks/services/BulkEdit/BulkEditService"; + +import { PathPopout } from "../PathPopout"; +import { StreamHeaderProps } from "../StreamHeader"; +import { HeaderCell } from "../styles"; +import styles from "./CatalogTreeTableRow.module.scss"; + +export const CatalogTreeTableRow: React.FC = ({ + stream, + destName, + destNamespace, + // onSelectSyncMode, + onSelectStream, + // availableSyncModes, + pkType, + onPrimaryKeyChange, + onCursorChange, + primitiveFields, + cursorType, + // isRowExpanded, + fields, + onExpand, + changedSelected, + hasError, + disabled, +}) => { + const { primaryKey, syncMode, cursorField, destinationSyncMode } = stream.config ?? {}; + const isStreamEnabled = stream.config?.selected; + + const { defaultCursorField } = stream.stream ?? {}; + const syncSchema = useMemo( + () => ({ + syncMode, + destinationSyncMode, + }), + [syncMode, destinationSyncMode] + ); + + const [isSelected, selectForBulkEdit] = useBulkEditSelect(stream.id); + + const paths = useMemo(() => primitiveFields.map((field) => field.path), [primitiveFields]); + const fieldCount = fields?.length ?? 0; + const onRowClick = fieldCount > 0 ? () => onExpand() : undefined; + + const iconStyle = classnames(styles.icon, { + [styles.plus]: isStreamEnabled, + [styles.minus]: !isStreamEnabled, + }); + + const streamHeaderContentStyle = classnames(styles.streamHeaderContent, { + [styles.enabledChange]: changedSelected && isStreamEnabled, + [styles.disabledChange]: changedSelected && !isStreamEnabled, + [styles.selected]: isSelected, + [styles.error]: hasError, + }); + + const checkboxCellCustomStyle = classnames(styles.checkboxCell, styles.streamRowCheckboxCell); + + return ( + + {!disabled && ( +
+ {changedSelected && ( +
+ {isStreamEnabled ? ( + + ) : ( + + )} +
+ )} + +
+ )} + + + + {fieldCount} + + {stream.stream?.namespace || } + + {stream.stream?.name} + + {disabled ? ( + + {syncSchema.syncMode} + + ) : ( + // TODO: Replace with Dropdown/Popout + syncSchema.syncMode + )} + + + {cursorType && ( + } + onPathChange={onCursorChange} + /> + )} + + + {pkType && ( + } + onPathChange={onPrimaryKeyChange} + /> + )} + + + + + + {destNamespace} + + + {destName} + + + {disabled ? ( + + {syncSchema.destinationSyncMode} + + ) : ( + // TODO: Replace with Dropdown/Popout + syncSchema.destinationSyncMode + )} + + + ); +}; diff --git a/airbyte-webapp/src/components/connection/CatalogTree/next/StreamConnectionHeader.module.scss b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamConnectionHeader.module.scss new file mode 100644 index 000000000000..c2a9085a9b8e --- /dev/null +++ b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamConnectionHeader.module.scss @@ -0,0 +1,20 @@ +@use "scss/variables"; + +$icon-size: 15px; + +.container { + display: flex; + flex-direction: row; + justify-content: space-between; + padding: variables.$spacing-lg calc(variables.$spacing-lg * 2); +} + +.connector { + display: flex; + flex-direction: row; +} + +.icon { + height: $icon-size; + width: $icon-size; +} diff --git a/airbyte-webapp/src/components/connection/CatalogTree/next/StreamConnectionHeader.tsx b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamConnectionHeader.tsx new file mode 100644 index 000000000000..cb65e0ca8f92 --- /dev/null +++ b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamConnectionHeader.tsx @@ -0,0 +1,27 @@ +import { faArrowRight } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; + +import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; +import { useDestinationDefinition } from "services/connector/DestinationDefinitionService"; +import { useSourceDefinition } from "services/connector/SourceDefinitionService"; +import { getIcon } from "utils/imageUtils"; + +import styles from "./StreamConnectionHeader.module.scss"; + +const renderIcon = (icon?: string): JSX.Element =>
{getIcon(icon)}
; + +export const StreamConnectionHeader: React.FC = () => { + const { + connection: { source, destination }, + } = useConnectionEditService(); + const sourceDefinition = useSourceDefinition(source.sourceDefinitionId); + const destinationDefinition = useDestinationDefinition(destination.destinationDefinitionId); + + return ( +
+
{renderIcon(sourceDefinition.icon)} Source
+ +
{renderIcon(destinationDefinition.icon)} Destination
+
+ ); +}; diff --git a/airbyte-webapp/src/components/connection/CatalogTree/next/StreamDetailsPanel.module.scss b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamDetailsPanel.module.scss new file mode 100644 index 000000000000..706de7384422 --- /dev/null +++ b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamDetailsPanel.module.scss @@ -0,0 +1,19 @@ +@use "scss/colors"; +@use "scss/variables"; +@use "scss/z-indices"; + +.dialog { + z-index: z-indices.$modal; +} + +.container { + position: fixed; + bottom: 0; + left: variables.$width-size-menu; + z-index: 1000; + width: calc(100% - variables.$width-size-menu); + height: calc(100vh - 100px); + background: colors.$white; + border-radius: 15px 15px 0 0; + box-shadow: 0 0 22px rgba(colors.$dark-blue-900, 12%); +} diff --git a/airbyte-webapp/src/components/connection/CatalogTree/next/StreamDetailsPanel.tsx b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamDetailsPanel.tsx new file mode 100644 index 000000000000..3697534e3332 --- /dev/null +++ b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamDetailsPanel.tsx @@ -0,0 +1,54 @@ +import { Dialog } from "@headlessui/react"; + +import { Overlay } from "components/ui/Overlay/Overlay"; + +import { AirbyteStream } from "core/request/AirbyteClient"; + +import { StreamConnectionHeader } from "./StreamConnectionHeader"; +import styles from "./StreamDetailsPanel.module.scss"; +import { StreamFieldsTable, StreamFieldsTableProps } from "./StreamFieldsTable"; +import { StreamPanelHeader } from "./StreamPanelHeader"; + +interface StreamDetailsPanelProps extends StreamFieldsTableProps { + disabled?: boolean; + onClose: () => void; + onSelectedChange: () => void; + stream?: AirbyteStream; +} + +export const StreamDetailsPanel: React.FC = ({ + config, + disabled, + onPkSelect, + onCursorSelect, + onClose, + onSelectedChange, + shouldDefinePk, + shouldDefineCursor, + stream, + syncSchemaFields, +}) => { + return ( + + + + + + + + + ); +}; diff --git a/airbyte-webapp/src/components/connection/CatalogTree/next/StreamFieldsTable.tsx b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamFieldsTable.tsx new file mode 100644 index 000000000000..52dc5610ca25 --- /dev/null +++ b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamFieldsTable.tsx @@ -0,0 +1,45 @@ +import { SyncSchemaField, SyncSchemaFieldObject } from "core/domain/catalog"; +import { AirbyteStreamConfiguration } from "core/request/AirbyteClient"; + +import { pathDisplayName } from "../PathPopout"; +import { TreeRowWrapper } from "../TreeRowWrapper"; +import { StreamFieldsTableHeader } from "./StreamFieldsTableHeader"; +import { StreamFieldsTableRow } from "./StreamFieldsTableRow"; + +export interface StreamFieldsTableProps { + config?: AirbyteStreamConfiguration; + onCursorSelect: (cursorPath: string[]) => void; + onPkSelect: (pkPath: string[]) => void; + shouldDefinePk: boolean; + shouldDefineCursor: boolean; + syncSchemaFields: SyncSchemaField[]; +} + +export const StreamFieldsTable: React.FC = ({ + config, + onPkSelect, + onCursorSelect, + shouldDefineCursor, + shouldDefinePk, + syncSchemaFields, +}) => { + return ( + <> + + + + {syncSchemaFields.map((field) => ( + + + + ))} + + ); +}; diff --git a/airbyte-webapp/src/components/connection/CatalogTree/next/StreamFieldsTableHeader.tsx b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamFieldsTableHeader.tsx new file mode 100644 index 000000000000..7db0ce0f52c8 --- /dev/null +++ b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamFieldsTableHeader.tsx @@ -0,0 +1,33 @@ +import React from "react"; +import { FormattedMessage } from "react-intl"; + +import { HeaderCell, NameContainer } from "../styles"; + +export const StreamFieldsTableHeader: React.FC = React.memo(() => ( + <> + + + + + + + + + + + + + + + + + + + {/* + In the design, but we may be unable to get the destination data type + + + + */} + +)); diff --git a/airbyte-webapp/src/components/connection/CatalogTree/next/StreamFieldsTableRow.tsx b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamFieldsTableRow.tsx new file mode 100644 index 000000000000..ea584c08da62 --- /dev/null +++ b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamFieldsTableRow.tsx @@ -0,0 +1,66 @@ +import { faArrowRight } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import React from "react"; + +import { Cell } from "components/SimpleTableComponents"; +import { CheckBox } from "components/ui/CheckBox"; +import { RadioButton } from "components/ui/RadioButton"; + +import { SyncSchemaField } from "core/domain/catalog"; +import { AirbyteStreamConfiguration } from "core/request/AirbyteClient"; +import { equal } from "utils/objects"; +import { useTranslateDataType } from "utils/useTranslateDataType"; + +import DataTypeCell from "../DataTypeCell"; +import { pathDisplayName } from "../PathPopout"; +import { NameContainer } from "../styles"; + +interface StreamFieldsTableRowProps { + isPrimaryKeyEnabled: boolean; + isCursorEnabled: boolean; + + onPrimaryKeyChange: (pk: string[]) => void; + onCursorChange: (cs: string[]) => void; + field: SyncSchemaField; + config: AirbyteStreamConfiguration | undefined; +} + +const StreamFieldsTableRowComponent: React.FC = ({ + onPrimaryKeyChange, + onCursorChange, + field, + config, + isCursorEnabled, + isPrimaryKeyEnabled, +}) => { + const dataType = useTranslateDataType(field); + const name = pathDisplayName(field.path); + + const isCursor = equal(config?.cursorField, field.path); + const isPrimaryKey = !!config?.primaryKey?.some((p) => equal(p, field.path)); + + return ( + <> + + {name} + + {dataType} + {isCursorEnabled && onCursorChange(field.path)} />} + + {isPrimaryKeyEnabled && onPrimaryKeyChange(field.path)} />} + + + + + + {field.cleanedName} + + {/* + In the design, but we may be unable to get the destination data type + {dataType} + */} + + ); +}; + +export const StreamFieldsTableRow = React.memo(StreamFieldsTableRowComponent); diff --git a/airbyte-webapp/src/components/connection/CatalogTree/next/StreamPanelHeader.module.scss b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamPanelHeader.module.scss new file mode 100644 index 000000000000..557c6a268c23 --- /dev/null +++ b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamPanelHeader.module.scss @@ -0,0 +1,21 @@ +@use "scss/colors"; +@use "scss/variables"; + +.container { + border-radius: 10px; + margin: variables.$spacing-md variables.$spacing-lg; + background-color: colors.$grey-50; + display: flex; + flex-direction: row; + justify-content: space-between; + + & > :not(button) { + padding: variables.$spacing-md; + } +} + +.properties { + display: flex; + flex-direction: row; + gap: variables.$spacing-xl; +} diff --git a/airbyte-webapp/src/components/connection/CatalogTree/next/StreamPanelHeader.tsx b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamPanelHeader.tsx new file mode 100644 index 000000000000..e40cdb29ba90 --- /dev/null +++ b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamPanelHeader.tsx @@ -0,0 +1,55 @@ +import { FormattedMessage } from "react-intl"; + +import { CrossIcon } from "components/icons/CrossIcon"; +import { Button } from "components/ui/Button"; +import { Switch } from "components/ui/Switch"; + +import { AirbyteStream, AirbyteStreamConfiguration } from "core/request/AirbyteClient"; + +import styles from "./StreamPanelHeader.module.scss"; + +interface StreamPanelHeaderProps { + config?: AirbyteStreamConfiguration; + disabled?: boolean; + onClose: () => void; + onSelectedChange: () => void; + stream?: AirbyteStream; +} + +interface SomethingProps { + messageId: string; + value?: string; +} + +export const StreamProperty: React.FC = ({ messageId, value }) => { + return ( + + + + + : {value} + + ); +}; + +export const StreamPanelHeader: React.FC = ({ + config, + disabled, + onClose, + onSelectedChange, + stream, +}) => { + return ( +
+
+ +
+
+ + + +
+
+ ); +}; diff --git a/airbyte-webapp/src/components/icons/RotateIcon.tsx b/airbyte-webapp/src/components/icons/RotateIcon.tsx index 00af9a51eb62..7ce8c6868753 100644 --- a/airbyte-webapp/src/components/icons/RotateIcon.tsx +++ b/airbyte-webapp/src/components/icons/RotateIcon.tsx @@ -2,10 +2,12 @@ import { theme } from "theme"; interface Props { color?: string; + width?: string; + height?: string; } -export const RotateIcon = ({ color = theme.greyColor20 }: Props) => ( - +export const RotateIcon = ({ color = theme.greyColor20, width = "20", height = "20" }: Props) => ( + void; + height?: string; + lineNumberCharacterWidth?: number; } -export const CodeEditor: React.FC = ({ code, height, language }) => { +export const CodeEditor: React.FC = ({ + value, + language, + theme, + readOnly, + onChange, + height, + lineNumberCharacterWidth, +}) => { + const setAirbyteTheme = (monaco: Monaco) => { + monaco.editor.defineTheme("airbyte", { + base: "vs-dark", + inherit: true, + rules: [ + { token: "string", foreground: styles.tokenString }, + { token: "type", foreground: styles.tokenType }, + { token: "number", foreground: styles.tokenNumber }, + { token: "delimiter", foreground: styles.tokenDelimiter }, + { token: "keyword", foreground: styles.tokenKeyword }, + ], + colors: { + "editor.background": "#00000000", // transparent, so that parent background is shown instead + }, + }); + + monaco.editor.setTheme("airbyte"); + }; + return ( ; + +const Template: ComponentStory = (args) => ; + +export const Primary = Template.bind({}); +Primary.args = { + size: "md", + children: + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.", +}; diff --git a/airbyte-webapp/src/components/ui/Heading/Heading.tsx b/airbyte-webapp/src/components/ui/Heading/Heading.tsx new file mode 100644 index 000000000000..1e17b54c5f28 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Heading/Heading.tsx @@ -0,0 +1,37 @@ +import classNames from "classnames"; +import React from "react"; + +import styles from "./heading.module.scss"; + +type HeadingSize = "sm" | "md" | "lg" | "xl"; +type HeadingElementType = "h1" | "h2" | "h3" | "h4" | "h5" | "h6"; + +interface HeadingProps { + className?: string; + centered?: boolean; + as: HeadingElementType; + size?: HeadingSize; +} + +const getHeadingClassNames = ({ size, centered }: Required>) => { + const sizes: Record = { + sm: styles.sm, + md: styles.md, + lg: styles.lg, + xl: styles.xl, + }; + + return classNames(styles.heading, sizes[size], centered && styles.centered); +}; + +export const Heading: React.FC> = React.memo( + ({ as, centered = false, children, className: classNameProp, size = "md", ...remainingProps }) => { + const className = classNames(getHeadingClassNames({ centered, size }), classNameProp); + + return React.createElement(as, { + ...remainingProps, + className, + children, + }); + } +); diff --git a/airbyte-webapp/src/components/ui/Text/heading.module.scss b/airbyte-webapp/src/components/ui/Heading/heading.module.scss similarity index 100% rename from airbyte-webapp/src/components/ui/Text/heading.module.scss rename to airbyte-webapp/src/components/ui/Heading/heading.module.scss diff --git a/airbyte-webapp/src/components/ui/Heading/index.ts b/airbyte-webapp/src/components/ui/Heading/index.ts new file mode 100644 index 000000000000..0776e15b8a78 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Heading/index.ts @@ -0,0 +1 @@ +export { Heading } from "./Heading"; diff --git a/airbyte-webapp/src/components/ui/Modal/Modal.module.scss b/airbyte-webapp/src/components/ui/Modal/Modal.module.scss index 290d12289781..566759f29356 100644 --- a/airbyte-webapp/src/components/ui/Modal/Modal.module.scss +++ b/airbyte-webapp/src/components/ui/Modal/Modal.module.scss @@ -8,15 +8,6 @@ } } -.backdrop { - position: fixed; - top: 0; - left: 0; - right: 0; - bottom: 0; - background: rgba(colors.$black, 0.5); -} - .modalPageContainer { position: absolute; z-index: z-indices.$modal; diff --git a/airbyte-webapp/src/components/ui/Modal/Modal.tsx b/airbyte-webapp/src/components/ui/Modal/Modal.tsx index 857174bb281a..942dadc3ac0b 100644 --- a/airbyte-webapp/src/components/ui/Modal/Modal.tsx +++ b/airbyte-webapp/src/components/ui/Modal/Modal.tsx @@ -3,6 +3,7 @@ import classNames from "classnames"; import React, { useState } from "react"; import { Card } from "../Card"; +import { Overlay } from "../Overlay/Overlay"; import styles from "./Modal.module.scss"; export interface ModalProps { @@ -37,7 +38,7 @@ export const Modal: React.FC> = ({ return ( -
+
{cardless ? ( diff --git a/airbyte-webapp/src/components/ui/Overlay/Overlay.module.scss b/airbyte-webapp/src/components/ui/Overlay/Overlay.module.scss new file mode 100644 index 000000000000..0102f7db41b6 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Overlay/Overlay.module.scss @@ -0,0 +1,10 @@ +@use "scss/colors"; + +.container { + position: fixed; + top: 0; + left: 0; + right: 0; + bottom: 0; + background: rgba(colors.$black, 0.5); +} diff --git a/airbyte-webapp/src/components/ui/Overlay/Overlay.tsx b/airbyte-webapp/src/components/ui/Overlay/Overlay.tsx new file mode 100644 index 000000000000..1d9bdea7f3b8 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Overlay/Overlay.tsx @@ -0,0 +1,3 @@ +import styles from "./Overlay.module.scss"; + +export const Overlay: React.FC = () => diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/calculateInitialCatalog.test.ts b/airbyte-webapp/src/views/Connection/ConnectionForm/calculateInitialCatalog.test.ts index f9323574f5ea..a4d7b5929c89 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/calculateInitialCatalog.test.ts +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/calculateInitialCatalog.test.ts @@ -1,5 +1,5 @@ import { SyncSchema, SyncSchemaStream } from "core/domain/catalog"; -import { DestinationSyncMode, SyncMode } from "core/request/AirbyteClient"; +import { DestinationSyncMode, StreamDescriptor, SyncMode } from "core/request/AirbyteClient"; import calculateInitialCatalog from "./calculateInitialCatalog"; @@ -11,6 +11,7 @@ const mockSyncSchemaStream: SyncSchemaStream = { sourceDefinedPrimaryKey: [["new_primary_key"]], jsonSchema: {}, name: "test", + namespace: "namespace-test", supportedSyncModes: [], }, config: { @@ -535,4 +536,60 @@ describe("calculateInitialCatalog", () => { // cursor field expect(calculatedStreams[0].config?.cursorField).toEqual(config?.cursorField); }); + + it("should calculate optimal sync mode if stream is new", () => { + const { stream: sourceDefinedStream, config } = mockSyncSchemaStream; + + const newStreamDescriptors: StreamDescriptor[] = [{ name: "test", namespace: "namespace-test" }]; + + const { streams: calculatedStreams } = calculateInitialCatalog( + { + streams: [ + { + id: "1", + stream: { + ...sourceDefinedStream, + name: "test", + namespace: "namespace-test", + sourceDefinedCursor: true, + defaultCursorField: ["id"], + supportedSyncModes: [SyncMode.incremental], + }, + config: { + ...config, + destinationSyncMode: DestinationSyncMode.overwrite, + syncMode: SyncMode.incremental, + }, + }, + { + id: "1", + stream: { + ...sourceDefinedStream, + name: "test2", + namespace: "namespace-test", + sourceDefinedCursor: true, + defaultCursorField: ["id"], + supportedSyncModes: [SyncMode.incremental], + }, + config: { + ...config, + destinationSyncMode: DestinationSyncMode.overwrite, + syncMode: SyncMode.incremental, + }, + }, + ], + }, + [DestinationSyncMode.append_dedup], + true, + newStreamDescriptors + ); + + // new stream has its sync mode calculated + expect(calculatedStreams[0].config?.syncMode).toEqual(SyncMode.incremental); + expect(calculatedStreams[0].config?.destinationSyncMode).toEqual(DestinationSyncMode.append_dedup); + + // existing stream remains as-is + expect(calculatedStreams[1].config?.syncMode).toEqual(SyncMode.incremental); + expect(calculatedStreams[1].config?.destinationSyncMode).toEqual(DestinationSyncMode.overwrite); + }); }); diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/calculateInitialCatalog.ts b/airbyte-webapp/src/views/Connection/ConnectionForm/calculateInitialCatalog.ts index dcc2876025d9..983c83eebc22 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/calculateInitialCatalog.ts +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/calculateInitialCatalog.ts @@ -1,5 +1,10 @@ import { SyncSchema, SyncSchemaStream } from "core/domain/catalog"; -import { DestinationSyncMode, SyncMode, AirbyteStreamConfiguration } from "core/request/AirbyteClient"; +import { + DestinationSyncMode, + SyncMode, + AirbyteStreamConfiguration, + StreamDescriptor, +} from "core/request/AirbyteClient"; const getDefaultCursorField = (streamNode: SyncSchemaStream): string[] => { if (streamNode.stream?.defaultCursorField?.length) { @@ -119,18 +124,24 @@ const getOptimalSyncMode = ( const calculateInitialCatalog = ( schema: SyncSchema, supportedDestinationSyncModes: DestinationSyncMode[], - isNotCreateMode?: boolean -): SyncSchema => ({ - streams: schema.streams.map((apiNode, id) => { - const nodeWithId: SyncSchemaStream = { ...apiNode, id: id.toString() }; - const nodeStream = verifySourceDefinedProperties(verifySupportedSyncModes(nodeWithId), isNotCreateMode || false); - - if (isNotCreateMode) { - return nodeStream; - } - - return getOptimalSyncMode(verifyConfigCursorField(nodeStream), supportedDestinationSyncModes); - }), -}); + isNotCreateMode?: boolean, + newStreamDescriptors?: StreamDescriptor[] +): SyncSchema => { + return { + streams: schema.streams.map((apiNode, id) => { + const nodeWithId: SyncSchemaStream = { ...apiNode, id: id.toString() }; + const nodeStream = verifySourceDefinedProperties(verifySupportedSyncModes(nodeWithId), isNotCreateMode || false); + + // if the stream is new since a refresh, we want to verify cursor and get optimal sync modes + const matches = newStreamDescriptors?.some( + (streamId) => streamId.name === nodeStream?.stream?.name && streamId.namespace === nodeStream.stream?.namespace + ); + if (isNotCreateMode && !matches) { + return nodeStream; + } + return getOptimalSyncMode(verifyConfigCursorField(nodeStream), supportedDestinationSyncModes); + }), + }; +}; export default calculateInitialCatalog; diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/components/OperationsSection.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/components/OperationsSection.tsx index 1e5a77f78819..7ca075943f1f 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/components/OperationsSection.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/components/OperationsSection.tsx @@ -3,7 +3,7 @@ import React from "react"; import { useIntl } from "react-intl"; import { Card } from "components/ui/Card"; -import { Text } from "components/ui/Text"; +import { Heading } from "components/ui/Heading"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; import { FeatureItem, useFeature } from "hooks/services/Feature"; @@ -37,14 +37,14 @@ export const OperationsSection: React.FC = ({
{supportsNormalization || supportsTransformations ? ( - + {[ supportsNormalization && formatMessage({ id: "connectionForm.normalization.title" }), supportsTransformations && formatMessage({ id: "connectionForm.transformation.title" }), ] .filter(Boolean) .join(" & ")} - + ) : null} {supportsNormalization && } {supportsTransformations && ( diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/components/Section.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/components/Section.tsx index e9701a09f24b..44d8f4d1025a 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/components/Section.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/components/Section.tsx @@ -1,5 +1,5 @@ import { Card } from "components/ui/Card"; -import { Text } from "components/ui/Text"; +import { Heading } from "components/ui/Heading"; import styles from "./Section.module.scss"; @@ -11,9 +11,9 @@ export const Section: React.FC> = ({ title
{title && ( - + {title} - + )} {children}
diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/components/SyncCatalogField.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/components/SyncCatalogField.tsx index 7218199752ab..801c954db751 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/components/SyncCatalogField.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/components/SyncCatalogField.tsx @@ -3,7 +3,7 @@ import React, { useCallback } from "react"; import { FormattedMessage } from "react-intl"; import { CatalogTree } from "components/connection/CatalogTree"; -import { Text } from "components/ui/Text"; +import { Heading } from "components/ui/Heading"; import { SyncSchemaStream } from "core/domain/catalog"; import { DestinationSyncMode } from "core/request/AirbyteClient"; @@ -39,9 +39,9 @@ const SyncCatalogFieldComponent: React.FC
- + - + {mode !== "readonly" && additionalControl}
diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx index 996988201b41..a8d99c9e4101 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx @@ -27,7 +27,7 @@ import { import { ConnectionFormMode, ConnectionOrPartialConnection } from "hooks/services/ConnectionForm/ConnectionFormService"; import { ValuesProps } from "hooks/services/useConnectionHook"; import { useCurrentWorkspace } from "services/workspaces/WorkspacesService"; -import { validateCronExpression } from "utils/cron"; +import { validateCronExpression, validateCronFrequencyOneHourOrMore } from "utils/cron"; import calculateInitialCatalog from "./calculateInitialCatalog"; @@ -75,7 +75,15 @@ export function useDefaultTransformation(): OperationCreate { }; } -export const connectionValidationSchema = (mode: ConnectionFormMode) => +interface CreateConnectionValidationSchemaArgs { + allowSubOneHourCronExpressions: boolean; + mode: ConnectionFormMode; +} + +export const createConnectionValidationSchema = ({ + mode, + allowSubOneHourCronExpressions, +}: CreateConnectionValidationSchemaArgs) => yup .object({ // The connection name during Editing is handled separately from the form @@ -102,8 +110,14 @@ export const connectionValidationSchema = (mode: ConnectionFormMode) => .object({ cronExpression: yup .string() + .trim() .required("form.empty.error") - .test("validCron", "form.cronExpression.error", validateCronExpression), + .test("validCron", "form.cronExpression.error", validateCronExpression) + .test( + "validCronFrequency", + "form.cronExpression.underOneHourNotAllowed", + (expression) => allowSubOneHourCronExpressions || validateCronFrequencyOneHourOrMore(expression) + ), cronTimeZone: yup.string().required("form.empty.error"), }) .defined("form.empty.error"), @@ -252,14 +266,21 @@ export const useInitialValues = ( destDefinition: DestinationDefinitionSpecificationRead, isNotCreateMode?: boolean ): FormikConnectionFormValues => { + const { catalogDiff } = connection; + + const newStreamDescriptors = catalogDiff?.transforms + .filter((transform) => transform.transformType === "add_stream") + .map((stream) => stream.streamDescriptor); + const initialSchema = useMemo( () => calculateInitialCatalog( connection.syncCatalog, destDefinition?.supportedDestinationSyncModes || [], - isNotCreateMode + isNotCreateMode, + newStreamDescriptors ), - [connection.syncCatalog, destDefinition, isNotCreateMode] + [connection.syncCatalog, destDefinition?.supportedDestinationSyncModes, isNotCreateMode, newStreamDescriptors] ); return useMemo(() => { diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/FormSection.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/FormSection.tsx index faed191650da..8396b9c92447 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/FormSection.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/FormSection.tsx @@ -2,8 +2,8 @@ import React, { useMemo } from "react"; import { FormBlock } from "core/form/types"; -import { useServiceForm } from "../../serviceFormContext"; -import { makeConnectionConfigurationPath, OrderComparator } from "../../utils"; +import { useAuthentication } from "../../useAuthentication"; +import { OrderComparator } from "../../utils"; import { ArraySection } from "./ArraySection"; import { AuthSection } from "./auth/AuthSection"; import { ConditionSection } from "./ConditionSection"; @@ -18,9 +18,7 @@ interface FormNodeProps { const FormNode: React.FC = ({ sectionPath, formField, disabled }) => { if (formField._type === "formGroup") { - return ( - - ); + return ; } else if (formField._type === "formCondition") { return ; } else if (formField._type === "objectArray") { @@ -39,11 +37,12 @@ interface FormSectionProps { blocks: FormBlock[] | FormBlock; path?: string; skipAppend?: boolean; - hasOauth?: boolean; disabled?: boolean; } -export const FormSection: React.FC = ({ blocks = [], path, skipAppend, hasOauth, disabled }) => { +export const FormSection: React.FC = ({ blocks = [], path, skipAppend, disabled }) => { + const { isHiddenAuthField, shouldShowAuthButton } = useAuthentication(); + const sections = useMemo(() => { const flattenedBlocks = [blocks].flat(); @@ -54,29 +53,25 @@ export const FormSection: React.FC = ({ blocks = [], path, ski return flattenedBlocks; }, [blocks]); - const { selectedConnector, isAuthFlowSelected, authFieldsToHide } = useServiceForm(); - return ( <> - {hasOauth && } {sections - .filter( - (formField) => - !formField.airbyte_hidden && - // TODO: check that it is a good idea to add authFieldsToHide - (!isAuthFlowSelected || (isAuthFlowSelected && !authFieldsToHide.includes(formField.path))) - ) + .filter((formField) => !formField.airbyte_hidden && !isHiddenAuthField(formField.path)) .map((formField) => { const sectionPath = path ? (skipAppend ? path : `${path}.${formField.fieldKey}`) : formField.fieldKey; - const isAuthSection = - isAuthFlowSelected && - selectedConnector?.advancedAuth?.predicateKey && - sectionPath === makeConnectionConfigurationPath(selectedConnector?.advancedAuth?.predicateKey); - return ( - {isAuthSection && } + {/* + If the auth button should be rendered here, do so. In addition to the check useAuthentication does + we also need to check if the formField type is not a `formCondition`. We render a lot of OAuth buttons + in conditional fields in which case the path they should be rendered is the path of the conditional itself. + For conditional fields we're rendering this component twice, once "outside" of the conditional, which causes + the actual conditional frame to be rendered and once inside the conditional to render the actual content. + Since we want to only render the auth button inside the conditional and not above it, we filter out the cases + where the formField._type is formCondition, which will be the "outside rendering". + */} + {shouldShowAuthButton(sectionPath) && formField._type !== "formCondition" && } ); diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.test.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.test.tsx index 28a96d6ab612..ae3166b7daf5 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.test.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.test.tsx @@ -3,6 +3,7 @@ import { TestWrapper } from "test-utils/testutils"; import { useFormikOauthAdapter } from "views/Connector/ServiceForm/components/Sections/auth/useOauthFlowAdapter"; import { useServiceForm } from "views/Connector/ServiceForm/serviceFormContext"; +import { useAuthentication } from "views/Connector/ServiceForm/useAuthentication"; import { AuthButton } from "./AuthButton"; jest.setTimeout(10000); @@ -41,18 +42,22 @@ const baseUseServiceFormValues = { selectedService: undefined, }; +jest.mock("views/Connector/ServiceForm/useAuthentication"); +const mockUseAuthentication = useAuthentication as unknown as jest.Mock>; + describe("auth button", () => { beforeEach(() => { jest.clearAllMocks(); + + mockUseAuthentication.mockReturnValue({ hiddenAuthFieldErrors: {} }); }); it("initially renders with correct message and no status message", () => { // no auth errors mockUseServiceForm.mockImplementationOnce(() => { - const authErrors = {}; - const { selectedConnector, allowOAuthConnector, selectedService } = baseUseServiceFormValues; + const { selectedConnector, selectedService } = baseUseServiceFormValues; - return { authErrors, selectedConnector, allowOAuthConnector, selectedService }; + return { selectedConnector, selectedService }; }); // not done @@ -85,10 +90,9 @@ describe("auth button", () => { it("after successful authentication, it renders with correct message and success message", () => { // no auth errors mockUseServiceForm.mockImplementationOnce(() => { - const authErrors = {}; - const { selectedConnector, allowOAuthConnector, selectedService } = baseUseServiceFormValues; + const { selectedConnector, selectedService } = baseUseServiceFormValues; - return { authErrors, selectedConnector, allowOAuthConnector, selectedService }; + return { selectedConnector, selectedService }; }); // done @@ -114,13 +118,14 @@ describe("auth button", () => { expect(successMessage).toBeInTheDocument(); }); - it("if authError is true, it renders the correct message", () => { + it("renders an error if there are any auth fields with empty values", () => { // auth errors + mockUseAuthentication.mockReturnValue({ hiddenAuthFieldErrors: { field: "form.empty.error" } }); + mockUseServiceForm.mockImplementationOnce(() => { - const authErrors = { field: "form.empty.error" }; - const { selectedConnector, allowOAuthConnector, selectedService } = baseUseServiceFormValues; + const { selectedConnector, selectedService } = baseUseServiceFormValues; - return { authErrors, selectedConnector, allowOAuthConnector, selectedService }; + return { selectedConnector, selectedService }; }); // not done diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.tsx index 9ff34bfd9c61..b2ad1869d7eb 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.tsx @@ -8,6 +8,7 @@ import { Text } from "components/ui/Text"; import { ConnectorSpecification } from "core/domain/connector"; import { useServiceForm } from "../../../serviceFormContext"; +import { useAuthentication } from "../../../useAuthentication"; import styles from "./AuthButton.module.scss"; import GoogleAuthButton from "./GoogleAuthButton"; import { useFormikOauthAdapter } from "./useOauthFlowAdapter"; @@ -21,6 +22,7 @@ function isGoogleConnector(connectorDefinitionId: string): boolean { "71607ba1-c0ac-4799-8049-7f4b90dd50f7", // google sheets source "a4cbd2d1-8dbe-4818-b8bc-b90ad782d12a", // google sheets destination "ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734", // google workspace admin reports + "afa734e4-3571-11ec-991a-1e0031268139", // YouTube analytics ].includes(connectorDefinitionId); } @@ -39,8 +41,9 @@ function getAuthenticateMessageId(connectorDefinitionId: string): string { } export const AuthButton: React.FC = () => { - const { selectedService, authErrors, selectedConnector } = useServiceForm(); - const hasAuthError = Object.values(authErrors).includes("form.empty.error"); + const { selectedService, selectedConnector } = useServiceForm(); + const { hiddenAuthFieldErrors } = useAuthentication(); + const authRequiredError = Object.values(hiddenAuthFieldErrors).includes("form.empty.error"); // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const { loading, done, run, hasRun } = useFormikOauthAdapter(selectedConnector!); @@ -54,8 +57,8 @@ export const AuthButton: React.FC = () => { const Component = getButtonComponent(definitionId); const messageStyle = classnames(styles.message, { - [styles.error]: hasAuthError, - [styles.success]: !hasAuthError, + [styles.error]: authRequiredError, + [styles.success]: !authRequiredError, }); const buttonLabel = done ? ( @@ -72,7 +75,7 @@ export const AuthButton: React.FC = () => { )} - {hasAuthError && ( + {authRequiredError && ( diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/serviceFormContext.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/serviceFormContext.tsx index 820be4fc64e8..decf91db9bbf 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/serviceFormContext.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/serviceFormContext.tsx @@ -1,13 +1,11 @@ -import { getIn, useFormikContext } from "formik"; +import { useFormikContext } from "formik"; import React, { useContext, useMemo } from "react"; import { AnySchema } from "yup"; import { Connector, ConnectorDefinition, ConnectorDefinitionSpecification } from "core/domain/connector"; import { WidgetConfigMap } from "core/form/types"; -import { FeatureItem, useFeature } from "hooks/services/Feature"; import { ServiceFormValues } from "./types"; -import { makeConnectionConfigurationPath, serverProvidedOauthPaths } from "./utils"; interface ServiceFormContext { formType: "source" | "destination"; @@ -22,10 +20,7 @@ interface ServiceFormContext { selectedConnector?: ConnectorDefinitionSpecification; isLoadingSchema?: boolean; isEditMode?: boolean; - isAuthFlowSelected?: boolean; - authFieldsToHide: string[]; validationSchema: AnySchema; - authErrors: Record; } const serviceFormContext = React.createContext(null); @@ -64,9 +59,7 @@ export const ServiceFormContextProvider: React.FC { - const { values, resetForm, getFieldMeta, submitCount } = useFormikContext(); - - const allowOAuthConnector = useFeature(FeatureItem.AllowOAuthConnector); + const { values, resetForm } = useFormikContext(); const { serviceType } = values; const selectedService = useMemo( @@ -74,42 +67,10 @@ export const ServiceFormContextProvider: React.FC - allowOAuthConnector && - selectedConnector?.advancedAuth && - selectedConnector?.advancedAuth.predicateValue === - getIn(getValues(values), makeConnectionConfigurationPath(selectedConnector?.advancedAuth.predicateKey ?? [])), - [selectedConnector, allowOAuthConnector, values, getValues] - ); - - const authFieldsToHide = useMemo( - () => - isAuthFlowSelected - ? Object.values(serverProvidedOauthPaths(selectedConnector)).map((f) => - makeConnectionConfigurationPath(f.path_in_connector_config) - ) - : [], - [selectedConnector, isAuthFlowSelected] - ); - - const authErrors = useMemo(() => { - // key of field path, value of error code - return authFieldsToHide.reduce>((authErrors, fieldName) => { - const { error } = getFieldMeta(fieldName); - if (submitCount > 0 && error) { - authErrors[fieldName] = error; - } - return authErrors; - }, {}); - }, [authFieldsToHide, getFieldMeta, submitCount]); const ctx = useMemo(() => { const unfinishedFlows = widgetsInfo["_common.unfinishedFlows"] ?? {}; return { - authErrors, widgetsInfo, - isAuthFlowSelected, - authFieldsToHide, getValues, setUiWidgetsInfo, selectedService, @@ -135,10 +96,7 @@ export const ServiceFormContextProvider: React.FC ({ + ...jest.requireActual("formik"), + useFormikContext: jest.fn(), +})); + +const mockServiceForm = useServiceForm as unknown as jest.Mock>>; +const mockFormikContext = useFormikContext as unknown as jest.Mock>>; + +interface MockParams { + connector: Pick; + values: unknown; + submitCount?: number; + fieldMeta?: Record; +} + +const mockContext = ({ connector, values, submitCount, fieldMeta = {} }: MockParams) => { + mockFormikContext.mockReturnValue({ + values, + submitCount: submitCount ?? 0, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + getFieldMeta: (field) => (fieldMeta[field] ?? {}) as any, + }); + mockServiceForm.mockReturnValue({ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + selectedConnector: { ...connector, sourceDefinitionId: "12345", jobInfo: {} as any }, + }); +}; + +const useAuthentication = (withOauthFeature = true) => { + // eslint-disable-next-line react-hooks/rules-of-hooks + const { result } = renderHook(() => useAuthenticationHook(), { + wrapper: ({ children }) => ( + {children} + ), + }); + return result.current; +}; + +describe("useAuthentication", () => { + it("should return empty results for non OAuth connectors", () => { + mockContext({ connector: {}, values: {} }); + const result = useAuthentication(); + expect(result.hiddenAuthFieldErrors).toEqual({}); + expect(result.shouldShowAuthButton("field")).toBe(false); + expect(result.isHiddenAuthField("field")).toBe(false); + }); + + it("should not handle auth specifically if OAuth feature is disabled", () => { + mockContext({ + connector: { advancedAuth: predicateInsideConditional }, + values: { connectionConfiguration: { credentials: { auth_type: "oauth2.0" } } }, + }); + const result = useAuthentication(false); + expect(result.isHiddenAuthField(makeConnectionConfigurationPath(["credentials", "access_token"]))).toBe(false); + expect(result.shouldShowAuthButton(makeConnectionConfigurationPath(["credentials", "auth_type"]))).toBe(false); + }); + + describe("for advancedAuth connectors", () => { + describe("without a predicateKey", () => { + it("should calculate hiddenAuthFields correctly", () => { + mockContext({ connector: { advancedAuth: noPredicateAdvancedAuth }, values: {} }); + const result = useAuthentication(); + expect(result.isHiddenAuthField(makeConnectionConfigurationPath(["access_token"]))).toBe(true); + expect(result.isHiddenAuthField(makeConnectionConfigurationPath(["client_id"]))).toBe(false); + expect(result.isHiddenAuthField(makeConnectionConfigurationPath(["client_secret"]))).toBe(false); + }); + + it("should show the auth button on the root level", () => { + mockContext({ connector: { advancedAuth: noPredicateAdvancedAuth }, values: {} }); + const result = useAuthentication(); + expect(result.shouldShowAuthButton(makeConnectionConfigurationPath())).toBe(true); + }); + + it("should not return authErrors before submitting", () => { + const accessTokenField = makeConnectionConfigurationPath(["access_token"]); + mockContext({ + connector: { advancedAuth: noPredicateAdvancedAuth }, + values: {}, + fieldMeta: { [accessTokenField]: { error: "form.empty.error" } }, + submitCount: 0, + }); + const result = useAuthentication(); + expect(result.hiddenAuthFieldErrors).toEqual({}); + }); + + it("should return existing authErrors if submitted once", () => { + const accessTokenField = makeConnectionConfigurationPath(["access_token"]); + mockContext({ + connector: { advancedAuth: noPredicateAdvancedAuth }, + values: {}, + fieldMeta: { [accessTokenField]: { error: "form.empty.error" } }, + submitCount: 1, + }); + const result = useAuthentication(); + expect(result.hiddenAuthFieldErrors).toEqual({ [accessTokenField]: "form.empty.error" }); + }); + }); + + describe("with predicateKey inside conditional", () => { + it("should hide auth fields when predicate value matches", () => { + mockContext({ + connector: { advancedAuth: predicateInsideConditional }, + values: { connectionConfiguration: { credentials: { auth_type: "oauth2.0" } } }, + }); + const result = useAuthentication(); + expect(result.isHiddenAuthField(makeConnectionConfigurationPath(["credentials", "access_token"]))).toBe(true); + expect(result.isHiddenAuthField(makeConnectionConfigurationPath(["credentials", "client_id"]))).toBe(true); + expect(result.isHiddenAuthField(makeConnectionConfigurationPath(["credentials", "client_secret"]))).toBe(true); + }); + + it("should not hide auth fields when predicate value is a mismatch", () => { + mockContext({ + connector: { advancedAuth: predicateInsideConditional }, + values: { connectionConfiguration: { credentials: { auth_type: "token" } } }, + }); + const result = useAuthentication(); + expect(result.isHiddenAuthField(makeConnectionConfigurationPath(["credentials", "access_token"]))).toBe(false); + expect(result.isHiddenAuthField(makeConnectionConfigurationPath(["credentials", "client_id"]))).toBe(false); + expect(result.isHiddenAuthField(makeConnectionConfigurationPath(["credentials", "client_secret"]))).toBe(false); + }); + + it("should show the auth button inside the conditional if right option is selected", () => { + mockContext({ + connector: { advancedAuth: predicateInsideConditional }, + values: { connectionConfiguration: { credentials: { auth_type: "oauth2.0" } } }, + }); + const result = useAuthentication(); + expect(result.shouldShowAuthButton(makeConnectionConfigurationPath(["credentials", "auth_type"]))).toBe(true); + }); + + it("shouldn't show the auth button if the wrong conditional option is selected", () => { + mockContext({ + connector: { advancedAuth: predicateInsideConditional }, + values: { connectionConfiguration: { credentials: { auth_type: "token" } } }, + }); + const result = useAuthentication(); + expect(result.shouldShowAuthButton(makeConnectionConfigurationPath(["credentials", "auth_type"]))).toBe(false); + }); + + it("should not return authErrors before submitting", () => { + const accessTokenField = makeConnectionConfigurationPath(["credentials", "access_token"]); + const clientIdField = makeConnectionConfigurationPath(["credentials", "client_id"]); + mockContext({ + connector: { advancedAuth: predicateInsideConditional }, + values: { connectionConfiguration: { credentials: { auth_type: "oauth2.0" } } }, + fieldMeta: { [accessTokenField]: { error: "form.empty.error" }, [clientIdField]: { error: "another.error" } }, + submitCount: 0, + }); + const result = useAuthentication(); + expect(result.hiddenAuthFieldErrors).toEqual({}); + }); + + it("should return authErrors when conditional has correct option selected", () => { + const accessTokenField = makeConnectionConfigurationPath(["credentials", "access_token"]); + const clientIdField = makeConnectionConfigurationPath(["credentials", "client_id"]); + mockContext({ + connector: { advancedAuth: predicateInsideConditional }, + values: { connectionConfiguration: { credentials: { auth_type: "oauth2.0" } } }, + fieldMeta: { [accessTokenField]: { error: "form.empty.error" }, [clientIdField]: { error: "another.error" } }, + submitCount: 1, + }); + const result = useAuthentication(); + expect(result.hiddenAuthFieldErrors).toEqual({ + [accessTokenField]: "form.empty.error", + [clientIdField]: "another.error", + }); + }); + + it("should not return authErrors when conditional has the incorrect option selected", () => { + const accessTokenField = makeConnectionConfigurationPath(["credentials", "access_token"]); + const clientIdField = makeConnectionConfigurationPath(["credentials", "client_id"]); + mockContext({ + connector: { advancedAuth: predicateInsideConditional }, + values: { connectionConfiguration: { credentials: { auth_type: "token" } } }, + fieldMeta: { [accessTokenField]: { error: "form.empty.error" }, [clientIdField]: { error: "another.error" } }, + submitCount: 1, + }); + const result = useAuthentication(); + expect(result.hiddenAuthFieldErrors).toEqual({}); + }); + }); + }); +}); diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/useAuthentication.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/useAuthentication.tsx new file mode 100644 index 000000000000..b863bc4b5ead --- /dev/null +++ b/airbyte-webapp/src/views/Connector/ServiceForm/useAuthentication.tsx @@ -0,0 +1,210 @@ +import { getIn, useFormikContext } from "formik"; +import { JSONSchema7 } from "json-schema"; +import { useCallback, useMemo } from "react"; + +import { FeatureItem, useFeature } from "hooks/services/Feature"; + +import { useServiceForm } from "./serviceFormContext"; +import { ServiceFormValues } from "./types"; +import { makeConnectionConfigurationPath, serverProvidedOauthPaths } from "./utils"; + +type Path = Array; + +const isNumerical = (input: string | number): boolean => { + return typeof input === "number" || /^\d+$/.test(input); +}; + +/** + * Takes an array of strings or numbers and remove all elements from it that are either + * a number or a string that just contains a number. This will be used to remove index + * accessors into oneOf from paths, since they are not part of the field path later. + */ +const stripNumericalEntries = (paths: Path): string[] => { + return paths.filter((p): p is string => !isNumerical(p)); +}; + +/** + * Takes a list of paths in an array representation as well as a root path in array representation, concats + * them as well as prefix them with the `connectionConfiguration` prefix that Formik uses for all connector + * parameter values, and joins them to string paths. + */ +const convertAndPrefixPaths = (paths?: Path[], rootPath: Path = []): string[] => { + return ( + paths?.map((pathParts) => { + return makeConnectionConfigurationPath([...stripNumericalEntries(rootPath), ...stripNumericalEntries(pathParts)]); + }) ?? [] + ); +}; + +/** + * Returns true if the auth button should be shown for an advancedAuth specification. + * This will check if the connector has a predicateKey, and if so, check if the current form value + * of the corresponding field matches the predicateValue from the specification. + */ +const shouldShowButtonForAdvancedAuth = ( + predicateKey: string[] | undefined, + predicateValue: string | undefined, + values: ServiceFormValues +): boolean => { + return ( + !predicateKey || + predicateKey.length === 0 || + predicateValue === getIn(values, makeConnectionConfigurationPath(predicateKey)) + ); +}; + +/** + * Returns true if the auth button should be shown for an authSpecification connector. + */ +const shouldShowButtonForLegacyAuth = ( + spec: JSONSchema7, + rootPath: Path, + values: ServiceFormValues +): boolean => { + if (!rootPath.some((p) => isNumerical(p))) { + // If the root path of the auth parameters (which is also the place the button will be rendered) + // is not inside a conditional, i.e. none of the root path is a numerical value, we will always + // show the button. + return true; + } + + // If the spec had a root path inside a conditional, e.g. `credentials.0`, we need to figure + // out if that conditional is currently on the correct selected option. Unlike `advancedAuth` + // which has a `predicateValue`, the legacy auth configuration doesn't have the value for the conditional, + // so we need to find that ourselves first. + + // To find the path inside the connector spec that matches the `rootPath` we'll need to insert `properties` + // and `oneOf`, since they'll appear in the JSONSchema, e.g. this turns `credentials.0` to `properties.credentials.oneOf.0` + const specPath = rootPath.flatMap((path) => + isNumerical(path) ? ["oneOf", String(path)] : ["properties", String(path)] + ); + // Get the part of the spec that `rootPath` point to + const credentialsSpecRoot = getIn(spec, specPath) as JSONSchema7 | undefined; + + if (!credentialsSpecRoot?.properties) { + // if the path doesn't exist in the spec (which should not happen) we just show the auth button always. + return true; + } + + // To find the value we're expecting, we run through all properties inside that matching spec inside the conditional + // to find the one that has a `const` value in it, since this is the actual value that will be written into the conditional + // field itself once it's selected. + const constProperty = Object.entries(credentialsSpecRoot.properties) + .map(([key, prop]) => [key, typeof prop !== "boolean" ? prop.const : undefined] as const) + .find(([, constValue]) => !!constValue); + + // If none of the conditional properties is a const value, we'll also show the auth button always (should not happen) + if (!constProperty) { + return true; + } + + // Check if the value in the form matches the found `const` value from the spec. If so we know the conditional + // is on the right option. + const [key, constValue] = constProperty; + const value = getIn(values, makeConnectionConfigurationPath(stripNumericalEntries([...rootPath, key]))); + return value === constValue; +}; + +interface AuthenticationHook { + /** + * Returns whether a given field path should be hidden, because it's part of the + * OAuth flow and will be filled in by that. + */ + isHiddenAuthField: (fieldPath: string) => boolean; + /** + * A record of all formik errors in hidden authentication fields. The key will be the + * name of the field and the value an error code. If no error is present in a field + * it will be missing from this object. + */ + hiddenAuthFieldErrors: Record; + /** + * This will return true if the auth button should be visible and rendered in the place of + * the passed in field, and false otherwise. + */ + shouldShowAuthButton: (fieldPath: string) => boolean; +} + +export const useAuthentication = (): AuthenticationHook => { + const { values, getFieldMeta, submitCount } = useFormikContext(); + const { selectedConnector } = useServiceForm(); + + const allowOAuthConnector = useFeature(FeatureItem.AllowOAuthConnector); + + const advancedAuth = selectedConnector?.advancedAuth; + const legacyOauthSpec = selectedConnector?.authSpecification?.oauth2Specification; + + const spec = selectedConnector?.connectionSpecification as JSONSchema7; + + const isAuthButtonVisible = useMemo(() => { + const shouldShowAdvancedAuth = + advancedAuth && shouldShowButtonForAdvancedAuth(advancedAuth.predicateKey, advancedAuth.predicateValue, values); + const shouldShowLegacyAuth = + legacyOauthSpec && shouldShowButtonForLegacyAuth(spec, legacyOauthSpec.rootObject as Path, values); + return Boolean(allowOAuthConnector && (shouldShowAdvancedAuth || shouldShowLegacyAuth)); + }, [values, advancedAuth, legacyOauthSpec, spec, allowOAuthConnector]); + + // Fields that are filled by the OAuth flow and thus won't need to be shown in the UI if OAuth is available + const implicitAuthFieldPaths = useMemo( + () => [ + // Fields from `advancedAuth` connectors + ...(advancedAuth + ? Object.values(serverProvidedOauthPaths(selectedConnector)).map((f) => + makeConnectionConfigurationPath(f.path_in_connector_config) + ) + : []), + // Fields from legacy `authSpecification` connectors + ...(legacyOauthSpec + ? [ + ...convertAndPrefixPaths(legacyOauthSpec.oauthFlowInitParameters, legacyOauthSpec.rootObject as Path), + ...convertAndPrefixPaths(legacyOauthSpec.oauthFlowOutputParameters, legacyOauthSpec.rootObject as Path), + ] + : []), + ], + [advancedAuth, legacyOauthSpec, selectedConnector] + ); + + const isHiddenAuthField = useCallback( + (fieldPath: string) => { + // A field should be hidden due to OAuth if we have OAuth enabled and selected (in case it's inside a oneOf) + // and the field is part of the OAuth flow parameters. + return isAuthButtonVisible && implicitAuthFieldPaths.includes(fieldPath); + }, + [implicitAuthFieldPaths, isAuthButtonVisible] + ); + + const hiddenAuthFieldErrors = useMemo(() => { + if (!isAuthButtonVisible) { + // We don't want to return the errors if the auth button isn't visible. + return {}; + } + return implicitAuthFieldPaths.reduce>((authErrors, fieldName) => { + const { error } = getFieldMeta(fieldName); + if (submitCount > 0 && error) { + authErrors[fieldName] = error; + } + return authErrors; + }, {}); + }, [getFieldMeta, implicitAuthFieldPaths, isAuthButtonVisible, submitCount]); + + const shouldShowAuthButton = useCallback( + (fieldPath: string) => { + if (!isAuthButtonVisible) { + // Never show the auth button anywhere if its not enabled or visible (inside a conditional that's not selected) + return false; + } + + const path = advancedAuth + ? advancedAuth.predicateKey && makeConnectionConfigurationPath(advancedAuth.predicateKey) + : legacyOauthSpec && makeConnectionConfigurationPath(stripNumericalEntries(legacyOauthSpec.rootObject as Path)); + + return fieldPath === (path ?? makeConnectionConfigurationPath()); + }, + [advancedAuth, isAuthButtonVisible, legacyOauthSpec] + ); + + return { + isHiddenAuthField, + hiddenAuthFieldErrors, + shouldShowAuthButton, + }; +}; diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/useBuildForm.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/useBuildForm.tsx index 2bcb99677ea5..d3d0bef42c3f 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/useBuildForm.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/useBuildForm.tsx @@ -8,57 +8,17 @@ import { AnySchema } from "yup"; import { ConnectorDefinitionSpecification } from "core/domain/connector"; import { FormBlock, WidgetConfig, WidgetConfigMap } from "core/form/types"; import { buildPathInitialState } from "core/form/uiWidget"; -import { applyFuncAt, removeNestedPaths } from "core/jsonSchema"; import { jsonSchemaToUiWidget } from "core/jsonSchema/schemaToUiWidget"; import { buildYupFormForJsonSchema } from "core/jsonSchema/schemaToYup"; -import { FeatureItem, useFeature } from "hooks/services/Feature"; -import { DestinationDefinitionSpecificationRead } from "../../../core/request/AirbyteClient"; import { ServiceFormValues } from "./types"; -function upgradeSchemaLegacyAuth( - connectorSpecification: Required< - Pick - > -) { - const spec = connectorSpecification.authSpecification.oauth2Specification; - return applyFuncAt( - connectorSpecification.connectionSpecification as JSONSchema7Definition, - (spec?.rootObject ?? []) as Array, - (schema) => { - // Very hacky way to allow placing button within section - // @ts-expect-error json schema - schema.is_auth = true; - const schemaWithoutPaths = removeNestedPaths(schema, spec?.oauthFlowInitParameters ?? [], false); - - const schemaWithoutOutputPats = removeNestedPaths( - schemaWithoutPaths, - spec?.oauthFlowOutputParameters ?? [], - false - ); - - return schemaWithoutOutputPats; - } - ); -} - export function useBuildInitialSchema( connectorSpecification?: ConnectorDefinitionSpecification ): JSONSchema7Definition | undefined { - const allowOAuthConnector = useFeature(FeatureItem.AllowOAuthConnector); - return useMemo(() => { - if (allowOAuthConnector) { - if (connectorSpecification?.authSpecification && !connectorSpecification?.advancedAuth) { - return upgradeSchemaLegacyAuth({ - connectionSpecification: connectorSpecification?.connectionSpecification, - authSpecification: connectorSpecification.authSpecification, - }); - } - } - return connectorSpecification?.connectionSpecification as JSONSchema7Definition | undefined; - }, [allowOAuthConnector, connectorSpecification]); + }, [connectorSpecification]); } export interface BuildFormHook { diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/utils.ts b/airbyte-webapp/src/views/Connector/ServiceForm/utils.ts index e3645eabbe66..c9dd2ac272d2 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/utils.ts +++ b/airbyte-webapp/src/views/Connector/ServiceForm/utils.ts @@ -3,8 +3,8 @@ import { naturalComparator } from "utils/objects"; import { ConnectorDefinitionSpecification } from "../../../core/domain/connector"; -export function makeConnectionConfigurationPath(path: string[]): string { - return `connectionConfiguration.${path.join(".")}`; +export function makeConnectionConfigurationPath(path: string[] = []): string { + return ["connectionConfiguration", ...path].join("."); } type OAuthOutputSpec = { properties: Record } | undefined; diff --git a/airbyte-webapp/src/views/common/ErrorOccurredView/ErrorOccurredView.tsx b/airbyte-webapp/src/views/common/ErrorOccurredView/ErrorOccurredView.tsx index a715f3af23ec..cc6a3a6e2a1e 100644 --- a/airbyte-webapp/src/views/common/ErrorOccurredView/ErrorOccurredView.tsx +++ b/airbyte-webapp/src/views/common/ErrorOccurredView/ErrorOccurredView.tsx @@ -2,7 +2,7 @@ import React from "react"; import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; -import { Text } from "components/ui/Text"; +import { Heading } from "components/ui/Heading"; import styles from "./ErrorOccurredView.module.scss"; @@ -17,9 +17,9 @@ export const ErrorOccurredView: React.FC = ({ message, o
- + - +

{message}

{onCtaButtonClick && ctaButtonText && (
-
-
- Up - 
post /v1/state/create_or_update
-
Create or update the state for a connection. (createOrUpdateState)
-
- - -

Consumes

- This API call consumes the following media types via the Content-Type request header: -
    -
  • application/json
    • -
- -

Request body

-
-
ConnectionStateCreateOrUpdate ConnectionStateCreateOrUpdate (required)
- -
Body Parameter —
- -
- - - - -

Return type

-
- ConnectionState - -
- - - -

Example data

-
Content-Type: application/json
- 
{
-  "globalState" : {
-    "streamStates" : [ {
-      "streamDescriptor" : {
-        "name" : "name",
-        "namespace" : "namespace"
-      }
-    }, {
-      "streamDescriptor" : {
-        "name" : "name",
-        "namespace" : "namespace"
-      }
-    } ]
-  },
-  "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "streamState" : [ {
-    "streamDescriptor" : {
-      "name" : "name",
-      "namespace" : "namespace"
-    }
-  }, {
-    "streamDescriptor" : {
-      "name" : "name",
-      "namespace" : "namespace"
-    }
-  } ]
-}
- -

Produces

- This API call produces the following media types according to the Accept request header; - the media type will be conveyed by the Content-Type response header. -
    -
  • application/json
    • -
- -

Responses

-

200

- Successful operation - ConnectionState -

404

- Object with given id was not found. - NotFoundKnownExceptionInfo -

422

- Input failed validation - InvalidInputExceptionInfo -
-
Up @@ -803,144 +726,6 @@

422

InvalidInputExceptionInfo
-
-
- Up - 
post /v1/state/get
-
Fetch the current state for a connection. (getState)
-
- - -

Consumes

- This API call consumes the following media types via the Content-Type request header: -
    -
  • application/json
    • -
- -

Request body

-
-
ConnectionIdRequestBody ConnectionIdRequestBody (required)
- -
Body Parameter —
- -
- - - - -

Return type

-
- ConnectionState - -
- - - -

Example data

-
Content-Type: application/json
- 
{
-  "globalState" : {
-    "streamStates" : [ {
-      "streamDescriptor" : {
-        "name" : "name",
-        "namespace" : "namespace"
-      }
-    }, {
-      "streamDescriptor" : {
-        "name" : "name",
-        "namespace" : "namespace"
-      }
-    } ]
-  },
-  "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "streamState" : [ {
-    "streamDescriptor" : {
-      "name" : "name",
-      "namespace" : "namespace"
-    }
-  }, {
-    "streamDescriptor" : {
-      "name" : "name",
-      "namespace" : "namespace"
-    }
-  } ]
-}
- -

Produces

- This API call produces the following media types according to the Accept request header; - the media type will be conveyed by the Content-Type response header. -
    -
  • application/json
    • -
- -

Responses

-

200

- Successful operation - ConnectionState -

404

- Object with given id was not found. - NotFoundKnownExceptionInfo -

422

- Input failed validation - InvalidInputExceptionInfo -
-
-
-
- Up - 
post /v1/web_backend/state/get_type
-
Fetch the current state type for a connection. (getStateType)
-
- - -

Consumes

- This API call consumes the following media types via the Content-Type request header: -
    -
  • application/json
    • -
- -

Request body

-
-
ConnectionIdRequestBody ConnectionIdRequestBody (required)
- -
Body Parameter —
- -
- - - - -

Return type

-
- ConnectionStateType - -
- - - -

Example data

-
Content-Type: application/json
- 
null
- -

Produces

- This API call produces the following media types according to the Accept request header; - the media type will be conveyed by the Content-Type response header. -
    -
  • application/json
    • -
- -

Responses

-

200

- Successful operation - ConnectionStateType -

404

- Object with given id was not found. - NotFoundKnownExceptionInfo -

422

- Input failed validation - InvalidInputExceptionInfo -
-
Up @@ -4124,6 +3909,68 @@

422

InvalidInputExceptionInfo
+
+
+ Up + 
post /v1/jobs/get_normalization_status
+
Get normalization status to determine if we can bypass normalization phase (getAttemptNormalizationStatusesForJob)
+
+ + +

Consumes

+ This API call consumes the following media types via the Content-Type request header: +
    +
  • application/json
    • +
+ +

Request body

+
+
JobIdRequestBody JobIdRequestBody (optional)
+ +
Body Parameter —
+ +
+ + + + +

Return type

+
+ AttemptNormalizationStatusReadList + +
+ + + +

Example data

+
Content-Type: application/json
+ 
{
+  "attemptNormalizationStatuses" : [ {
+    "attemptNumber" : 0,
+    "recordsCommitted" : 6,
+    "hasRecordsCommitted" : true,
+    "hasNormalizationFailed" : true
+  }, {
+    "attemptNumber" : 0,
+    "recordsCommitted" : 6,
+    "hasRecordsCommitted" : true,
+    "hasNormalizationFailed" : true
+  } ]
+}
+ +

Produces

+ This API call produces the following media types according to the Accept request header; + the media type will be conveyed by the Content-Type response header. +
    +
  • application/json
    • +
+ +

Responses

+

200

+ Successful operation + AttemptNormalizationStatusReadList +
+
Up @@ -4343,13 +4190,75 @@

Produces

Responses

200

Successful operation - JobInfoRead -

404

- Object with given id was not found. - NotFoundKnownExceptionInfo -

422

- Input failed validation - InvalidInputExceptionInfo + JobInfoRead +

404

+ Object with given id was not found. + NotFoundKnownExceptionInfo +

422

+ Input failed validation + InvalidInputExceptionInfo +
+
+
+
+ Up + 
post /v1/jobs/get_normalization_status
+
Get normalization status to determine if we can bypass normalization phase (getAttemptNormalizationStatusesForJob)
+
+ + +

Consumes

+ This API call consumes the following media types via the Content-Type request header: +
    +
  • application/json
    • +
+ +

Request body

+
+
JobIdRequestBody JobIdRequestBody (optional)
+ +
Body Parameter —
+ +
+ + + + +

Return type

+
+ AttemptNormalizationStatusReadList + +
+ + + +

Example data

+
Content-Type: application/json
+ 
{
+  "attemptNormalizationStatuses" : [ {
+    "attemptNumber" : 0,
+    "recordsCommitted" : 6,
+    "hasRecordsCommitted" : true,
+    "hasNormalizationFailed" : true
+  }, {
+    "attemptNumber" : 0,
+    "recordsCommitted" : 6,
+    "hasRecordsCommitted" : true,
+    "hasNormalizationFailed" : true
+  } ]
+}
+ +

Produces

+ This API call produces the following media types according to the Accept request header; + the media type will be conveyed by the Content-Type response header. +
    +
  • application/json
    • +
+ +

Responses

+

200

+ Successful operation + AttemptNormalizationStatusReadList
@@ -8244,7 +8153,228 @@

422

InvalidInputExceptionInfo
+

State

+
+
+ Up + 
post /v1/state/create_or_update
+
Create or update the state for a connection. (createOrUpdateState)
+
+ + +

Consumes

+ This API call consumes the following media types via the Content-Type request header: +
    +
  • application/json
    • +
+ +

Request body

+
+
ConnectionStateCreateOrUpdate ConnectionStateCreateOrUpdate (required)
+ +
Body Parameter —
+ +
+ + + + +

Return type

+
+ ConnectionState + +
+ + + +

Example data

+
Content-Type: application/json
+ 
{
+  "globalState" : {
+    "streamStates" : [ {
+      "streamDescriptor" : {
+        "name" : "name",
+        "namespace" : "namespace"
+      }
+    }, {
+      "streamDescriptor" : {
+        "name" : "name",
+        "namespace" : "namespace"
+      }
+    } ]
+  },
+  "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "streamState" : [ {
+    "streamDescriptor" : {
+      "name" : "name",
+      "namespace" : "namespace"
+    }
+  }, {
+    "streamDescriptor" : {
+      "name" : "name",
+      "namespace" : "namespace"
+    }
+  } ]
+}
+ +

Produces

+ This API call produces the following media types according to the Accept request header; + the media type will be conveyed by the Content-Type response header. +
    +
  • application/json
    • +
+ +

Responses

+

200

+ Successful operation + ConnectionState +

404

+ Object with given id was not found. + NotFoundKnownExceptionInfo +

422

+ Input failed validation + InvalidInputExceptionInfo +
+
+
+
+ Up + 
post /v1/state/get
+
Fetch the current state for a connection. (getState)
+
+ + +

Consumes

+ This API call consumes the following media types via the Content-Type request header: +
    +
  • application/json
    • +
+ +

Request body

+
+
ConnectionIdRequestBody ConnectionIdRequestBody (required)
+ +
Body Parameter —
+ +
+ + + + +

Return type

+
+ ConnectionState + +
+ + + +

Example data

+
Content-Type: application/json
+ 
{
+  "globalState" : {
+    "streamStates" : [ {
+      "streamDescriptor" : {
+        "name" : "name",
+        "namespace" : "namespace"
+      }
+    }, {
+      "streamDescriptor" : {
+        "name" : "name",
+        "namespace" : "namespace"
+      }
+    } ]
+  },
+  "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "streamState" : [ {
+    "streamDescriptor" : {
+      "name" : "name",
+      "namespace" : "namespace"
+    }
+  }, {
+    "streamDescriptor" : {
+      "name" : "name",
+      "namespace" : "namespace"
+    }
+  } ]
+}
+ +

Produces

+ This API call produces the following media types according to the Accept request header; + the media type will be conveyed by the Content-Type response header. +
    +
  • application/json
    • +
+ +

Responses

+

200

+ Successful operation + ConnectionState +

404

+ Object with given id was not found. + NotFoundKnownExceptionInfo +

422

+ Input failed validation + InvalidInputExceptionInfo +
+

WebBackend

+
+
+ Up + 
post /v1/web_backend/state/get_type
+
Fetch the current state type for a connection. (getStateType)
+
+ + +

Consumes

+ This API call consumes the following media types via the Content-Type request header: +
    +
  • application/json
    • +
+ +

Request body

+
+
ConnectionIdRequestBody ConnectionIdRequestBody (required)
+ +
Body Parameter —
+ +
+ + + + +

Return type

+
+ ConnectionStateType + +
+ + + +

Example data

+
Content-Type: application/json
+ 
null
+ +

Produces

+ This API call produces the following media types according to the Accept request header; + the media type will be conveyed by the Content-Type response header. +
    +
  • application/json
    • +
+ +

Responses

+

200

+ Successful operation + ConnectionStateType +

404

+ Object with given id was not found. + NotFoundKnownExceptionInfo +

422

+ Input failed validation + InvalidInputExceptionInfo +
+
Up @@ -9841,6 +9971,8 @@

Table of Contents

  • AttemptFailureSummary -
  • AttemptFailureType -
  • AttemptInfoRead -
    • +
  • AttemptNormalizationStatusRead -
    • +
  • AttemptNormalizationStatusReadList -
  • AttemptRead -
  • AttemptStats -
  • AttemptStatus -
    • @@ -10102,6 +10234,23 @@

    AttemptInfoRead - logs

    LogRead
    +
    +

    AttemptNormalizationStatusRead - Up

    +
    +
    +
    attemptNumber (optional)
    Integer format: int32
    +
    hasRecordsCommitted (optional)
    Boolean
    +
    recordsCommitted (optional)
    Long format: int64
    +
    hasNormalizationFailed (optional)
    Boolean
    +
    +
    +
    +

    AttemptNormalizationStatusReadList - Up

    +
    +
    +
    attemptNormalizationStatuses (optional)
    array[AttemptNormalizationStatusRead]
    +
    +

    AttemptRead - Up

    diff --git a/docs/understanding-airbyte/airbyte-protocol.md b/docs/understanding-airbyte/airbyte-protocol.md index 440c8feecf03..381e03f05aa1 100644 --- a/docs/understanding-airbyte/airbyte-protocol.md +++ b/docs/understanding-airbyte/airbyte-protocol.md @@ -28,6 +28,7 @@ The Airbyte Protocol is versioned independently of the Airbyte Platform, and the | Version | Date of Change | Pull Request(s) | Subject | | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------ | :------------------------------------------------------------------------------- | +| `v0.3.1` | 2022-10-12 | [17907](https://github.com/airbytehq/airbyte/pull/17907) | `AirbyteControlMessage.ConnectorConfig` added | | `v0.3.0` | 2022-09-09 | [16479](https://github.com/airbytehq/airbyte/pull/16479) | `AirbyteLogMessage.stack_trace` added | | `v0.2.0` | 2022-06-10 | [13573](https://github.com/airbytehq/airbyte/pull/13573) & [12586](https://github.com/airbytehq/airbyte/pull/12586) | `STREAM` and `GLOBAL` STATE messages | | `v0.1.1` | 2022-06-06 | [13356](https://github.com/airbytehq/airbyte/pull/13356) | Add a namespace in association with the stream name | @@ -803,6 +804,55 @@ AirbyteErrorTraceMessage: - config_error ``` +## AirbyteControlMessage + +An `AirbyteControlMessage` is for connectors to signal to the Airbyte Platform or Orchestrator that an action with a side-effect should be taken. This means that the Orchestrator will likely be altering some stored data about the connector, connection, or sync. + +```yaml +AirbyteControlMessage: + type: object + additionalProperties: true + required: + - type + - emitted_at + properties: + type: + title: orchestrator type + description: "the type of orchestrator message" + type: string + enum: + - CONNECTOR_CONFIG + emitted_at: + description: "the time in ms that the message was emitted" + type: number + connectorConfig: + description: "connector config orchestrator message: the updated config for the platform to store for this connector" + "$ref": "#/definitions/AirbyteControlConnectorConfigMessage" +``` + +### AirbyteControlConnectorConfigMessage + +`AirbyteControlConnectorConfigMessage` allows a connector to update its configuration in the middle of a sync. This is valuable for connectors with short-lived or single-use credentials. + +Emitting this message signals to the orchestrator process that it should update its persistence layer, replacing the connector's current configuration with the config present in the `.config` field of the message. + +The config in the `AirbyteControlConnectorConfigMessage` must conform to connector's specification's schema, and the orchestrator process is expected to validate these messages. If the output config does not conform to the specification's schema, the orchestrator process should raise an exception and terminate the sync. + +```yaml +AirbyteControlConnectorConfigMessage: + type: object + additionalProperties: true + required: + - config + properties: + config: + description: "the config items from this connector's spec to update" + type: object + additionalProperties: true +``` + +For example, if the currently persisted config file is `{"api_key": 123, start_date: "01-01-2022"}` and the following `AirbyteControlConnectorConfigMessage` is output `{type: ORCHESTRATOR, connectorConfig: {"config": {"api_key": 456}, "emitted_at": }}` then the persisted configuration is merged, and will become `{"api_key": 456, start_date: "01-01-2022"}`. + # Acknowledgements We'd like to note that we were initially inspired by Singer.io's [specification](https://github.com/singer-io/getting-started/blob/master/docs/SPEC.md#singer-specification) and would like to acknowledge that some of their design choices helped us bootstrap our project. We've since made a lot of modernizations to our protocol and specification, but don't want to forget the tools that helped us get started. diff --git a/docs/understanding-airbyte/supported-data-types.md b/docs/understanding-airbyte/supported-data-types.md index 1d5b76f673c0..976173491e50 100644 --- a/docs/understanding-airbyte/supported-data-types.md +++ b/docs/understanding-airbyte/supported-data-types.md @@ -2,31 +2,33 @@ AirbyteRecords are required to conform to the Airbyte type system. This means that all sources must produce schemas and records within these types, and all destinations must handle records that conform to this type system. -Because Airbyte's interfaces are JSON-based, this type system is realized using [JSON schemas](https://json-schema.org/). In order to work around some limitations of JSON schemas, schemas may declare an additional `airbyte_type` annotation. This is used to disambiguate certain types that JSON schema does not explicitly differentiate between. See the [specific types](#specific-types) section for details. +Because Airbyte's interfaces are JSON-based, this type system is realized using [JSON schemas](https://json-schema.org/). In order to work around some limitations of JSON schemas, we define our own types - see [well_known_types.yaml](https://github.com/airbytehq/airbyte/blob/111131a193359027d0081de1290eb4bb846662ef/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/well_known_types.yaml). Sources should use `$ref` to reference these types, rather than directly defining JsonSchema entries. -This type system does not (generally) constrain values. Sources may declare streams using additional features of JSON schema (such as the `length` property for strings), but those constraints will be ignored by all other Airbyte components. The exception is in numeric types; `integer` and `number` fields must be representable within 64-bit primitives. +In an older version of the protocol, we relied on an `airbyte_type` property in schemas. This has been replaced by the well-known type schemas. All "old-style" types map onto well-known types. For example, a legacy connector producing a field of type `{"type": "string", "airbyte_type": "timestamp_with_timezone"}` is treated as producing `{"$ref": "WellKnownTypes.json#definitions/TimestampWithTimezone"}`. + +This type system does not (generally) constrain values. The exception is in numeric types; `integer` and `number` fields must be representable within 64-bit primitives. ## The types This table summarizes the available types. See the [Specific Types](#specific-types) section for explanation of optional parameters. -| Airbyte type | JSON Schema | Examples | -| -------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- | -| Boolean | `{"type": "boolean"}` | `true` or `false` | -| String | `{"type": "string"}` | `"foo bar"` | -| Date | `{"type": "string", "format": "date"}` | `"2021-01-23"` | -| Datetime with timezone | `{"type": "string", "format": "date-time", "airbyte_type": "timestamp_with_timezone"}` | `"2022-11-22T01:23:45+05:00"` | -| Datetime without timezone | `{"type": "string", "format": "date-time", "airbyte_type": "timestamp_without_timezone"}` | `"2022-11-22T01:23:45"` | -| Integer | `{"type": "integer"}` | `42` | -| Big integer (unrepresentable as a 64-bit two's complement int) | `{"type": "string", "airbyte_type": "big_integer"}` | `"12345678901234567890123456789012345678"` | -| Number | `{"type": "number"}` | `1234.56` | -| Big number (unrepresentable as a 64-bit IEEE 754 float) | `{"type": "string", "airbyte_type": "big_number"}` | `"1,000,000,...,000.1234"` with 500 0's | -| Array | `{"type": "array"}`; optionally `items` and `additionalItems` | `[1, 2, 3]` | -| Object | `{"type": "object"}`; optionally `properties` and `additionalProperties` | `{"foo": "bar"}` | -| Untyped (i.e. any value is valid) | `{}` | | -| Union | `{"anyOf": [...]}` or `{"oneOf": [...]}` | | - -Note that some of these may be destination-dependent. For example, Snowflake `NUMERIC` columns can be at most 38 digits wide, but Postgres `NUMERIC` columns may have up to 131072 digits before the decimal point. +| Airbyte type | JSON Schema | Examples | +| -------------------------------------------------------------- | ------------------------------------------------------------------------ | ------------------------------------------------------------------------------- | +| String | `{"$ref": "WellKnownTypes.json#definitions/String"}` | `"foo bar"` | +| Binary data, represented as a base64 string | `{"$ref": "WellKnownTypes.json#definitions/BinaryData"}` | `"Zm9vIGJhcgo="` | +| Boolean | `{"$ref": "WellKnownTypes.json#definitions/Boolean"}` | `true` or `false` | +| Date | `{"$ref": "WellKnownTypes.json#definitions/Date"}` | `"2021-01-23"`, `"2021-01-23 BC"` | +| Timestamp with timezone | `{"$ref": "WellKnownTypes.json#definitions/TimestampWithTimezone"}` | `"2022-11-22T01:23:45.123456+05:00"`, `"2022-11-22T01:23:45Z BC"` | +| Timestamp without timezone | `{"$ref": "WellKnownTypes.json#definitions/TimestampWithoutTimezone"}` | `"2022-11-22T01:23:45"`, `"2022-11-22T01:23:45.123456 BC"` | +| Time with timezone | `{"$ref": "WellKnownTypes.json#definitions/TimeWithTimezone"}` | `"01:23:45.123456+05:00"`, `"01:23:45Z"` | +| Time without timezone | `{"$ref": "WellKnownTypes.json#definitions/TimeWithoutTimezone"}` | `"01:23:45.123456"`, `"01:23:45"` | +| Integer | `{"$ref": "WellKnownTypes.json#definitions/Integer"}` | `42`, `NaN`, `Infinity`, `-Infinity` | +| Number | `{"$ref": "WellKnownTypes.json#definitions/Number"}` | `1234.56`, `NaN`, `Infinity`, `-Infinity` | +| Array | `{"type": "array"}`; optionally `items` and `additionalItems` | `[1, 2, 3]` | +| Object | `{"type": "object"}`; optionally `properties` and `additionalProperties` | `{"foo": "bar"}` | +| Union | `{"anyOf": [...]}` or `{"oneOf": [...]}` | | + +Note that some of these may be destination-dependent. For example, different warehouses may impose different limits on string column length. ### Record structure As a reminder, sources expose a `discover` command, which returns a list of [`AirbyteStreams`](https://github.com/airbytehq/airbyte/blob/111131a193359027d0081de1290eb4bb846662ef/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L122), and a `read` method, which emits a series of [`AirbyteRecordMessages`](https://github.com/airbytehq/airbyte/blob/111131a193359027d0081de1290eb4bb846662ef/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L46-L66). The type system determines what a valid `json_schema` is for an `AirbyteStream`, which in turn dictates what messages `read` is allowed to emit. @@ -39,15 +41,16 @@ For example, a source could produce this `AirbyteStream` (remember that the `jso "type": "object", "properties": { "username": { - "type": "string" + "$ref": "WellKnownTypes.json#definitions/String" }, "age": { - "type": "integer" + "$ref": "WellKnownTypes.json#definitions/Integer" }, "appointments": { "type": "array", - "items": "string", - "airbyte_type": "timestamp_with_timezone" + "items": { + "$ref": "WellKnownTypes.json#definitions/TimestampWithTimezone" + } } } } @@ -69,26 +72,7 @@ Along with this `AirbyteRecordMessage` (observe that the `data` field conforms t The top-level `object` must conform to the type system. This [means](#objects) that all of the fields must also conform to the type system. #### Nulls -Many sources cannot guarantee that all fields are present on all records. As such, they may replace the `type` entry in the schema with `["null", "the_real_type"]`. For example, this schema is the correct way for a source to declare that the `age` field may be missing from some records: -```json -{ - "type": "object", - "properties": { - "username": { - "type": "string" - }, - "age": { - "type": ["null", "integer"] - } - } -} -``` -This would then be a valid record: -```json -{"username": "someone42"} -``` - -Nullable fields are actually the more common case, but this document omits them in other examples for the sake of clarity. +Many sources cannot guarantee that all fields are present on all records. In these cases, sources should simply not list them as `required` fields. In most cases, sources do not need to list fields as required; by default, all fields are treated as nullable. #### Unsupported types As an escape hatch, destinations which cannot handle a certain type should just fall back to treating those values as strings. For example, let's say a source discovers a stream with this schema: @@ -99,14 +83,13 @@ As an escape hatch, destinations which cannot handle a certain type should just "appointments": { "type": "array", "items": { - "type": "string", - "airbyte_type": "timestamp_with_timezone" + "$ref": "WellKnownTypes.json#definitions/TimestampWithTimezone" } } } } ``` -Along with records that look like this: +Along with records which contain data that looks like this: ```json {"appointments": ["2021-11-22T01:23:45+00:00", "2022-01-22T14:00:00+00:00"]} ``` @@ -127,37 +110,42 @@ And emitted this record: {"appointments": "[\"2021-11-22T01:23:45+00:00\", \"2022-01-22T14:00:00+00:00\"]"} ``` +Of course, destinations are free to choose the most convenient/reasonable stringification for any given value. JSON serialization is just one possible strategy. + ### Specific types #### Boolean -Airbyte boolean type represents one of the two values `true` or `false` and they are are lower case. Note that values that evaluates to true or false such as data type String `"true"` or `"false"` or Integer like `1` or `0` are not accepted by the Schema. +Airbyte boolean type represents one of the two values `true` or `false` and they are are lower case. Note that values that evaluate to true or false such as data type String `"true"` or `"false"` or Integer like `1` or `0` are not accepted by the Schema. #### Dates and timestamps -Airbyte has three temporal types: `date`, `timestamp_with_timezone`, and `timestamp_without_timezone`. These are represented as strings with specific `format` (either `date` or `date-time`). +Airbyte has five temporal types: `date`, `timestamp_with_timezone`, `timestamp_without_timezone`, `time_with_timezone`, and `time_without_timezone`. These are represented as strings with specific `format` (either `date` or `date-time`). -However, JSON schema does not have a built-in way to indicate whether a field includes timezone information. For example, given the schema +However, JSON schema does not have a built-in way to indicate whether a field includes timezone information. For example, given this JsonSchema: ```json { "type": "object", "properties": { "created_at": { "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" + "format": "date-time" } } } ``` -Both `{"created_at": "2021-11-22T01:23:45+00:00"}` and `{"created_at": "2021-11-22T01:23:45"}` are valid records. The `airbyte_type` annotation resolves this ambiguity; sources producing `date-time` fields **must** set the `airbyte_type` to either `timestamp_with_timezone` or `timestamp_without_timezone`. +Both `{"created_at": "2021-11-22T01:23:45+00:00"}` and `{"created_at": "2021-11-22T01:23:45"}` are valid records. + +The protocol's type definitions resolve this ambiguity; sources producing timestamp-ish fields **must** choose either `TimestampWithTimezone` or `TimestampWithoutTimezone` (or time with/without timezone). -#### Unrepresentable numbers -64-bit integers and floating-point numbers (AKA `long` and `double`) cannot represent every number in existence. The `big_integer` and `big_number` types indicate that the source may produce numbers outside the ranges of `long` and `double`s. +All of these must be represented as RFC 3339§5.6 strings, extended with BC era support. See the type definition descriptions for specifics. -Note that these are declared as `"type": "string"`. This is intended to make parsing more safe by preventing accidental overflow/loss-of-precision. +#### Numeric values +Integers are extended to accept infinity/-infinity/NaN values. Most sources will not actually produce those values, and destinations may not fully support them. + +64-bit integers and floating-point numbers (AKA `long` and `double`) cannot represent every number in existence. Sources should use the string type if their fields may exceed `int64`/`float64` ranges. #### Arrays Arrays contain 0 or more items, which must have a defined type. These types should also conform to the type system. Arrays may require that all of their elements be the same type (`"items": {whatever type...}`), or they may require specific types for the first N entries (`"items": [{first type...}, {second type...}, ... , {Nth type...}]`, AKA tuple-type). -Tuple-typed arrays can configure the type of any additional elements using the `additionalItems` field; by default, any type is allowed. They may also pass a boolean to enable/disable additional elements, with `"additionalItems": true` being equivalent to `"additionalItems": {}` and `"additionalItems": false` meaning that only the tuple-defined items are allowed. +Tuple-typed arrays can configure the type of any additional elements using the `additionalItems` field; by default, any type is allowed. They may also pass a boolean to enable/disable additional elements, with `"additionalItems": true` being equivalent to `"additionalItems": {"$ref": "WellKnownTypes.json#definitions/String"}` and `"additionalItems": false` meaning that only the tuple-defined items are allowed. Destinations may have a difficult time supporting tuple-typed arrays without very specific handling, and as such are permitted to somewhat loosen their requirements. For example, many Avro-based destinations simply declare an array of a union of all allowed types, rather than requiring the correct type in each position of the array. @@ -167,5 +155,7 @@ As with arrays, objects may declare `properties`, each of which should have a ty #### Unions In some cases, sources may want to use multiple types for the same field. For example, a user might have a property which holds either an object, or a `string` explanation of why that data is missing. This is supported with JSON schema's `oneOf` and `anyOf` types. +Note that JsonSchema's `allOf` combining structure is not accepted within the protocol, because all of the protocol type definitions are mutually exclusive. + #### Untyped values -In some unusual cases, a property may not have type information associated with it. This is represented by the empty schema `{}`. As many destinations do not allow untyped data, this will frequently trigger the [string-typed escape hatch](#unsupported-types). +In some unusual cases, a property may not have type information associated with it. Sources must cast these properties to string, and discover them as `{"$ref": "WellKnownTypes.json#definitions/String"}`. diff --git a/tools/bin/acceptance_test_kube.sh b/tools/bin/acceptance_test_kube.sh index c90baa69e380..eb1d86de442d 100755 --- a/tools/bin/acceptance_test_kube.sh +++ b/tools/bin/acceptance_test_kube.sh @@ -32,7 +32,7 @@ echo "Listing nodes scheduled for pods..." kubectl describe pods | grep "Name\|Node" # allocates a lot of time to start kube. takes a while for postgres+temporal to work things out -sleep 120s +sleep 120 if [ -n "$CI" ]; then server_logs () { kubectl logs deployment.apps/airbyte-server > /tmp/kubernetes_logs/server.txt; } diff --git a/tools/bin/deploy_docusaurus b/tools/bin/deploy_docusaurus index fcfaa900ae68..07eab74a2609 100755 --- a/tools/bin/deploy_docusaurus +++ b/tools/bin/deploy_docusaurus @@ -18,7 +18,7 @@ else fi # if a string -if $(git remote get-url origin | grep --quiet "http"); then +if test "$(tty)" != "not a tty" && $(git remote get-url origin | grep --quiet "http"); then set +o xtrace echo -e "$red_text""This program requires a ssh-based github repo""$default_text" echo -e "$red_text""https://docs.github.com/en/authentication/connecting-to-github-with-ssh/adding-a-new-ssh-key-to-your-github-account""$default_text" @@ -41,9 +41,9 @@ echo -e "$blue_text""Current path:""$default_text" pwd -# Yarn check (which is commonly used to check for program existance) +# Yarn check (which is commonly used to check for program existence) # -s/--silent doesn't exist in cloud's Ubuntu -if ! which brew > /dev/null; then +if [[ `uname -s` = 'Darwin' && ! `which brew` ]] > /dev/null; then echo -e "$red_text""homebrew not found HALP!!\n\n""$default_text" echo -e "$red_text""try this: https://brew.sh\n\n""$default_text" exit 1 @@ -85,20 +85,19 @@ if test "$(tty)" == "not a tty"; then set +o xtrace echo -e "$blue_text""github email not found adding Octavia's""$default_text" set -o xtrace - git config user.email="octavia-squidington-iii@users.noreply.github.com" - git config user.name="octavia-squidington-iii" + git config --global user.email "octavia-squidington-iii@users.noreply.github.com" + git config --global user.name "octavia-squidington-iii" echo "machine github.com login octavia-squidington-iii password $GITHUB_TOKEN" > $HOME/.netrc # context https://v1.docusaurus.io/docs/en/publishing#using-github-pages # write a prod website to airbytehq/airbyte gh_pages branch - # NOT ACTUALLY WORKING - GIT_USER="octavia-squidington-iii" yarn run publish-gh-pages + # 'publish-gh-pages' NOT ACTUALLY WORKING, we use 'deploy' instead: + GIT_USER="octavia-squidington-iii" yarn run deploy else yarn run deploy fi - # Git makes more sense from / cd .. pwd @@ -129,21 +128,11 @@ git add CNAME # Skip pre-commit hooks fire_elmo.jpg PRE_COMMIT_ALLOW_NO_CONFIG=1 git commit --message "Adds CNAME to deploy for $revision" +git push + # non functional. for debugging git rev-parse --abbrev-ref HEAD -if test "$(tty)" == "not a tty"; then - # note that this is NOT airbyte repo - git push --force https://octavia-squidington-iii:$GITHUB_TOKEN@github.com/airbytehq/airbytehq.github.io.git -else - git push --force git@github.com:airbytehq/airbytehq.github.io.git -fi - -# Let's leave the tire fire of force pushes -git checkout - - - -# Want to push from your own computer? uncomment this line and comment out the push above set +o xtrace echo -e "$blue_text""Script exiting 0 GREAT SUCCESS!!!?""$default_text"