diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f82f2b5ba91f5..c1577406f64bb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,8 @@ repos: rev: v0.8.8 hooks: - id: licenseheaders - args: ["--tmpl=LICENSE_SHORT", "--ext=py", "-f"] + args: + ["--tmpl=LICENSE_SHORT", "--ext=py", "-x=**/models/__init__.py", "-f"] - repo: https://github.com/psf/black rev: 22.3.0 hooks: diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 727ab95d0e2ae..739ca1d8c5f68 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.16.2 +Fix the naming of OAuthAuthenticator + ## 0.16.1 Include declarative_component_schema.yaml in the publish to PyPi diff --git a/airbyte-cdk/python/README.md b/airbyte-cdk/python/README.md index d318c7cb81f8a..a660ef8dba851 100644 --- a/airbyte-cdk/python/README.md +++ b/airbyte-cdk/python/README.md @@ -66,6 +66,13 @@ pip install -e ".[dev]" # [dev] installs development-only dependencies * Perform static type checks using `mypy airbyte_cdk`. `MyPy` configuration is in `.mypy.ini`. * The `type_check_and_test.sh` script bundles both type checking and testing in one convenient command. Feel free to use it! +##### Autogenerated files +If the iteration you are working on includes changes to the models, you might want to regenerate them. In order to do that, you can run: +```commandline +SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace +``` +This will generate the files based on the schemas, add the license information and format the code. If you want to only do the former and rely on pre-commit to the others, you can run the appropriate generation command i.e. `./gradlew generateProtocolClassFiles` or `./gradlew generateComponentManifestClassFiles`. + #### Testing All tests are located in the `unit_tests` directory. Run `pytest --cov=airbyte_cdk unit_tests/` to run them. This also presents a test coverage report. diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 40a9e9e6bc275..1eef180598d03 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -363,8 +363,8 @@ definitions: $options: type: object additionalProperties: true - DeclarativeOauth2Authenticator: - description: Authenticator for requests using Oauth authentication + OAuthAuthenticator: + description: Authenticator for requests using OAuth 2.0 authentication type: object required: - type @@ -375,7 +375,7 @@ definitions: properties: type: type: string - enum: [DeclarativeOauth2Authenticator] + enum: [OAuthAuthenticator] client_id: type: string client_secret: @@ -580,7 +580,7 @@ definitions: - "$ref": "#/definitions/BasicHttpAuthenticator" - "$ref": "#/definitions/BearerAuthenticator" - "$ref": "#/definitions/CustomAuthenticator" - - "$ref": "#/definitions/DeclarativeOauth2Authenticator" + - "$ref": "#/definitions/OAuthAuthenticator" - "$ref": "#/definitions/NoAuth" - "$ref": "#/definitions/SessionTokenAuthenticator" error_handler: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/__init__.py new file mode 100644 index 0000000000000..de07af17cd150 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/__init__.py @@ -0,0 +1,2 @@ +# generated by generate-component-manifest-files +from .declarative_component_schema import * diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py new file mode 100644 index 0000000000000..e2153536819c5 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -0,0 +1,490 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +# generated by datamodel-codegen: +# filename: declarative_component_schema.yaml + +from __future__ import annotations + +from enum import Enum +from typing import Any, Dict, List, Optional, Union + +from pydantic import BaseModel, Extra, Field +from typing_extensions import Literal + + +class AddedFieldDefinition(BaseModel): + type: Literal["AddedFieldDefinition"] + path: List[str] + value: str + + +class AddFields(BaseModel): + type: Literal["AddFields"] + fields: List[AddedFieldDefinition] + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class ApiKeyAuthenticator(BaseModel): + type: Literal["ApiKeyAuthenticator"] + api_token: str + header: Optional[str] = None + + +class BasicHttpAuthenticator(BaseModel): + type: Literal["BasicHttpAuthenticator"] + username: str + password: Optional[str] = None + + +class BearerAuthenticator(BaseModel): + type: Literal["BearerAuthenticator"] + api_token: str + + +class CheckStream(BaseModel): + type: Literal["CheckStream"] + stream_names: List[str] + + +class ConstantBackoffStrategy(BaseModel): + type: Literal["ConstantBackoffStrategy"] + backoff_time_in_seconds: Union[float, str] + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CustomAuthenticator(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["CustomAuthenticator"] + class_name: str + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CustomBackoffStrategy(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["CustomBackoffStrategy"] + class_name: str + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CustomErrorHandler(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["CustomErrorHandler"] + class_name: str + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CustomPaginationStrategy(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["CustomPaginationStrategy"] + class_name: str + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CustomRecordExtractor(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["CustomRecordExtractor"] + class_name: str + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CustomRetriever(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["CustomRetriever"] + class_name: str + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CustomStreamSlicer(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["CustomStreamSlicer"] + class_name: str + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CustomTransformation(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["CustomTransformation"] + class_name: str + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class OAuthAuthenticator(BaseModel): + type: Literal["OAuthAuthenticator"] + client_id: str + client_secret: str + refresh_token: str + token_refresh_endpoint: str + access_token_name: Optional[str] = None + expires_in_name: Optional[str] = None + grant_type: Optional[str] = None + refresh_request_body: Optional[Dict[str, Any]] = None + scopes: Optional[List[str]] = None + token_expiry_date: Optional[str] = None + + +class DefaultSchemaLoader(BaseModel): + type: Literal["DefaultSchemaLoader"] + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class ExponentialBackoffStrategy(BaseModel): + type: Literal["ExponentialBackoffStrategy"] + factor: Optional[Union[float, str]] = 5 + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class HttpMethodEnum(Enum): + GET = "GET" + POST = "POST" + + +class Action(Enum): + SUCCESS = "SUCCESS" + FAIL = "FAIL" + RETRY = "RETRY" + IGNORE = "IGNORE" + + +class HttpResponseFilter(BaseModel): + type: Literal["HttpResponseFilter"] + action: Action + error_message: Optional[str] = None + error_message_contains: Optional[str] = None + http_codes: Optional[List[int]] = None + predicate: Optional[str] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class InlineSchemaLoader(BaseModel): + type: Literal["InlineSchemaLoader"] + schema_: Optional[Dict[str, Any]] = Field(None, alias="schema") + + +class InterpolatedRequestOptionsProvider(BaseModel): + type: Literal["InterpolatedRequestOptionsProvider"] + request_body_data: Optional[Union[str, Dict[str, str]]] = None + request_body_json: Optional[Union[str, Dict[str, str]]] = None + request_headers: Optional[Union[str, Dict[str, str]]] = None + request_parameters: Optional[Union[str, Dict[str, str]]] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class Type(Enum): + JsonFileSchemaLoader = "JsonFileSchemaLoader" + JsonSchema = "JsonSchema" + + +class JsonFileSchemaLoader(BaseModel): + type: Type + file_path: Optional[str] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class JsonDecoder(BaseModel): + type: Literal["JsonDecoder"] + + +class MinMaxDatetime(BaseModel): + type: Literal["MinMaxDatetime"] + datetime: str + datetime_format: Optional[str] = "" + max_datetime: Optional[str] = None + min_datetime: Optional[str] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class NoAuth(BaseModel): + type: Literal["NoAuth"] + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class NoPagination(BaseModel): + type: Literal["NoPagination"] + + +class OffsetIncrement(BaseModel): + type: Literal["OffsetIncrement"] + page_size: Union[float, str] + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class PageIncrement(BaseModel): + type: Literal["PageIncrement"] + page_size: int + start_from_page: Optional[int] = 0 + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class PrimaryKey(BaseModel): + __root__: Union[str, List[str], List[List[str]]] = Field(..., description="The stream field to be used to distinguish unique rows") + + +class RecordFilter(BaseModel): + type: Literal["RecordFilter"] + backoff_time_in_seconds: Optional[str] = "" + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class RemoveFields(BaseModel): + type: Literal["RemoveFields"] + field_pointers: List[List[str]] + + +class InjectInto(Enum): + request_parameter = "request_parameter" + header = "header" + path = "path" + body_data = "body_data" + body_json = "body_json" + + +class RequestOption(BaseModel): + type: Literal["RequestOption"] + inject_into: InjectInto + field_name: Optional[str] = None + + +class Schemas(BaseModel): + pass + + class Config: + extra = Extra.allow + + +class SessionTokenAuthenticator(BaseModel): + type: Literal["SessionTokenAuthenticator"] + api_url: str + header: str + login_url: str + session_token: str + session_token_response_key: str + username: str + validate_session_url: str + password: Optional[str] = "" + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class SingleSlice(BaseModel): + type: Literal["SingleSlice"] + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class Spec(BaseModel): + type: Literal["Spec"] + connection_specification: Dict[str, Any] + documentation_url: str + + +class WaitTimeFromHeader(BaseModel): + type: Literal["WaitTimeFromHeader"] + header: str + regex: Optional[str] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class WaitUntilTimeFromHeader(BaseModel): + type: Literal["WaitUntilTimeFromHeader"] + header: str + min_wait: Optional[Union[float, str]] = None + regex: Optional[str] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CursorPagination(BaseModel): + type: Literal["CursorPagination"] + cursor_value: str + page_size: Optional[int] = None + stop_condition: Optional[str] = None + decoder: Optional[JsonDecoder] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class DatetimeStreamSlicer(BaseModel): + type: Literal["DatetimeStreamSlicer"] + cursor_field: str + datetime_format: str + end_datetime: Union[str, MinMaxDatetime] + start_datetime: Union[str, MinMaxDatetime] + step: str + end_time_option: Optional[RequestOption] = None + lookback_window: Optional[str] = None + start_time_option: Optional[RequestOption] = None + stream_state_field_end: Optional[str] = None + stream_state_field_start: Optional[str] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class DefaultErrorHandler(BaseModel): + type: Literal["DefaultErrorHandler"] + backoff_strategies: Optional[ + List[ + Union[ + ConstantBackoffStrategy, + CustomBackoffStrategy, + ExponentialBackoffStrategy, + WaitTimeFromHeader, + WaitUntilTimeFromHeader, + ] + ] + ] = None + max_retries: Optional[int] = 5 + response_filters: Optional[List[HttpResponseFilter]] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class DefaultPaginator(BaseModel): + type: Literal["DefaultPaginator"] + pagination_strategy: Union[CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement] + url_base: str + decoder: Optional[JsonDecoder] = None + page_size_option: Optional[RequestOption] = None + page_token_option: Optional[RequestOption] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class DpathExtractor(BaseModel): + type: Literal["DpathExtractor"] + field_pointer: List[str] + decoder: Optional[JsonDecoder] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class ListStreamSlicer(BaseModel): + type: Literal["ListStreamSlicer"] + cursor_field: str + slice_values: Union[str, List[str]] + request_option: Optional[RequestOption] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class RecordSelector(BaseModel): + type: Literal["RecordSelector"] + extractor: Union[CustomRecordExtractor, DpathExtractor] + record_filter: Optional[RecordFilter] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CompositeErrorHandler(BaseModel): + type: Literal["CompositeErrorHandler"] + error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class HttpRequester(BaseModel): + type: Literal["HttpRequester"] + name: str + path: str + url_base: str + authenticator: Optional[ + Union[ + ApiKeyAuthenticator, + BasicHttpAuthenticator, + BearerAuthenticator, + CustomAuthenticator, + OAuthAuthenticator, + NoAuth, + SessionTokenAuthenticator, + ] + ] = None + error_handler: Optional[Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler]] = None + http_method: Optional[Union[str, HttpMethodEnum]] = "GET" + request_options_provider: Optional[InterpolatedRequestOptionsProvider] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class DeclarativeSource(BaseModel): + type: Literal["DeclarativeSource"] + check: CheckStream + streams: List[DeclarativeStream] + version: str + schemas: Optional[Schemas] = None + spec: Optional[Spec] = None + + +class CartesianProductStreamSlicer(BaseModel): + type: Literal["CartesianProductStreamSlicer"] + stream_slicers: List[ + Union[ + CustomStreamSlicer, + DatetimeStreamSlicer, + ListStreamSlicer, + SingleSlice, + SubstreamSlicer, + ] + ] + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class DeclarativeStream(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["DeclarativeStream"] + retriever: Union[CustomRetriever, SimpleRetriever] + checkpoint_interval: Optional[int] = None + name: Optional[str] = "" + primary_key: Optional[Union[str, List[str], List[List[str]]]] = "" + schema_loader: Optional[Union[DefaultSchemaLoader, InlineSchemaLoader, JsonFileSchemaLoader]] = None + stream_cursor_field: Optional[Union[str, List[str]]] = None + transformations: Optional[List[Union[AddFields, CustomTransformation, RemoveFields]]] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class ParentStreamConfig(BaseModel): + type: Literal["ParentStreamConfig"] + parent_key: str + stream: DeclarativeStream + stream_slice_field: str + request_option: Optional[RequestOption] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class SimpleRetriever(BaseModel): + type: Literal["SimpleRetriever"] + record_selector: RecordSelector + requester: HttpRequester + name: Optional[str] = "" + paginator: Optional[Union[DefaultPaginator, NoPagination]] = None + primary_key: Optional[PrimaryKey] = None + stream_slicer: Optional[ + Union[ + CartesianProductStreamSlicer, + CustomStreamSlicer, + DatetimeStreamSlicer, + ListStreamSlicer, + SingleSlice, + SubstreamSlicer, + ] + ] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class SubstreamSlicer(BaseModel): + type: Literal["SubstreamSlicer"] + parent_stream_configs: List[ParentStreamConfig] + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +CompositeErrorHandler.update_forward_refs() +DeclarativeSource.update_forward_refs() +CartesianProductStreamSlicer.update_forward_refs() +DeclarativeStream.update_forward_refs() +SimpleRetriever.update_forward_refs() diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py index a6ea7c2afac9a..8db72d0718100 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py @@ -19,6 +19,8 @@ from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.requesters import RequestOption +from airbyte_cdk.sources.declarative.requesters.error_handlers import HttpResponseFilter from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.constant_backoff_strategy import ConstantBackoffStrategy from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.exponential_backoff_strategy import ( ExponentialBackoffStrategy, @@ -37,6 +39,7 @@ from airbyte_cdk.sources.declarative.requesters.paginators.strategies.cursor_pagination_strategy import CursorPaginationStrategy from airbyte_cdk.sources.declarative.requesters.paginators.strategies.offset_increment import OffsetIncrement from airbyte_cdk.sources.declarative.requesters.paginators.strategies.page_increment import PageIncrement +from airbyte_cdk.sources.declarative.requesters.request_options import InterpolatedRequestOptionsProvider from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever from airbyte_cdk.sources.declarative.schema.inline_schema_loader import InlineSchemaLoader from airbyte_cdk.sources.declarative.schema.json_file_schema_loader import JsonFileSchemaLoader @@ -45,7 +48,7 @@ from airbyte_cdk.sources.declarative.stream_slicers.datetime_stream_slicer import DatetimeStreamSlicer from airbyte_cdk.sources.declarative.stream_slicers.list_stream_slicer import ListStreamSlicer from airbyte_cdk.sources.declarative.stream_slicers.single_slice import SingleSlice -from airbyte_cdk.sources.declarative.stream_slicers.substream_slicer import SubstreamSlicer +from airbyte_cdk.sources.declarative.stream_slicers.substream_slicer import ParentStreamConfig, SubstreamSlicer from airbyte_cdk.sources.declarative.transformations import RemoveFields from airbyte_cdk.sources.declarative.transformations.add_fields import AddFields @@ -69,8 +72,10 @@ "DpathExtractor": DpathExtractor, "ExponentialBackoffStrategy": ExponentialBackoffStrategy, "HttpRequester": HttpRequester, + "HttpResponseFilter": HttpResponseFilter, "InlineSchemaLoader": InlineSchemaLoader, "InterpolatedBoolean": InterpolatedBoolean, + "InterpolatedRequestOptionsProvider": InterpolatedRequestOptionsProvider, "InterpolatedString": InterpolatedString, "JsonSchema": JsonFileSchemaLoader, # todo remove after hacktoberfest and update connectors to use JsonFileSchemaLoader "JsonFileSchemaLoader": JsonFileSchemaLoader, @@ -81,7 +86,9 @@ "OAuthAuthenticator": DeclarativeOauth2Authenticator, "OffsetIncrement": OffsetIncrement, "PageIncrement": PageIncrement, + "ParentStreamConfig": ParentStreamConfig, "RecordSelector": RecordSelector, + "RequestOption": RequestOption, "RemoveFields": RemoveFields, "SimpleRetriever": SimpleRetriever, "SingleSlice": SingleSlice, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py index 42ed89899972f..8be72eb778283 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py @@ -26,7 +26,7 @@ "DeclarativeSource.streams": "DeclarativeStream", # DeclarativeStream "DeclarativeStream.retriever": "SimpleRetriever", - "DeclarativeStream.schema_loader": "DefaultSchemaLoader", + "DeclarativeStream.schema_loader": "JsonFileSchemaLoader", # DefaultErrorHandler "DefaultErrorHandler.response_filters": "HttpResponseFilter", # DefaultPaginator diff --git a/airbyte-cdk/python/bin/generate-component-manifest-files.sh b/airbyte-cdk/python/bin/generate-component-manifest-files.sh new file mode 100755 index 0000000000000..31d5066dce382 --- /dev/null +++ b/airbyte-cdk/python/bin/generate-component-manifest-files.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -e + +[ -z "$ROOT_DIR" ] && exit 1 + +YAML_DIR=airbyte-cdk/python/airbyte_cdk/sources/declarative +OUTPUT_DIR=airbyte-cdk/python/airbyte_cdk/sources/declarative/models + +function main() { + rm -rf "$ROOT_DIR/$OUTPUT_DIR"/*.py + echo "# generated by generate-component-manifest-files" > "$ROOT_DIR/$OUTPUT_DIR"/__init__.py + + for f in "$ROOT_DIR/$YAML_DIR"/*.yaml; do + filename_wo_ext=$(basename "$f" | cut -d . -f 1) + echo "from .$filename_wo_ext import *" >> "$ROOT_DIR/$OUTPUT_DIR"/__init__.py + + docker run --user "$(id -u):$(id -g)" -v "$ROOT_DIR":/airbyte airbyte/code-generator:dev \ + --input "/airbyte/$YAML_DIR/$filename_wo_ext.yaml" \ + --output "/airbyte/$OUTPUT_DIR/$filename_wo_ext.py" \ + --use-title-as-name \ + --disable-timestamp \ + --enum-field-as-literal one + done +} + +main "$@" diff --git a/airbyte-cdk/python/build.gradle b/airbyte-cdk/python/build.gradle index c5cf8cc3af33e..addc32e7c80f9 100644 --- a/airbyte-cdk/python/build.gradle +++ b/airbyte-cdk/python/build.gradle @@ -13,6 +13,12 @@ task generateProtocolClassFiles(type: Exec) { dependsOn ':tools:code-generator:airbyteDocker' } +task generateComponentManifestClassFiles(type: Exec) { + environment 'ROOT_DIR', rootDir.absolutePath + commandLine 'bin/generate-component-manifest-files.sh' + dependsOn ':tools:code-generator:airbyteDocker' +} + task validateSourceYamlManifest(type: Exec) { environment 'ROOT_DIR', rootDir.absolutePath commandLine 'bin/validate-yaml-schema.sh' @@ -22,3 +28,8 @@ blackFormat.dependsOn generateProtocolClassFiles isortFormat.dependsOn generateProtocolClassFiles flakeCheck.dependsOn generateProtocolClassFiles installReqs.dependsOn generateProtocolClassFiles + +blackFormat.dependsOn generateComponentManifestClassFiles +isortFormat.dependsOn generateComponentManifestClassFiles +flakeCheck.dependsOn generateComponentManifestClassFiles +installReqs.dependsOn generateComponentManifestClassFiles diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 572207ff44376..d3ac09c8c966c 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.16.1", + version="0.16.2", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py index f724e98013a66..f77e30b684783 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py @@ -17,7 +17,7 @@ { "type": "DeclarativeStream", "retriever": {"type": "SimpleRetriever"}, - "schema_loader": {"type": "DefaultSchemaLoader"}, + "schema_loader": {"type": "JsonFileSchemaLoader"}, } ], }, diff --git a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java index 5ea0a2dc6b9f4..eb47985e15c1a 100644 --- a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java @@ -16,6 +16,7 @@ import io.airbyte.integrations.standardtest.source.TestDataHolder; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.JsonSchemaType; +import java.math.BigDecimal; import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.testcontainers.containers.Db2Container; @@ -126,7 +127,7 @@ protected void initTests() { .airbyteType(JsonSchemaType.NUMBER) .fullSourceDataType("DECIMAL(31, 0)") .addInsertValues("null", "1", "DECIMAL((-1 + 10E+29), 31, 0)", "DECIMAL((1 - 10E+29), 31, 0)") - .addExpectedValues(null, "1", "1.0E30", "-1.0E30") + .addExpectedValues(null, "1", "%.0f".formatted(Double.valueOf("1.0E30")), "%.0f".formatted(Double.valueOf("-1.0E30"))) .build()); addDataTypeTestData( TestDataHolder.builder() diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java index 7d8541597a85d..c80b0998f91df 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java @@ -146,7 +146,10 @@ protected void initTests() { .sourceType("NUMBER") .airbyteType(JsonSchemaType.NUMBER) .addInsertValues("null", "1", "123.45", "power(10, -130)", "9.99999999999999999999 * power(10, 125)") - .addExpectedValues(null, "1", "123.45", String.valueOf(Math.pow(10, -130)), String.valueOf(9.99999999999999999999 * Math.pow(10, 125))) + /* The 999990000… below is the plain string representation of 9.999 * power(10, 125) */ + /* because normalization expects a plain integer strings whereas `Math.pow(10, 125)` returns a scientific notation */ + .addExpectedValues(null, "1", "123.45", String.valueOf(Math.pow(10, -130)), + "999999999999999999999000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") .build()); addDataTypeTestData( @@ -155,7 +158,7 @@ protected void initTests() { .airbyteType(JsonSchemaType.NUMBER) .fullSourceDataType("NUMBER(6,-2)") .addInsertValues("123.89") - .addExpectedValues("100.0") + .addExpectedValues("100") .build()); addDataTypeTestData( @@ -164,7 +167,7 @@ protected void initTests() { .airbyteType(JsonSchemaType.NUMBER) .fullSourceDataType("FLOAT(5)") .addInsertValues("1.34", "126.45") - .addExpectedValues("1.3", "130.0") + .addExpectedValues("1.3", "130") .build()); addDataTypeTestData( diff --git a/airbyte-integrations/connectors/source-pardot/pardot.md b/airbyte-integrations/connectors/source-pardot/pardot.md index 72ae9ff15455c..e877c48a103a8 100644 --- a/airbyte-integrations/connectors/source-pardot/pardot.md +++ b/airbyte-integrations/connectors/source-pardot/pardot.md @@ -8,27 +8,27 @@ The Pardot supports full refresh syncs Several output streams are available from this source: -* [Campaigns](https://developer.salesforce.com/docs/marketing/pardot/guide/campaigns-v4.html) -* [EmailClicks](https://developer.salesforce.com/docs/marketing/pardot/guide/batch-email-clicks-v4.html) -* [ListMembership](https://developer.salesforce.com/docs/marketing/pardot/guide/list-memberships-v4.html) -* [Lists](https://developer.salesforce.com/docs/marketing/pardot/guide/lists-v4.html) -* [ProspectAccounts](https://developer.salesforce.com/docs/marketing/pardot/guide/prospect-accounts-v4.html) -* [Prospects](https://developer.salesforce.com/docs/marketing/pardot/guide/prospects-v4.html) -* [Users](https://developer.salesforce.com/docs/marketing/pardot/guide/users-v4.html) -* [VisitorActivities](https://developer.salesforce.com/docs/marketing/pardot/guide/visitor-activities-v4.html) -* [Visitors](https://developer.salesforce.com/docs/marketing/pardot/guide/visitors-v4.html) -* [Visits](https://developer.salesforce.com/docs/marketing/pardot/guide/visits-v4.html) +- [Campaigns](https://developer.salesforce.com/docs/marketing/pardot/guide/campaigns-v4.html) +- [EmailClicks](https://developer.salesforce.com/docs/marketing/pardot/guide/batch-email-clicks-v4.html) +- [ListMembership](https://developer.salesforce.com/docs/marketing/pardot/guide/list-memberships-v4.html) +- [Lists](https://developer.salesforce.com/docs/marketing/pardot/guide/lists-v4.html) +- [ProspectAccounts](https://developer.salesforce.com/docs/marketing/pardot/guide/prospect-accounts-v4.html) +- [Prospects](https://developer.salesforce.com/docs/marketing/pardot/guide/prospects-v4.html) +- [Users](https://developer.salesforce.com/docs/marketing/pardot/guide/users-v4.html) +- [VisitorActivities](https://developer.salesforce.com/docs/marketing/pardot/guide/visitor-activities-v4.html) +- [Visitors](https://developer.salesforce.com/docs/marketing/pardot/guide/visitors-v4.html) +- [Visits](https://developer.salesforce.com/docs/marketing/pardot/guide/visits-v4.html) If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose) ### Features -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental Sync | No | -| SSL connection | No | -| Namespaces | No | +| Feature | Supported? | +| :---------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental Sync | No | +| SSL connection | No | +| Namespaces | No | ### Performance considerations @@ -38,26 +38,19 @@ The Pardot connector should not run into Pardot API limitations under normal usa ### Requirements -* Pardot Account -* Pardot Business Unit ID -* Client ID -* Client Secret -* Refresh Token -* Start Date -* Is Sandbox environment? +- Pardot Account +- Pardot Business Unit ID +- Client ID +- Client Secret +- Refresh Token +- Start Date +- Is Sandbox environment? ### Setup guide -* `pardot_business_unit_id`: Pardot Business ID, can be found at Setup > Pardot > Pardot Account Setup -* `client_id`: The Consumer Key that can be found when viewing your app in Salesforce -* `client_secret`: The Consumer Secret that can be found when viewing your app in Salesforce -* `refresh_token`: Salesforce Refresh Token used for Airbyte to access your Salesforce account. If you don't know what this is, follow [this guide](https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b) to retrieve it. -* `start_date`: UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. Leave blank to skip this filter -* `is_sandbox`: Whether or not the the app is in a Salesforce sandbox. If you do not know what this, assume it is false. - -## Changelog - -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.0 | 2021-10-16 | [7091](https://github.com/airbytehq/airbyte/pull/7091) | 🎉 New Source: Pardot | - +- `pardot_business_unit_id`: Pardot Business ID, can be found at Setup > Pardot > Pardot Account Setup +- `client_id`: The Consumer Key that can be found when viewing your app in Salesforce +- `client_secret`: The Consumer Secret that can be found when viewing your app in Salesforce +- `refresh_token`: Salesforce Refresh Token used for Airbyte to access your Salesforce account. If you don't know what this is, follow [this guide](https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b) to retrieve it. +- `start_date`: UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. Leave blank to skip this filter +- `is_sandbox`: Whether or not the the app is in a Salesforce sandbox. If you do not know what this, assume it is false. diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityImpl.java index 514a56eba4f95..794b501d8b624 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityImpl.java @@ -7,6 +7,7 @@ import static io.airbyte.metrics.lib.ApmTraceConstants.ACTIVITY_TRACE_OPERATION_NAME; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.CONNECTION_ID_KEY; +import com.google.common.annotations.VisibleForTesting; import datadog.trace.api.Trace; import io.airbyte.commons.temporal.config.WorkerMode; import io.airbyte.commons.temporal.exception.RetryableException; @@ -19,6 +20,7 @@ import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.persistence.job.WorkspaceHelper; import io.airbyte.persistence.job.models.Job; import io.airbyte.validation.json.JsonValidationException; import io.micronaut.context.annotation.Requires; @@ -32,23 +34,35 @@ import java.util.Date; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.TimeZone; import java.util.UUID; import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import org.joda.time.DateTimeZone; import org.quartz.CronExpression; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Slf4j @Singleton @Requires(env = WorkerMode.CONTROL_PLANE) public class ConfigFetchActivityImpl implements ConfigFetchActivity { + private static final Logger LOGGER = LoggerFactory.getLogger(ConfigFetchActivityImpl.class); private final static long MS_PER_SECOND = 1000L; private final static long MIN_CRON_INTERVAL_SECONDS = 60; + private static final Set SCHEDULING_NOISE_WORKSPACE_IDS = Set.of( + // Testing + UUID.fromString("0ace5e1f-4787-43df-8919-456f5f4d03d1"), + UUID.fromString("20810d92-41a4-4cfd-85db-fb50e77cf36b"), + // Prod + UUID.fromString("226edbc1-4a9c-4401-95a9-90435d667d9d")); + private static final long SCHEDULING_NOISE_CONSTANT = 15; private final ConfigRepository configRepository; private final JobPersistence jobPersistence; + private final WorkspaceHelper workspaceHelper; private final Integer syncJobMaxAttempts; private final Supplier currentSecondsSupplier; @@ -56,8 +70,18 @@ public ConfigFetchActivityImpl(final ConfigRepository configRepository, final JobPersistence jobPersistence, @Value("${airbyte.worker.sync.max-attempts}") final Integer syncJobMaxAttempts, @Named("currentSecondsSupplier") final Supplier currentSecondsSupplier) { + this(configRepository, jobPersistence, new WorkspaceHelper(configRepository, jobPersistence), syncJobMaxAttempts, currentSecondsSupplier); + } + + @VisibleForTesting + protected ConfigFetchActivityImpl(final ConfigRepository configRepository, + final JobPersistence jobPersistence, + final WorkspaceHelper workspaceHelper, + @Value("${airbyte.worker.sync.max-attempts}") final Integer syncJobMaxAttempts, + @Named("currentSecondsSupplier") final Supplier currentSecondsSupplier) { this.configRepository = configRepository; this.jobPersistence = jobPersistence; + this.workspaceHelper = workspaceHelper; this.syncJobMaxAttempts = syncJobMaxAttempts; this.currentSecondsSupplier = currentSecondsSupplier; } @@ -128,8 +152,10 @@ private ScheduleRetrieverOutput getTimeToWaitFromScheduleType(final StandardSync + MIN_CRON_INTERVAL_SECONDS : currentSecondsSupplier.get()) * MS_PER_SECOND); final Date nextRunStart = cronExpression.getNextValidTimeAfter(new Date(earliestNextRun)); - final Duration timeToWait = Duration.ofSeconds( + Duration timeToWait = Duration.ofSeconds( Math.max(0, nextRunStart.getTime() / MS_PER_SECOND - currentSecondsSupplier.get())); + + timeToWait = addSchedulingNoiseForAllowListedWorkspace(timeToWait, standardSync); return new ScheduleRetrieverOutput(timeToWait); } catch (final ParseException e) { throw (DateTimeException) new DateTimeException(e.getMessage()).initCause(e); @@ -137,6 +163,30 @@ private ScheduleRetrieverOutput getTimeToWaitFromScheduleType(final StandardSync } } + private Duration addSchedulingNoiseForAllowListedWorkspace(Duration timeToWait, StandardSync standardSync) { + final UUID workspaceId; + try { + workspaceId = workspaceHelper.getWorkspaceForConnectionId(standardSync.getConnectionId()); + } catch (JsonValidationException | ConfigNotFoundException e) { + // We tolerate exceptions and fail open by doing nothing. + return timeToWait; + } + if (!SCHEDULING_NOISE_WORKSPACE_IDS.contains(workspaceId)) { + // Only apply to a specific set of workspaces. + return timeToWait; + } + if (!standardSync.getScheduleType().equals(ScheduleType.CRON)) { + // Only apply noise to cron connections. + return timeToWait; + } + + // We really do want to add some scheduling noise for this connection. + final long minutesToWait = (long) (Math.random() * SCHEDULING_NOISE_CONSTANT); + LOGGER.debug("Adding {} minutes noise to wait", minutesToWait); + // Note: we add an extra second to make the unit tests pass in case `minutesToWait` was 0. + return timeToWait.plusMinutes(minutesToWait).plusSeconds(1); + } + /** * @param standardSync * @param connectionId diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityTest.java index 69aa56cd40b32..e2f1e103d7c0e 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityTest.java @@ -4,6 +4,9 @@ package io.airbyte.workers.temporal.scheduling.activities; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + import io.airbyte.config.BasicSchedule; import io.airbyte.config.Cron; import io.airbyte.config.Schedule; @@ -14,11 +17,13 @@ import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.persistence.job.WorkspaceHelper; import io.airbyte.persistence.job.models.Job; import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.temporal.scheduling.activities.ConfigFetchActivity.ScheduleRetrieverInput; import io.airbyte.workers.temporal.scheduling.activities.ConfigFetchActivity.ScheduleRetrieverOutput; import java.io.IOException; +import java.time.Duration; import java.time.Instant; import java.util.Calendar; import java.util.Optional; @@ -31,7 +36,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) @@ -45,6 +49,9 @@ class ConfigFetchActivityTest { @Mock private JobPersistence mJobPersistence; + @Mock + private WorkspaceHelper mWorkspaceHelper; + @Mock private Job mJob; @@ -69,13 +76,14 @@ class ConfigFetchActivityTest { .withTimeUnit(BasicSchedule.TimeUnit.MINUTES) .withUnits(5L))); + public static final String UTC = "UTC"; private final static StandardSync standardSyncWithCronScheduleType = new StandardSync() .withScheduleType(ScheduleType.CRON) .withStatus(Status.ACTIVE) .withScheduleData(new ScheduleData() .withCron(new Cron() .withCronExpression("0 0 12 * * ?") - .withCronTimeZone("UTC"))); + .withCronTimeZone(UTC))); private final static StandardSync standardSyncWithScheduleDisable = new StandardSync() .withSchedule(new Schedule() @@ -93,7 +101,8 @@ class ConfigFetchActivityTest { @BeforeEach void setup() { configFetchActivity = - new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, SYNC_JOB_MAX_ATTEMPTS, () -> Instant.now().getEpochSecond()); + new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, mWorkspaceHelper, SYNC_JOB_MAX_ATTEMPTS, + () -> Instant.now().getEpochSecond()); } @Nested @@ -102,10 +111,10 @@ class TimeToWaitTest { @Test @DisplayName("Test that the job gets scheduled if it is not manual and if it is the first run with legacy schedule schema") void testFirstJobNonManual() throws IOException, JsonValidationException, ConfigNotFoundException { - Mockito.when(mJobPersistence.getLastReplicationJob(connectionId)) + when(mJobPersistence.getLastReplicationJob(connectionId)) .thenReturn(Optional.empty()); - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithLegacySchedule); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -119,7 +128,7 @@ void testFirstJobNonManual() throws IOException, JsonValidationException, Config @Test @DisplayName("Test that the job will wait for a long time if it is manual in the legacy schedule schema") void testManual() throws IOException, JsonValidationException, ConfigNotFoundException { - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithoutSchedule); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -133,7 +142,7 @@ void testManual() throws IOException, JsonValidationException, ConfigNotFoundExc @Test @DisplayName("Test that the job will wait for a long time if it is disabled") void testDisable() throws IOException, JsonValidationException, ConfigNotFoundException { - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithScheduleDisable); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -147,7 +156,7 @@ void testDisable() throws IOException, JsonValidationException, ConfigNotFoundEx @Test @DisplayName("Test that the connection will wait for a long time if it is deleted") void testDeleted() throws IOException, JsonValidationException, ConfigNotFoundException { - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithScheduleDeleted); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -163,13 +172,13 @@ void testDeleted() throws IOException, JsonValidationException, ConfigNotFoundEx void testWait() throws IOException, JsonValidationException, ConfigNotFoundException { configFetchActivity = new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, SYNC_JOB_MAX_ATTEMPTS, () -> 60L * 3); - Mockito.when(mJob.getStartedAtInSecond()) + when(mJob.getStartedAtInSecond()) .thenReturn(Optional.of(60L)); - Mockito.when(mJobPersistence.getLastReplicationJob(connectionId)) + when(mJobPersistence.getLastReplicationJob(connectionId)) .thenReturn(Optional.of(mJob)); - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithLegacySchedule); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -185,13 +194,13 @@ void testWait() throws IOException, JsonValidationException, ConfigNotFoundExcep void testNotWaitIfLate() throws IOException, JsonValidationException, ConfigNotFoundException { configFetchActivity = new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, SYNC_JOB_MAX_ATTEMPTS, () -> 60L * 10); - Mockito.when(mJob.getStartedAtInSecond()) + when(mJob.getStartedAtInSecond()) .thenReturn(Optional.of(60L)); - Mockito.when(mJobPersistence.getLastReplicationJob(connectionId)) + when(mJobPersistence.getLastReplicationJob(connectionId)) .thenReturn(Optional.of(mJob)); - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithLegacySchedule); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -207,7 +216,7 @@ void testNotWaitIfLate() throws IOException, JsonValidationException, ConfigNotF @Test @DisplayName("Test that the job will wait a long time if it is MANUAL scheduleType") void testManualScheduleType() throws IOException, JsonValidationException, ConfigNotFoundException { - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithManualScheduleType); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -221,10 +230,10 @@ void testManualScheduleType() throws IOException, JsonValidationException, Confi @Test @DisplayName("Test that the job will be immediately scheduled if it is a BASIC_SCHEDULE type on the first run") void testBasicScheduleTypeFirstRun() throws IOException, JsonValidationException, ConfigNotFoundException { - Mockito.when(mJobPersistence.getLastReplicationJob(connectionId)) + when(mJobPersistence.getLastReplicationJob(connectionId)) .thenReturn(Optional.empty()); - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithBasicScheduleType); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -240,13 +249,13 @@ void testBasicScheduleTypeFirstRun() throws IOException, JsonValidationException void testBasicScheduleSubsequentRun() throws IOException, JsonValidationException, ConfigNotFoundException { configFetchActivity = new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, SYNC_JOB_MAX_ATTEMPTS, () -> 60L * 3); - Mockito.when(mJob.getStartedAtInSecond()) + when(mJob.getStartedAtInSecond()) .thenReturn(Optional.of(60L)); - Mockito.when(mJobPersistence.getLastReplicationJob(connectionId)) + when(mJobPersistence.getLastReplicationJob(connectionId)) .thenReturn(Optional.of(mJob)); - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithBasicScheduleType); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -260,19 +269,22 @@ void testBasicScheduleSubsequentRun() throws IOException, JsonValidationExceptio @Test @DisplayName("Test that the job will wait to be scheduled if it is a CRON type") void testCronScheduleSubsequentRun() throws IOException, JsonValidationException, ConfigNotFoundException { - final Calendar mockRightNow = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + final Calendar mockRightNow = Calendar.getInstance(TimeZone.getTimeZone(UTC)); mockRightNow.set(Calendar.HOUR_OF_DAY, 0); mockRightNow.set(Calendar.MINUTE, 0); mockRightNow.set(Calendar.SECOND, 0); mockRightNow.set(Calendar.MILLISECOND, 0); + when(mWorkspaceHelper.getWorkspaceForConnectionId(any())).thenReturn(UUID.randomUUID()); + configFetchActivity = - new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, SYNC_JOB_MAX_ATTEMPTS, () -> mockRightNow.getTimeInMillis() / 1000L); + new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, mWorkspaceHelper, SYNC_JOB_MAX_ATTEMPTS, + () -> mockRightNow.getTimeInMillis() / 1000L); - Mockito.when(mJobPersistence.getLastReplicationJob(connectionId)) + when(mJobPersistence.getLastReplicationJob(connectionId)) .thenReturn(Optional.of(mJob)); - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithCronScheduleType); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -286,20 +298,23 @@ void testCronScheduleSubsequentRun() throws IOException, JsonValidationException @Test @DisplayName("Test that the job will only be scheduled once per minimum cron interval") void testCronScheduleMinimumInterval() throws IOException, JsonValidationException, ConfigNotFoundException { - final Calendar mockRightNow = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + final Calendar mockRightNow = Calendar.getInstance(TimeZone.getTimeZone(UTC)); mockRightNow.set(Calendar.HOUR_OF_DAY, 12); mockRightNow.set(Calendar.MINUTE, 0); mockRightNow.set(Calendar.SECOND, 0); mockRightNow.set(Calendar.MILLISECOND, 0); + when(mWorkspaceHelper.getWorkspaceForConnectionId(any())).thenReturn(UUID.randomUUID()); + configFetchActivity = - new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, SYNC_JOB_MAX_ATTEMPTS, () -> mockRightNow.getTimeInMillis() / 1000L); + new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, mWorkspaceHelper, SYNC_JOB_MAX_ATTEMPTS, + () -> mockRightNow.getTimeInMillis() / 1000L); - Mockito.when(mJob.getStartedAtInSecond()).thenReturn(Optional.of(mockRightNow.getTimeInMillis() / 1000L)); - Mockito.when(mJobPersistence.getLastReplicationJob(connectionId)) + when(mJob.getStartedAtInSecond()).thenReturn(Optional.of(mockRightNow.getTimeInMillis() / 1000L)); + when(mJobPersistence.getLastReplicationJob(connectionId)) .thenReturn(Optional.of(mJob)); - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithCronScheduleType); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -310,6 +325,36 @@ void testCronScheduleMinimumInterval() throws IOException, JsonValidationExcepti .hasHours(24); } + @Test + @DisplayName("Test that for specific workspace ids, we add some noise in the cron scheduling") + void testCronSchedulingNoise() throws IOException, JsonValidationException, ConfigNotFoundException { + final Calendar mockRightNow = Calendar.getInstance(TimeZone.getTimeZone(UTC)); + mockRightNow.set(Calendar.HOUR_OF_DAY, 0); + mockRightNow.set(Calendar.MINUTE, 0); + mockRightNow.set(Calendar.SECOND, 0); + mockRightNow.set(Calendar.MILLISECOND, 0); + + when(mWorkspaceHelper.getWorkspaceForConnectionId(any())).thenReturn(UUID.fromString("226edbc1-4a9c-4401-95a9-90435d667d9d")); + + configFetchActivity = + new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, mWorkspaceHelper, SYNC_JOB_MAX_ATTEMPTS, + () -> mockRightNow.getTimeInMillis() / 1000L); + + when(mJob.getStartedAtInSecond()).thenReturn(Optional.of(mockRightNow.getTimeInMillis() / 1000L)); + when(mJobPersistence.getLastReplicationJob(connectionId)) + .thenReturn(Optional.of(mJob)); + + when(mConfigRepository.getStandardSync(connectionId)) + .thenReturn(standardSyncWithCronScheduleType); + + final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); + + final ScheduleRetrieverOutput output = configFetchActivity.getTimeToWait(input); + + // Note: compareTo returns positive if the left side is greater than the right. + Assertions.assertThat(output.getTimeToWait().compareTo(Duration.ofHours(12)) > 0).isTrue(); + } + @Nested class TestGetMaxAttempt { diff --git a/build.gradle b/build.gradle index 8c30547fc84c4..5d7c2213598f6 100644 --- a/build.gradle +++ b/build.gradle @@ -485,6 +485,7 @@ subprojects { subproj -> task('generate') { dependsOn subprojects.collect { it.getTasksByName('generateProtocolClassFiles', true) } + dependsOn subprojects.collect { it.getTasksByName('generateComponentManifestClassFiles', true) } dependsOn subprojects.collect { it.getTasksByName('generateJsonSchema2Pojo', true) } } @@ -527,6 +528,11 @@ subprojects { if (generateFilesTask != null) { licenseTask.dependsOn generateFilesTask } + + def generateManifestFilesTask = project.tasks.findByName('generateComponentManifestClassFiles') + if (generateManifestFilesTask != null) { + licenseTask.dependsOn generateManifestFilesTask + } } task listAllDependencies(type: DependencyReportTask) {} } diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index 0cb5b588a050b..3b84349557775 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.43.3" +version: "0.43.6" # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml index c64f4eca2910f..7c4adab12c491 100644 --- a/charts/airbyte-connector-builder-server/Chart.yaml +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.43.3" +version: "0.43.6" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index 5ccf6b44c7a6a..6271c65e831c4 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.43.3" +version: "0.43.6" # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index c7a04c748892e..860e2d61ae46d 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.43.3" +version: "0.43.6" # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index 68b3190f789fc..1f527d8ebd901 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.43.3" +version: "0.43.6" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index 55ef02d983414..9d4386e765e8f 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.43.3" +version: "0.43.6" # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index c28bb59ddd4e7..73d1e47a107c9 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.43.3" +version: "0.43.6" # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index 7d984701a8383..b02c76aff5096 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.43.3" +version: "0.43.6" # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock index 2805c64937679..e6dcb88b3cbda 100644 --- a/charts/airbyte/Chart.lock +++ b/charts/airbyte/Chart.lock @@ -4,30 +4,30 @@ dependencies: version: 1.17.1 - name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.43.3 + version: 0.43.6 - name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.43.3 + version: 0.43.6 - name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.43.3 + version: 0.43.6 - name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.43.3 + version: 0.43.6 - name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.43.3 + version: 0.43.6 - name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.43.3 + version: 0.43.6 - name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.43.3 + version: 0.43.6 - name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.43.3 + version: 0.43.6 - name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.43.3 -digest: sha256:04b9338c23554bf1fcf64b4e296ba262bd24be24253cad9b7c8567b404f67ddb -generated: "2022-12-13T00:15:39.706036573Z" + version: 0.43.6 +digest: sha256:4c97218bd5a68ef3b9d1bc28b047d6d7fa02d30a7c42596d11b0d45f028c1eaa +generated: "2022-12-15T19:17:27.989410129Z" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index aa60439c52041..37472c054b56d 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.43.3 +version: 0.43.6 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -32,36 +32,36 @@ dependencies: - condition: airbyte-bootloader.enabled name: airbyte-bootloader repository: "https://airbytehq.github.io/helm-charts/" - version: 0.43.3 + version: 0.43.6 - condition: temporal.enabled name: temporal repository: "https://airbytehq.github.io/helm-charts/" - version: 0.43.3 + version: 0.43.6 - condition: webapp.enabled name: webapp repository: "https://airbytehq.github.io/helm-charts/" - version: 0.43.3 + version: 0.43.6 - condition: server.enabled name: server repository: "https://airbytehq.github.io/helm-charts/" - version: 0.43.3 + version: 0.43.6 - condition: worker.enabled name: worker repository: "https://airbytehq.github.io/helm-charts/" - version: 0.43.3 + version: 0.43.6 - condition: pod-sweeper.enabled name: pod-sweeper repository: "https://airbytehq.github.io/helm-charts/" - version: 0.43.3 + version: 0.43.6 - condition: metrics.enabled name: metrics repository: "https://airbytehq.github.io/helm-charts/" - version: 0.43.3 + version: 0.43.6 - condition: cron.enabled name: cron repository: "https://airbytehq.github.io/helm-charts/" - version: 0.43.3 + version: 0.43.6 - condition: connector-builder-server.enabled name: connector-builder-server repository: "https://airbytehq.github.io/helm-charts/" - version: 0.43.3 + version: 0.43.6 diff --git a/docs/integrations/destinations/firestore.md b/docs/integrations/destinations/firestore.md new file mode 100644 index 0000000000000..c82a9f12068eb --- /dev/null +++ b/docs/integrations/destinations/firestore.md @@ -0,0 +1,9 @@ +# Firestore + +The Firestore destination for Airbyte + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :----------------------------------------------------- | :---------------------------- | +| 0.1.1 | 2021-11-21 | [8158](https://github.com/airbytehq/airbyte/pull/8158) | Publish Destination Firestore | diff --git a/docs/integrations/destinations/kvdb.md b/docs/integrations/destinations/kvdb.md new file mode 100644 index 0000000000000..1d9a4341e8fc6 --- /dev/null +++ b/docs/integrations/destinations/kvdb.md @@ -0,0 +1,9 @@ +# KVDB + +The KVDB destination for Airbyte + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :----------------------------------------------------- | :---------------------------- | +| 0.1.0 | 2021-07-19 | [4786](https://github.com/airbytehq/airbyte/pull/4786) | Python Demo Destination: KVDB | diff --git a/docs/integrations/destinations/sftp-json.md b/docs/integrations/destinations/sftp-json.md index 6f3c90d0ae7f9..8de20277a7639 100644 --- a/docs/integrations/destinations/sftp-json.md +++ b/docs/integrations/destinations/sftp-json.md @@ -13,11 +13,11 @@ Each file will contain a collection of `json` objects which correspond directly #### Features -| Feature | Supported | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | Yes | -| Namespaces | No | +| Feature | Supported | +| :------------------------ | :-------- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | Yes | +| Namespaces | No | #### Performance considerations @@ -34,3 +34,9 @@ The `filename` **should not** have an extension in the configuration, as `.jsonl If `destination_path` is set to `/myfolder/files` and `filename` is set to `mydata`, the resulting file will be `/myfolder/files/mydata.jsonl`. These files can then be accessed by creating an SFTP connection to the server and navigating to the `destination_path`. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :----------------------------------------------------- | :---------------------------- | +| 0.1.0 | 2022-11-24 | [4924](https://github.com/airbytehq/airbyte/pull/4924) | 🎉 New Destination: SFTP JSON | diff --git a/docs/integrations/sources/appfollow.md b/docs/integrations/sources/appfollow.md index 21f56ccac73ca..d8a19b02bcae1 100644 --- a/docs/integrations/sources/appfollow.md +++ b/docs/integrations/sources/appfollow.md @@ -37,4 +37,4 @@ The Appfollow connector ideally should gracefully handle Appfollow API limitatio | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------- | -| 0.1.0 | 2022-08-11 | [14418](https://github.com/airbytehq/airbyte/pull/14418) | New Source: Appfollow | +| 0.1.1 | 2022-08-11 | [14418](https://github.com/airbytehq/airbyte/pull/14418) | New Source: Appfollow | diff --git a/docs/integrations/sources/appstore.md b/docs/integrations/sources/appstore-singer.md similarity index 100% rename from docs/integrations/sources/appstore.md rename to docs/integrations/sources/appstore-singer.md diff --git a/docs/integrations/sources/aws-cloudtrail.md b/docs/integrations/sources/aws-cloudtrail.md index eb711cd2ec685..6c44cd568c9f3 100644 --- a/docs/integrations/sources/aws-cloudtrail.md +++ b/docs/integrations/sources/aws-cloudtrail.md @@ -10,26 +10,26 @@ This Source Connector is based on a [Boto3 CloudTrail](https://boto3.amazonaws.c This Source is capable of syncing the following core Streams: -* [Management Events](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cloudtrail.html#CloudTrail.Client.lookup_events) +- [Management Events](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cloudtrail.html#CloudTrail.Client.lookup_events) Insight events are not supported right now. Only Management events are available. ### Data type mapping | Integration Type | Airbyte Type | Notes | -| :--- | :--- | :--- | -| `string` | `string` | | -| `number` | `integer` | | -| `array` | `array` | | -| `object` | `object` | | +| :--------------- | :----------- | :---- | +| `string` | `string` | | +| `number` | `integer` | | +| `array` | `array` | | +| `object` | `object` | | ### Features -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental Sync | Yes | | -| Namespaces | No | | +| Feature | Supported?\(Yes/No\) | Notes | +| :---------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental Sync | Yes | | +| Namespaces | No | | ### Performance considerations @@ -39,9 +39,9 @@ The rate of lookup requests for `events` stream is limited to two per second, pe ### Requirements -* AWS Access key ID -* AWS Secret access key -* AWS region name +- AWS Access key ID +- AWS Secret access key +- AWS region name ### Setup guide @@ -49,10 +49,10 @@ Please, follow this [steps](https://docs.aws.amazon.com/powershell/latest/usergu ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.3 | 2021-12-23 | [8434](https://github.com/airbytehq/airbyte/pull/8434) | Update fields in source-connectors specifications | -| 0.1.2 | 2021-08-04 | [5152](https://github.com/airbytehq/airbyte/pull/5152) | Fix connector spec.json | -| 0.1.1 | 2021-07-06 | [4539](https://github.com/airbytehq/airbyte/pull/4539) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | -| 0.1.0 | 2021-06-23 | [4122](https://github.com/airbytehq/airbyte/pull/4122) | Initial release supporting the LookupEvent API | - +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------ | +| 0.1.4 | 2022-04-11 | [11763](https://github.com/airbytehq/airbyte/pull/11763) | Upgrade to Python 3.9 | +| 0.1.3 | 2021-12-23 | [8434](https://github.com/airbytehq/airbyte/pull/8434) | Update fields in source-connectors specifications | +| 0.1.2 | 2021-08-04 | [5152](https://github.com/airbytehq/airbyte/pull/5152) | Fix connector spec.json | +| 0.1.1 | 2021-07-06 | [4539](https://github.com/airbytehq/airbyte/pull/4539) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | +| 0.1.0 | 2021-06-23 | [4122](https://github.com/airbytehq/airbyte/pull/4122) | Initial release supporting the LookupEvent API | diff --git a/docs/integrations/sources/clickup-api.md b/docs/integrations/sources/clickup-api.md new file mode 100644 index 0000000000000..3801d57089346 --- /dev/null +++ b/docs/integrations/sources/clickup-api.md @@ -0,0 +1,9 @@ +# ClickUP API + +The Airbyte Source for [ClickUP API](https://clickup.com/api/) + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :--------------------- | +| 0.1.0 | 2022-11-13 | [17770](https://github.com/airbytehq/airbyte/pull/17770) | 🎉 New Source: ClickUp | diff --git a/docs/integrations/sources/clockify.md b/docs/integrations/sources/clockify.md new file mode 100644 index 0000000000000..5e5c386f57630 --- /dev/null +++ b/docs/integrations/sources/clockify.md @@ -0,0 +1,9 @@ +# Clockify + +The Airbyte Source for [Clockify](https://clockify.me) + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------- | +| 0.1.0 | 2022-10-26 | [17767](https://github.com/airbytehq/airbyte/pull/17767) | 🎉 New Connector: Clockify [python cdk] | diff --git a/docs/integrations/sources/coda.md b/docs/integrations/sources/coda.md index 301f22536f3b5..b7c5438bd0409 100755 --- a/docs/integrations/sources/coda.md +++ b/docs/integrations/sources/coda.md @@ -7,6 +7,7 @@ This page contains the setup guide and reference information for the Coda source You can find or create authentication tokens within [Coda](https://coda.io/account#apiSettings). ## Setup guide + ## Step 1: Set up the Coda connector in Airbyte ### For Airbyte Cloud: @@ -22,7 +23,7 @@ You can find or create authentication tokens within [Coda](https://coda.io/accou ### For Airbyte OSS: 1. Navigate to the Airbyte Open Source dashboard. -2. Set the name for your source. +2. Set the name for your source. 3. Enter your `auth_token` - Coda Authentication Token with the necessary permissions \(described below\). 4. Enter your `doc_id` - Document id for a specific document created on Coda. You can check it under [Advanced Settings](https://coda.io/account) by exporting data and copying the id in doc_manifest.json from the downloaded zip. @@ -33,7 +34,7 @@ You can find or create authentication tokens within [Coda](https://coda.io/accou The Coda source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): | Feature | Supported? | -| :---------------- |:-----------| +| :---------------- | :--------- | | Full Refresh Sync | Yes | | Incremental Sync | No | | SSL connection | No | @@ -41,22 +42,25 @@ The Coda source connector supports the following [sync modes](https://docs.airby ## Supported Streams -* [Docs](https://coda.io/developers/apis/v1#tag/Docs/operation/listDocs) -* [Permissions](https://coda.io/developers/apis/v1#tag/Permissions/operation/getPermissions) -* [Categories](https://coda.io/developers/apis/v1#tag/Publishing/operation/listCategories) -* [Pages](https://coda.io/developers/apis/v1#tag/Pages/operation/listPages) -* [Tables](https://coda.io/developers/apis/v1#tag/Tables/operation/listTables) -* [Formulas](https://coda.io/developers/apis/v1#tag/Formulas/operation/listFormulas) -* [Controls](https://coda.io/developers/apis/v1#tag/Controls/operation/listControls) - - +- [Docs](https://coda.io/developers/apis/v1#tag/Docs/operation/listDocs) +- [Permissions](https://coda.io/developers/apis/v1#tag/Permissions/operation/getPermissions) +- [Categories](https://coda.io/developers/apis/v1#tag/Publishing/operation/listCategories) +- [Pages](https://coda.io/developers/apis/v1#tag/Pages/operation/listPages) +- [Tables](https://coda.io/developers/apis/v1#tag/Tables/operation/listTables) +- [Formulas](https://coda.io/developers/apis/v1#tag/Formulas/operation/listFormulas) +- [Controls](https://coda.io/developers/apis/v1#tag/Controls/operation/listControls) ## Data type map | Integration Type | Airbyte Type | -|:-----------------| :----------- | +| :--------------- | :----------- | | `string` | `string` | | `integer` | `number` | | `array` | `array` | | `object` | `object` | +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------- | +| 0.1.0 | 2022-11-17 | [18675](https://github.com/airbytehq/airbyte/pull/18675) | 🎉 New source: Coda [python cdk] | diff --git a/docs/integrations/sources/coinmarketcap.md b/docs/integrations/sources/coinmarketcap.md index db336c0fa74e2..ca07a8374b802 100644 --- a/docs/integrations/sources/coinmarketcap.md +++ b/docs/integrations/sources/coinmarketcap.md @@ -6,18 +6,18 @@ This source can sync data from the [Coinmarketcap API](https://coinmarketcap.com ## This Source Supports the Following Streams -* [categories](https://coinmarketcap.com/api/documentation/v1/#operation/getV1CryptocurrencyCategories) -* [listing](https://coinmarketcap.com/api/documentation/v1/#operation/getV1CryptocurrencyListingsLatest) -* [quotes](https://coinmarketcap.com/api/documentation/v1/#operation/getV2CryptocurrencyQuotesLatest) -* [fiat](https://coinmarketcap.com/api/documentation/v1/#tag/fiat) -* [exchange](https://coinmarketcap.com/api/documentation/v1/#tag/exchange) +- [categories](https://coinmarketcap.com/api/documentation/v1/#operation/getV1CryptocurrencyCategories) +- [listing](https://coinmarketcap.com/api/documentation/v1/#operation/getV1CryptocurrencyListingsLatest) +- [quotes](https://coinmarketcap.com/api/documentation/v1/#operation/getV2CryptocurrencyQuotesLatest) +- [fiat](https://coinmarketcap.com/api/documentation/v1/#tag/fiat) +- [exchange](https://coinmarketcap.com/api/documentation/v1/#tag/exchange) ### Features -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental Sync | No | | +| Feature | Supported?\(Yes/No\) | Notes | +| :---------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental Sync | No | | ### Performance considerations @@ -27,13 +27,14 @@ Coinmarketcap APIs are under rate limits for the number of API calls allowed per ### Requirements -* [API token](https://coinmarketcap.com/api/documentation/v1/#section/Authentication) -* Data Type: - * latest - * historical +- [API token](https://coinmarketcap.com/api/documentation/v1/#section/Authentication) +- Data Type: + - latest + - historical ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :-------------------------------------------------------- | :----------------------------------------- | -| 0.1.0 | 2022-10-29 | [#18565](https://github.com/airbytehq/airbyte/pull/18565) | 🎉 New Source: Coinmarketcap API [low-code CDK] | \ No newline at end of file +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :-------------------------------------------------------- | :---------------------------------------------- | +| 0.1.1 | 2022-11-01 | [#18790](https://github.com/airbytehq/airbyte/pull/18790) | Correct coinmarket spec | +| 0.1.0 | 2022-10-29 | [#18565](https://github.com/airbytehq/airbyte/pull/18565) | 🎉 New Source: Coinmarketcap API [low-code CDK] | diff --git a/docs/integrations/sources/copper.md b/docs/integrations/sources/copper.md index ee70d64531fc0..939670690390f 100644 --- a/docs/integrations/sources/copper.md +++ b/docs/integrations/sources/copper.md @@ -7,31 +7,37 @@ This page contains the setup guide and reference information for the Copper sour This Copper source uses the [Copper API](https://developer.copper.com/). ## Setup guide + ### Step 1: Set up Copper #### For Airbyte OSS: + 1. Navigate to the Airbyte Open Source dashboard -2. Enter a name for your source +2. Enter a name for your source 3. Enter your Copper `api_key`. This can be generated by logging into Copper -> Settings -> Integrations -> API Keys 4. Enter your Copper `user_email`. The email used to login to Copper -7. Click **Set up source** +5. Click **Set up source** ## Supported sync modes The Copper source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): -| Feature | Supported? | -| :--- | :-- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | No | -| Replicate Incremental Deletes | No | -| SSL connection | Yes | -| Namespaces | No | +| Feature | Supported? | +| :---------------------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | No | +| Replicate Incremental Deletes | No | +| SSL connection | Yes | +| Namespaces | No | ## Supported Streams -* [People](https://developer.copper.com/people/list-people-search.html) -* [Companies](https://developer.copper.com/companies/list-companies-search.html) -* [Projects](https://developer.copper.com/projects/list-projects-search.html) +- [People](https://developer.copper.com/people/list-people-search.html) +- [Companies](https://developer.copper.com/companies/list-companies-search.html) +- [Projects](https://developer.copper.com/projects/list-projects-search.html) +## Changelog +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :-------------------------------------------------------- | :--------------------------------- | +| 0.1.0 | 2022-11-17 | [#18848](https://github.com/airbytehq/airbyte/pull/18848) | 🎉 New Source: Copper [python cdk] | diff --git a/docs/integrations/sources/dixa.md b/docs/integrations/sources/dixa.md index 16a66c9f159c1..c1a1b1c961bf6 100644 --- a/docs/integrations/sources/dixa.md +++ b/docs/integrations/sources/dixa.md @@ -2,34 +2,34 @@ ## Sync overview -This source can sync data for the [Dixa conversation\_export API](https://support.dixa.help/en/articles/174-export-conversations-via-api). It supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. +This source can sync data for the [Dixa conversation_export API](https://support.dixa.help/en/articles/174-export-conversations-via-api). It supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. ### Output schema This Source is capable of syncing the following Streams: -* [Conversation export](https://support.dixa.help/en/articles/174-export-conversations-via-api) +- [Conversation export](https://support.dixa.help/en/articles/174-export-conversations-via-api) ### Data type mapping | Integration Type | Airbyte Type | Notes | -| :--- | :--- | :--- | -| `string` | `string` | | -| `int` | `integer` | | -| `timestamp` | `integer` | | -| `array` | `array` | | +| :--------------- | :----------- | :---- | +| `string` | `string` | | +| `int` | `integer` | | +| `timestamp` | `integer` | | +| `array` | `array` | | ### Features -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental Sync | Yes | | -| Namespaces | No | | +| Feature | Supported?\(Yes/No\) | Notes | +| :---------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental Sync | Yes | | +| Namespaces | No | | ### Performance considerations -The connector is limited by standard Dixa conversation\_export API [limits](https://support.dixa.help/en/articles/174-export-conversations-via-api). It should not run into limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. +The connector is limited by standard Dixa conversation_export API [limits](https://support.dixa.help/en/articles/174-export-conversations-via-api). It should not run into limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. When using the connector, keep in mind that increasing the `batch_size` parameter will decrease the number of requests sent to the API, but increase the response and processing time. @@ -37,7 +37,7 @@ When using the connector, keep in mind that increasing the `batch_size` paramete ### Requirements -* Dixa API token +- Dixa API token ### Setup guide @@ -49,9 +49,9 @@ When using the connector, keep in mind that increasing the `batch_size` paramete ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.2 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.1.1 | 2021-08-12 | [5367](https://github.com/airbytehq/airbyte/pull/5367) | Migrated to CI Sandbox, refactorred code structure for future support | -| 0.1.0 | 2021-07-07 | [4358](https://github.com/airbytehq/airbyte/pull/4358) | New source | - +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------- | +| 0.1.3 | 2022-07-07 | [14437](https://github.com/airbytehq/airbyte/pull/14437) | 🎉 Source Dixa: bump version 0.1.3 | +| 0.1.2 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | +| 0.1.1 | 2021-08-12 | [5367](https://github.com/airbytehq/airbyte/pull/5367) | Migrated to CI Sandbox, refactorred code structure for future support | +| 0.1.0 | 2021-07-07 | [4358](https://github.com/airbytehq/airbyte/pull/4358) | New source | diff --git a/docs/integrations/sources/dv360.md b/docs/integrations/sources/dv-360.md similarity index 67% rename from docs/integrations/sources/dv360.md rename to docs/integrations/sources/dv-360.md index c1f3b6e76d7cd..9e4341f1d8479 100644 --- a/docs/integrations/sources/dv360.md +++ b/docs/integrations/sources/dv-360.md @@ -8,21 +8,20 @@ DoubleClick Bid Manager API `v1.1` is the latest available and recommended versi ## Features -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental Sync | Yes | - +| Feature | Supported? | +| :---------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental Sync | Yes | ## Supported Tables This source is capable of syncing the following tables and their data: -* Audience Composition -* Floodlight -* Reach -* Standard -* Unique Reach Audience +- Audience Composition +- Floodlight +- Reach +- Standard +- Unique Reach Audience **Note**: It is recommended to first build the desired report in the UI to avoid any errors, since there are several limilations and requirements pertaining to reporting types, filters, dimensions, and metrics (such as valid combinations of metrics and dimensions). @@ -33,7 +32,7 @@ Available filters and metrics are provided in this [page](https://developers.goo ## Getting Started \(Airbyte-Cloud\) 1. Click `Authenticate your Display & Video 360 account` to sign in with Google and authorize your account. -2. Get the partner ID for your account. +2. Get the partner ID for your account. 3. Fill out a start date, and optionally, an end date and filters (check the [Queries documentation](https://developers.google.com/bid-manager/v1.1/queries)) . 4. You're done. @@ -43,28 +42,28 @@ Available filters and metrics are provided in this [page](https://developers.goo You can use the [setup tool](https://console.developers.google.com/start/api?id=doubleclickbidmanager&credential=client_key) to create credentials and enable the DBM API in the Google API Console. -* access_token -* refresh_token -* token_uri -* client_id -* client_secret -* start_date -* end_date -* partner_id -* filters +- access_token +- refresh_token +- token_uri +- client_id +- client_secret +- start_date +- end_date +- partner_id +- filters #### Setup guide -* [Getting Started guide](https://developers.google.com/bid-manager/guides/getting-started-api) +- [Getting Started guide](https://developers.google.com/bid-manager/guides/getting-started-api) -* [Using OAuth 2.0 to Access Google APIs](https://developers.google.com/identity/protocols/oauth2/web-server#enable-apis) +- [Using OAuth 2.0 to Access Google APIs](https://developers.google.com/identity/protocols/oauth2/web-server#enable-apis) ## Rate Limiting & Performance Considerations \(Airbyte Open Source\) This source is constrained by the limits set by the DBM API. You can read more about those limits in the [Display & Video 360 docs](https://developers.google.com/bid-manager/guides/scheduled-reports/best-practices#consider_reporting_quotas). -## CHANGELOG +## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.1 | 2022--| [11828](https://github.com/airbytehq/airbyte/pull/11828) | Release Native Display & Video 360 Connector | \ No newline at end of file +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------- | +| 0.1.0 | 2022-09-28 | [11828](https://github.com/airbytehq/airbyte/pull/11828) | Release Native Display & Video 360 Connector | diff --git a/docs/integrations/sources/exchangeratesapi.md b/docs/integrations/sources/exchange-rates.md similarity index 85% rename from docs/integrations/sources/exchangeratesapi.md rename to docs/integrations/sources/exchange-rates.md index 6b4f79a305a1b..aeaefd189861b 100644 --- a/docs/integrations/sources/exchangeratesapi.md +++ b/docs/integrations/sources/exchange-rates.md @@ -12,8 +12,8 @@ It contains one stream: `exchange_rates` Each record in the stream contains many fields: -* The date of the record -* One field for every supported [currency](https://www.ecb.europa.eu/stats/policy_and_exchange_rates/euro_reference_exchange_rates/html/index.en.html) which contain the value of that currency on that date. +- The date of the record +- One field for every supported [currency](https://www.ecb.europa.eu/stats/policy_and_exchange_rates/euro_reference_exchange_rates/html/index.en.html) which contain the value of that currency on that date. #### Data type mapping @@ -21,17 +21,17 @@ Currencies are `number` and the date is a `string`. #### Features -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | Yes | -| Namespaces | No | +| Feature | Supported? | +| :------------------------ | :--------- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | Yes | +| Namespaces | No | ### Getting started ### Requirements -* API Access Key +- API Access Key ### Setup guide @@ -44,14 +44,14 @@ If you have `free` subscription plan \(you may check it [here](https://manage.ex ## Changelog -| Version | Date | Pull Request | Subject | -|:--------| :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------- | -| 1.2.7 | 2022-10-31 | [18726](https://github.com/airbytehq/airbyte/pull/18726) | Fix handling error during check connection | -| 1.2.6 | 2022-08-23 | [15884](https://github.com/airbytehq/airbyte/pull/15884) | Migrated to new API Layer endpoint | -| 0.2.6 | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Update connector to use a `spec.yaml` | -| 0.2.5 | 2021-11-12 | [7936](https://github.com/airbytehq/airbyte/pull/7936) | Add ignore_weekends boolean option | -| 0.2.4 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.2.3 | 2021-06-06 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for kubernetes support | -| 0.2.2 | 2021-05-28 | [3677](https://github.com/airbytehq/airbyte/pull/3677) | Adding clearer error message when a currency isn't supported. access_key field in spec.json was marked as sensitive | -| 0.2.0 | 2021-05-26 | [3566](https://github.com/airbytehq/airbyte/pull/3566) | Move from `api.ratesapi.io/` to `api.exchangeratesapi.io/`. Add required field `access_key` to `config.json`. | -| 0.1.0 | 2021-04-19 | [2942](https://github.com/airbytehq/airbyte/pull/2942) | Implement Exchange API using the CDK | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------ | +| 1.2.7 | 2022-10-31 | [18726](https://github.com/airbytehq/airbyte/pull/18726) | Fix handling error during check connection | +| 1.2.6 | 2022-08-23 | [15884](https://github.com/airbytehq/airbyte/pull/15884) | Migrated to new API Layer endpoint | +| 0.2.6 | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Update connector to use a `spec.yaml` | +| 0.2.5 | 2021-11-12 | [7936](https://github.com/airbytehq/airbyte/pull/7936) | Add ignore_weekends boolean option | +| 0.2.4 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | +| 0.2.3 | 2021-06-06 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for kubernetes support | +| 0.2.2 | 2021-05-28 | [3677](https://github.com/airbytehq/airbyte/pull/3677) | Adding clearer error message when a currency isn't supported. access_key field in spec.json was marked as sensitive | +| 0.2.0 | 2021-05-26 | [3566](https://github.com/airbytehq/airbyte/pull/3566) | Move from `api.ratesapi.io/` to `api.exchangeratesapi.io/`. Add required field `access_key` to `config.json`. | +| 0.1.0 | 2021-04-19 | [2942](https://github.com/airbytehq/airbyte/pull/2942) | Implement Exchange API using the CDK | diff --git a/docs/integrations/sources/file.md b/docs/integrations/sources/file.md index 780070d110143..8b5d56626cd29 100644 --- a/docs/integrations/sources/file.md +++ b/docs/integrations/sources/file.md @@ -54,12 +54,15 @@ This source produces a single table for the target file as it replicates only on ## Getting Started + **For Airbyte Cloud:** Setup through Airbyte Cloud will be exactly the same as the open-source setup, except for the fact that local files are disabled. + + **For Airbyte Open Source:** 1. Once the File Source is selected, you should define both the storage provider along its URL and format of the file. @@ -68,20 +71,20 @@ Setup through Airbyte Cloud will be exactly the same as the open-source setup, e #### Provider Specific Information -* In case of Google Drive, it is necesary to use the Download URL, the format for that is `https://drive.google.com/uc?export=download&id=[DRIVE_FILE_ID]` where `[DRIVE_FILE_ID]` is the string found in the Share URL here `https://drive.google.com/file/d/[DRIVE_FILE_ID]/view?usp=sharing` -* In case of GCS, it is necessary to provide the content of the service account keyfile to access private buckets. See settings of [BigQuery Destination](../destinations/bigquery.md) -* In case of AWS S3, the pair of `aws_access_key_id` and `aws_secret_access_key` is necessary to access private S3 buckets. -* In case of AzBlob, it is necessary to provide the `storage_account` in which the blob you want to access resides. Either `sas_token` [(info)](https://docs.microsoft.com/en-us/azure/storage/blobs/sas-service-create?tabs=dotnet) or `shared_key` [(info)](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?tabs=azure-portal) is necessary to access private blobs. +- In case of Google Drive, it is necesary to use the Download URL, the format for that is `https://drive.google.com/uc?export=download&id=[DRIVE_FILE_ID]` where `[DRIVE_FILE_ID]` is the string found in the Share URL here `https://drive.google.com/file/d/[DRIVE_FILE_ID]/view?usp=sharing` +- In case of GCS, it is necessary to provide the content of the service account keyfile to access private buckets. See settings of [BigQuery Destination](../destinations/bigquery.md) +- In case of AWS S3, the pair of `aws_access_key_id` and `aws_secret_access_key` is necessary to access private S3 buckets. +- In case of AzBlob, it is necessary to provide the `storage_account` in which the blob you want to access resides. Either `sas_token` [(info)](https://docs.microsoft.com/en-us/azure/storage/blobs/sas-service-create?tabs=dotnet) or `shared_key` [(info)](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?tabs=azure-portal) is necessary to access private blobs. ### Reader Options -The Reader in charge of loading the file format is currently based on [Pandas IO Tools](https://pandas.pydata.org/pandas-docs/stable/user\_guide/io.html). It is possible to customize how to load the file into a Pandas DataFrame as part of this Source Connector. This is doable in the `reader_options` that should be in JSON format and depends on the chosen file format. See pandas' documentation, depending on the format: +The Reader in charge of loading the file format is currently based on [Pandas IO Tools](https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html). It is possible to customize how to load the file into a Pandas DataFrame as part of this Source Connector. This is doable in the `reader_options` that should be in JSON format and depends on the chosen file format. See pandas' documentation, depending on the format: -For example, if the format `CSV` is selected, then options from the [read\_csv](https://pandas.pydata.org/pandas-docs/stable/user\_guide/io.html#io-read-csv-table) functions are available. +For example, if the format `CSV` is selected, then options from the [read_csv](https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#io-read-csv-table) functions are available. -* It is therefore possible to customize the `delimiter` (or `sep`) to in case of tab separated files. -* Header line can be ignored with `header=0` and customized with `names` -* etc +- It is therefore possible to customize the `delimiter` (or `sep`) to in case of tab separated files. +- Header line can be ignored with `header=0` and customized with `names` +- etc We would therefore provide in the `reader_options` the following json: @@ -89,7 +92,7 @@ We would therefore provide in the `reader_options` the following json: { "sep" : "\t", "header" : 0, "names": "column1, column2"} ``` -In case you select `JSON` format, then options from the [read\_json](https://pandas.pydata.org/pandas-docs/stable/user\_guide/io.html#io-json-reader) reader are available. +In case you select `JSON` format, then options from the [read_json](https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#io-json-reader) reader are available. For example, you can use the `{"orient" : "records"}` to change how orientation of data is loaded (if data is `[{column -> value}, … , {column -> value}]`) @@ -103,19 +106,19 @@ Normally, Airbyte tries to infer the data type from the source, but you can use Here are a list of examples of possible file inputs: -| Dataset Name | Storage | URL | Reader Impl | Service Account | Description | -| ------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | ----------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| epidemiology | HTTPS | [https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv](https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv) | | | [COVID-19 Public dataset](https://console.cloud.google.com/marketplace/product/bigquery-public-datasets/covid19-public-data-program?filter=solution-type:dataset\&id=7d6cc408-53c8-4485-a187-b8cb9a5c0b56) on BigQuery | -| hr\_and\_financials | GCS | gs://airbyte-vault/financial.csv | smart\_open or gcfs | {"type": "service\_account", "private\_key\_id": "XXXXXXXX", ...} | data from a private bucket, a service account is necessary | -| landsat\_index | GCS | gcp-public-data-landsat/index.csv.gz | smart\_open | | Using smart\_open, we don't need to specify the compression (note the gs:// is optional too, same for other providers) | +| Dataset Name | Storage | URL | Reader Impl | Service Account | Description | +| ----------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------ | -------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| epidemiology | HTTPS | [https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv](https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv) | | | [COVID-19 Public dataset](https://console.cloud.google.com/marketplace/product/bigquery-public-datasets/covid19-public-data-program?filter=solution-type:dataset&id=7d6cc408-53c8-4485-a187-b8cb9a5c0b56) on BigQuery | +| hr_and_financials | GCS | gs://airbyte-vault/financial.csv | smart_open or gcfs | {"type": "service_account", "private_key_id": "XXXXXXXX", ...} | data from a private bucket, a service account is necessary | +| landsat_index | GCS | gcp-public-data-landsat/index.csv.gz | smart_open | | Using smart_open, we don't need to specify the compression (note the gs:// is optional too, same for other providers) | Examples with reader options: -| Dataset Name | Storage | URL | Reader Impl | Reader Options | Description | -| -------------- | ------- | ----------------------------------------------- | ----------- | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | -| landsat\_index | GCS | gs://gcp-public-data-landsat/index.csv.gz | GCFS | {"compression": "gzip"} | Additional reader options to specify a compression option to `read_csv` | -| GDELT | S3 | s3://gdelt-open-data/events/20190914.export.csv | | {"sep": "\t", "header": null} | Here is TSV data separated by tabs without header row from [AWS Open Data](https://registry.opendata.aws/gdelt/) | -| server\_logs | local | /local/logs.log | | {"sep": ";"} | After making sure a local text file exists at `/tmp/airbyte_local/logs.log` with logs file from some server that are delimited by ';' delimiters | +| Dataset Name | Storage | URL | Reader Impl | Reader Options | Description | +| ------------- | ------- | ----------------------------------------------- | ----------- | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | +| landsat_index | GCS | gs://gcp-public-data-landsat/index.csv.gz | GCFS | {"compression": "gzip"} | Additional reader options to specify a compression option to `read_csv` | +| GDELT | S3 | s3://gdelt-open-data/events/20190914.export.csv | | {"sep": "\t", "header": null} | Here is TSV data separated by tabs without header row from [AWS Open Data](https://registry.opendata.aws/gdelt/) | +| server_logs | local | /local/logs.log | | {"sep": ";"} | After making sure a local text file exists at `/tmp/airbyte_local/logs.log` with logs file from some server that are delimited by ';' delimiters | Example for SFTP: @@ -127,15 +130,16 @@ Please see (or add) more at `airbyte-integrations/connectors/source-file/integra ## Performance Considerations and Notes -In order to read large files from a remote location, this connector uses the [smart\_open](https://pypi.org/project/smart-open/) library. However, it is possible to switch to either [GCSFS](https://gcsfs.readthedocs.io/en/latest/) or [S3FS](https://s3fs.readthedocs.io/en/latest/) implementations as it is natively supported by the `pandas` library. This choice is made possible through the optional `reader_impl` parameter. +In order to read large files from a remote location, this connector uses the [smart_open](https://pypi.org/project/smart-open/) library. However, it is possible to switch to either [GCSFS](https://gcsfs.readthedocs.io/en/latest/) or [S3FS](https://s3fs.readthedocs.io/en/latest/) implementations as it is natively supported by the `pandas` library. This choice is made possible through the optional `reader_impl` parameter. -* Note that for local filesystem, the file probably have to be stored somewhere in the `/tmp/airbyte_local` folder with the same limitations as the [CSV Destination](../destinations/local-csv.md) so the `URL` should also starts with `/local/`. -* The JSON implementation needs to be tweaked in order to produce more complex catalog and is still in an experimental state: Simple JSON schemas should work at this point but may not be well handled when there are multiple layers of nesting. +- Note that for local filesystem, the file probably have to be stored somewhere in the `/tmp/airbyte_local` folder with the same limitations as the [CSV Destination](../destinations/local-csv.md) so the `URL` should also starts with `/local/`. +- The JSON implementation needs to be tweaked in order to produce more complex catalog and is still in an experimental state: Simple JSON schemas should work at this point but may not be well handled when there are multiple layers of nesting. ## Changelog | Version | Date | Pull Request | Subject | -| ------- | ---------- | -------------------------------------------------------- | -------------------------------------------------------- | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------- | +| 0.2.31 | 2022-11-17 | [19567](https://github.com/airbytehq/airbyte/pull/19567) | Source File: bump 0.2.31 | | 0.2.30 | 2022-11-10 | [19222](https://github.com/airbytehq/airbyte/pull/19222) | Use AirbyteConnectionStatus for "check" command | | 0.2.29 | 2022-11-08 | [18587](https://github.com/airbytehq/airbyte/pull/18587) | Fix pandas read_csv header none issue. | | 0.2.28 | 2022-10-27 | [18428](https://github.com/airbytehq/airbyte/pull/18428) | Added retry logic for `Connection reset error - 104` | @@ -159,7 +163,7 @@ In order to read large files from a remote location, this connector uses the [sm | 0.2.7 | 2021-10-28 | [7387](https://github.com/airbytehq/airbyte/pull/7387) | Migrate source to CDK structure, add SAT testing. | | 0.2.6 | 2021-08-26 | [5613](https://github.com/airbytehq/airbyte/pull/5613) | Add support to xlsb format | | 0.2.5 | 2021-07-26 | [4953](https://github.com/airbytehq/airbyte/pull/4953) | Allow non-default port for SFTP type | -| 0.2.4 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add AIRBYTE\_ENTRYPOINT for Kubernetes support | +| 0.2.4 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add AIRBYTE_ENTRYPOINT for Kubernetes support | | 0.2.3 | 2021-06-01 | [3771](https://github.com/airbytehq/airbyte/pull/3771) | Add Azure Storage Blob Files option | | 0.2.2 | 2021-04-16 | [2883](https://github.com/airbytehq/airbyte/pull/2883) | Fix CSV discovery memory consumption | | 0.2.1 | 2021-04-03 | [2726](https://github.com/airbytehq/airbyte/pull/2726) | Fix base connector versioning | diff --git a/docs/integrations/sources/gridly.md b/docs/integrations/sources/gridly.md index 1f98968588b62..64cbd1a8622f0 100644 --- a/docs/integrations/sources/gridly.md +++ b/docs/integrations/sources/gridly.md @@ -9,27 +9,31 @@ A Gridly account. ## Setup guide ### Get api Key + 1. To quickly get your API key, access your Gridly Dashboard, then select a Grid View and you can find the key in API quick start right panel. -![img.png](../../.gitbook/assets/gridly_api_key1.png) + ![img.png](../../.gitbook/assets/gridly_api_key1.png) 2. Owner and Administrators can go to Settings/API keys to create company-level API keys with scoped privileges and accesses. -![img.png](../../.gitbook/assets/gridly_api_key2.png) + ![img.png](../../.gitbook/assets/gridly_api_key2.png) ### Get grid id + The grid id is available in the url. Gridly support version control, by default the `grid id` is the same to the `branch id` when `Master` branch is selected. For fetching records on other branches, use `branch id` instead. ![img.png](../../.gitbook/assets/gridly_grid_id.png) - ## Supported sync modes -| Feature | Supported? | -| :---------------------------- |:-----------| -| Full Refresh Sync | Yes | -| Incremental Sync | No | - +| Feature | Supported? | +| :---------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental Sync | No | ## Supported Streams -* [Records](https://www.gridly.com/docs/api/#record) +- [Records](https://www.gridly.com/docs/api/#record) ## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------- | +| 0.1.1 | 2022-12-08 | [20048](https://github.com/airbytehq/airbyte/pull/20048) | Source Gridly: add icon and make grid_id parameter required | diff --git a/docs/integrations/sources/hellobaton.md b/docs/integrations/sources/hellobaton.md index 16d8d401d884f..950ac1c0060be 100644 --- a/docs/integrations/sources/hellobaton.md +++ b/docs/integrations/sources/hellobaton.md @@ -6,37 +6,37 @@ This source can sync data from the [baton API](https://app.hellobaton.com/api/re ## This Source Supports the Following Streams -* activity -* companies -* milestones -* phases -* project_attachments -* projects -* task_attachemnts -* tasks -* templates -* time_entries -* users +- activity +- companies +- milestones +- phases +- project_attachments +- projects +- task_attachemnts +- tasks +- templates +- time_entries +- users Baton adds new streams fairly regularly please submit an issue or PR if this project doesn't support required streams for your use case. ### Data type mapping | Integration Type | Airbyte Type | Notes | -| :--- | :--- | :--- | -| `string` | `string` | | -| `integer` | `integer` | | -| `number` | `number` | | -| `array` | `array` | | -| `object` | `object` | | +| :--------------- | :----------- | :---- | +| `string` | `string` | | +| `integer` | `integer` | | +| `number` | `number` | | +| `array` | `array` | | +| `object` | `object` | | ### Features -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental Sync | No | | -| Namespaces | No | | +| Feature | Supported?\(Yes/No\) | Notes | +| :---------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental Sync | No | | +| Namespaces | No | | ### Performance considerations @@ -46,5 +46,11 @@ The connector is rate limited at 1000 requests per minute per api key. If you fi ### Requirements -* Baton account -* Baton api key \ No newline at end of file +- Baton account +- Baton api key + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :----------------------------------------------------- | :------------------------ | +| 0.1.0 | 2022-01-14 | [8461](https://github.com/airbytehq/airbyte/pull/8461) | 🎉 New Source: Hellobaton | diff --git a/docs/integrations/sources/kustomer-singer.md b/docs/integrations/sources/kustomer-singer.md new file mode 100644 index 0000000000000..5fc12662e4e91 --- /dev/null +++ b/docs/integrations/sources/kustomer-singer.md @@ -0,0 +1,47 @@ +# Kustomer + +## Sync overview + +The Kustomer source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. + +This source can sync data for the [Kustomer API](https://developer.kustomer.com/kustomer-api-docs). + +This Source Connector is based on a [Singer tap](https://github.com/singer-io/tap-kustomer). + +### Output schema + +This Source is capable of syncing the following core Streams: + +- [Conversations](https://developer.kustomer.com/kustomer-api-docs/reference/conversations) +- [Customers](https://developer.kustomer.com/kustomer-api-docs/reference/customers) +- [KObjects](https://developer.kustomer.com/kustomer-api-docs/reference/kobjects-custom-objects) +- [Messages](https://developer.kustomer.com/kustomer-api-docs/reference/messages) +- [Notes](https://developer.kustomer.com/kustomer-api-docs/reference/notes) +- [Shortcuts](https://developer.kustomer.com/kustomer-api-docs/reference/shortcuts) +- [Tags](https://developer.kustomer.com/kustomer-api-docs/reference/tags-knowledge-base) +- [Teams](https://developer.kustomer.com/kustomer-api-docs/reference/teams) +- [Users](https://developer.kustomer.com/kustomer-api-docs/reference/users) + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :------------------------ | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | +| Namespaces | No | | + +### Performance considerations + +Kustomer has some [rate limit restrictions](https://developer.kustomer.com/kustomer-api-docs/reference/rate-limiting). + +## Requirements + +- **Kustomer API token**. See the [Kustomer docs](https://help.kustomer.com/api-keys-SJs5YTIWX) for information on how to obtain an API token. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :----------------------------------------------------- | :-------------------------------------------------------------------------------- | +| 0.1.2 | 2021-12-25 | [8578](https://github.com/airbytehq/airbyte/pull/8578) | Update fields in source-connectors specifications | +| 0.1.1 | 2021-12-22 | [8738](https://github.com/airbytehq/airbyte/pull/8738) | Deleted `user-agent`, `date_window_size`, `page_size_limit` from `spec.json` file | +| 0.1.0 | 2021-07-22 | [4550](https://github.com/airbytehq/airbyte/pull/4550) | Add Kustomer Source Connector | diff --git a/docs/integrations/sources/kustomer.md b/docs/integrations/sources/kustomer.md deleted file mode 100644 index cbd0471eebbee..0000000000000 --- a/docs/integrations/sources/kustomer.md +++ /dev/null @@ -1,47 +0,0 @@ -# Kustomer - -## Sync overview - -The Kustomer source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. - -This source can sync data for the [Kustomer API](https://developer.kustomer.com/kustomer-api-docs). - -This Source Connector is based on a [Singer tap](https://github.com/singer-io/tap-kustomer). - -### Output schema - -This Source is capable of syncing the following core Streams: - -* [Conversations](https://developer.kustomer.com/kustomer-api-docs/reference/conversations) -* [Customers](https://developer.kustomer.com/kustomer-api-docs/reference/customers) -* [KObjects](https://developer.kustomer.com/kustomer-api-docs/reference/kobjects-custom-objects) -* [Messages](https://developer.kustomer.com/kustomer-api-docs/reference/messages) -* [Notes](https://developer.kustomer.com/kustomer-api-docs/reference/notes) -* [Shortcuts](https://developer.kustomer.com/kustomer-api-docs/reference/shortcuts) -* [Tags](https://developer.kustomer.com/kustomer-api-docs/reference/tags-knowledge-base) -* [Teams](https://developer.kustomer.com/kustomer-api-docs/reference/teams) -* [Users](https://developer.kustomer.com/kustomer-api-docs/reference/users) - -### Features - -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental - Append Sync | Yes | | -| Namespaces | No | | - -### Performance considerations - -Kustomer has some [rate limit restrictions](https://developer.kustomer.com/kustomer-api-docs/reference/rate-limiting). - -## Requirements - -* **Kustomer API token**. See the [Kustomer docs](https://help.kustomer.com/api-keys-SJs5YTIWX) for information on how to obtain an API token. - -## Changelog - -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.1 | 2021-12-13 | [8738](https://github.com/airbytehq/airbyte/pull/8738) | Deleted `user-agent`, `date_window_size`, `page_size_limit` from `spec.json` file | -| 0.1.0 | 2021-07-22 | [4550](https://github.com/airbytehq/airbyte/pull/4550) | Add Kustomer Source Connector | - diff --git a/docs/integrations/sources/mailersend.md b/docs/integrations/sources/mailersend.md index 5d4601745480a..8c99fb4a7046f 100644 --- a/docs/integrations/sources/mailersend.md +++ b/docs/integrations/sources/mailersend.md @@ -23,5 +23,6 @@ MailerSend has a default [rate limit](https://developers.mailersend.com/general. ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--- | :----------- | :------ | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------- | +| 0.1.0 | 2022-11-13 | [18669](https://github.com/airbytehq/airbyte/pull/18669) | 🎉 New Source: Mailersend [low-code CDK] | diff --git a/docs/integrations/sources/microsoft-dataverse.md b/docs/integrations/sources/microsoft-dataverse.md index 7aca712b33287..3a17d967e7cf7 100644 --- a/docs/integrations/sources/microsoft-dataverse.md +++ b/docs/integrations/sources/microsoft-dataverse.md @@ -14,7 +14,7 @@ This source will automatically discover the schema of the Entities of your Datav ### Data type mapping | Integration Type | Airbyte Type | Notes | -|:-------------------|:--------------------------|:----------------------| +| :----------------- | :------------------------ | :-------------------- | | `String` | `string` | | | `UniqueIdentifier` | `string` | | | `DateTime` | `timestamp with timezone` | | @@ -33,7 +33,7 @@ Other types are defined as `string`. ### Features | Feature | Supported?\(Yes/No\) | Notes | -|:------------------------------|:---------------------|:-----------------------------------------------------------| +| :---------------------------- | :------------------- | :--------------------------------------------------------- | | Full Refresh Sync | Yes | | | Incremental Sync | Yes | | | CDC | Yes | Not all entities support it. Deleted data only have the ID | @@ -45,9 +45,9 @@ Other types are defined as `string`. ### Requirements -* Application \(client\) ID -* Directory \(tenant\) ID -* Client secrets +- Application \(client\) ID +- Directory \(tenant\) ID +- Client secrets ### Setup guide @@ -57,8 +57,8 @@ https://learn.microsoft.com/en-us/power-apps/developer/data-platform/authenticat The procedure to generate the credentials and setup the necessary permissions is well described in this post from Magnetism blog: https://blog.magnetismsolutions.com/blog/paulnieuwelaar/2021/9/21/setting-up-an-application-user-in-dynamics-365 - ## CHANGELOG -| Version | Date | Pull Request | Subject | -|:--------|:-----|:-------------|:--------| +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------- | +| 0.1.0 | 2022-11-14 | [18646](https://github.com/airbytehq/airbyte/pull/18646) | 🎉 New Source: Microsoft Dataverse [python cdk] | diff --git a/docs/integrations/sources/n8n.md b/docs/integrations/sources/n8n.md index d4e1404d68c5a..d212fdbcb4863 100644 --- a/docs/integrations/sources/n8n.md +++ b/docs/integrations/sources/n8n.md @@ -27,5 +27,6 @@ You need a n8n instance or use cloud version ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--- | :----------- | :------ | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------------- | +| 0.1.0 | 2022-11-08 | [18745](https://github.com/airbytehq/airbyte/pull/18745) | 🎉 New Source: N8n [low-code cdk] | diff --git a/docs/integrations/sources/pardot.md b/docs/integrations/sources/pardot.md index 2a02ae782d538..f8f304797a397 100644 --- a/docs/integrations/sources/pardot.md +++ b/docs/integrations/sources/pardot.md @@ -4,6 +4,7 @@ The Airbyte Source for [Salesforce Pardot](https://www.pardot.com/) ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :----------------------------------------------------- | :----------------- | -| 0.1.0 | 2021-11-09 | [7091](https://github.com/airbytehq/airbyte/pull/7091) | New Source: Pardot | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------- | +| 0.1.1 | 2022-12-16 | [19618](https://github.com/airbytehq/airbyte/pull/19618) | Fix `visitors` stream | +| 0.1.0 | 2021-11-19 | [7091](https://github.com/airbytehq/airbyte/pull/7091) | 🎉 New Source: Pardot | diff --git a/docs/integrations/sources/persistiq.md b/docs/integrations/sources/persistiq.md index fa466116718ed..40e6ee64ea9ad 100644 --- a/docs/integrations/sources/persistiq.md +++ b/docs/integrations/sources/persistiq.md @@ -6,22 +6,21 @@ The PersistIq source supports Full Refresh syncs only. This source syncs data for the [PersistIq API](https://apidocs.persistiq.com/#introduction). - ### Output schema This Source is capable of syncing the following streams: -* [Users](https://apidocs.persistiq.com/#users) -* [Leads](https://apidocs.persistiq.com/#leads) -* [Campaigns](https://apidocs.persistiq.com/#campaigns) +- [Users](https://apidocs.persistiq.com/#users) +- [Leads](https://apidocs.persistiq.com/#leads) +- [Campaigns](https://apidocs.persistiq.com/#campaigns) ### Features -| Feature | Supported?\(Yes/No\) -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | No | -| Namespaces | No | +| Feature | Supported?\(Yes/No\) | +| :------------------------ | :------------------- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | No | +| Namespaces | No | ### Performance considerations @@ -31,8 +30,14 @@ The PersistIq connector should not run into PersistIq API limitations under norm ### Requirements -* PersistIq API Key +- PersistIq API Key ### Setup guide Please read [How to find your API key](https://apidocs.persistiq.com/#introduction). + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :----------------------------------------------------- | :----------------------- | +| 0.1.0 | 2022-01-21 | [9515](https://github.com/airbytehq/airbyte/pull/9515) | 🎉 New Source: PersistIq | diff --git a/docs/integrations/sources/pivotal-tracker.md b/docs/integrations/sources/pivotal-tracker.md index 3b823e8c4754f..d533f6c6b061d 100644 --- a/docs/integrations/sources/pivotal-tracker.md +++ b/docs/integrations/sources/pivotal-tracker.md @@ -3,35 +3,36 @@ ## Overview The Pivotal Tracker source supports Full Refresh syncs. It supports pulling from : -* Activity -* Epics -* Labels -* Project Membership -* Projects -* Releases -* Stories + +- Activity +- Epics +- Labels +- Project Membership +- Projects +- Releases +- Stories ### Output schema Output streams: -* [Activity](https://www.pivotaltracker.com/help/api/rest/v5#Activity) -* [Epics](https://www.pivotaltracker.com/help/api/rest/v5#Epics) -* [Labels](https://www.pivotaltracker.com/help/api/rest/v5#Labels) -* [Project Membership](https://www.pivotaltracker.com/help/api/rest/v5#Project_Memberships) -* [Projects](https://www.pivotaltracker.com/help/api/rest/v5#Projects) -* [Releases](https://www.pivotaltracker.com/help/api/rest/v5#Releases) -* [Stories](https://www.pivotaltracker.com/help/api/rest/v5#Stories) +- [Activity](https://www.pivotaltracker.com/help/api/rest/v5#Activity) +- [Epics](https://www.pivotaltracker.com/help/api/rest/v5#Epics) +- [Labels](https://www.pivotaltracker.com/help/api/rest/v5#Labels) +- [Project Membership](https://www.pivotaltracker.com/help/api/rest/v5#Project_Memberships) +- [Projects](https://www.pivotaltracker.com/help/api/rest/v5#Projects) +- [Releases](https://www.pivotaltracker.com/help/api/rest/v5#Releases) +- [Stories](https://www.pivotaltracker.com/help/api/rest/v5#Stories) ### Features -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | Coming soon | +| Feature | Supported? | +| :---------------------------- | :---------- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | Coming soon | | Replicate Incremental Deletes | Coming soon | -| SSL connection | Yes | -| Namespaces | No | +| SSL connection | Yes | +| Namespaces | No | ### Performance considerations @@ -41,13 +42,15 @@ The Pivotal Trakcer connector should not run into Stripe API limitations under n ### Requirements -* Pivotal Trakcer API Token +- Pivotal Trakcer API Token ### Setup guide to create the API Token Access your profile [here](https://www.pivotaltracker.com/profile) go down and click in **Create New Token**. Use this to pull data from Pivotal Tracker. -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.0 | 2022-04-04 | [11060](https://github.com/airbytehq/airbyte/pull/11060) | Initial Release | +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------- | +| 0.1.0 | 2022-04-04 | [11060](https://github.com/airbytehq/airbyte/pull/11060) | Initial Release | diff --git a/docs/integrations/sources/plaid.md b/docs/integrations/sources/plaid.md index f9cc856147a19..889ffb39ea843 100644 --- a/docs/integrations/sources/plaid.md +++ b/docs/integrations/sources/plaid.md @@ -66,6 +66,8 @@ This guide will walk through how to create the credentials you need to run this - We should now have everything we need to configure this source in the UI. +## Changelog + | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------ | | 0.3.2 | 2022-08-02 | [15231](https://github.com/airbytehq/airbyte/pull/15231) | Added min_last_updated_datetime support for Capital One items | diff --git a/docs/integrations/sources/qonto.md b/docs/integrations/sources/qonto.md new file mode 100644 index 0000000000000..18536cfaf890d --- /dev/null +++ b/docs/integrations/sources/qonto.md @@ -0,0 +1,9 @@ +# Qonto + +The Airbyte Source for [Qonto](https://qonto.com) + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------------- | +| 0.1.0 | 2022-11-14 | [17452](https://github.com/airbytehq/airbyte/pull/17452) | 🎉 New Source: Qonto [python cdk] | diff --git a/docs/integrations/sources/quickbooks-singer.md b/docs/integrations/sources/quickbooks-singer.md new file mode 100644 index 0000000000000..540ea748cc73f --- /dev/null +++ b/docs/integrations/sources/quickbooks-singer.md @@ -0,0 +1,84 @@ +# QuickBooks + +## Overview + +The QuickBooks source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. + +This source wraps the [Singer QuickBooks Tap](https://github.com/singer-io/tap-quickbooks). + +### Output schema + +This Source is capable of syncing the following [Streams](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/most-commonly-used/account): + +- [Accounts](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/account) +- [BillPayments](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/billpayment) +- [Budgets](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/budget) +- [Bills](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/bill) +- [Classes](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/class) +- [CreditMemos](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/creditmemo) +- [Customers](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/customer) +- [Departments](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/department) +- [Deposits](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/deposit) +- [Employees](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/employee) +- [Estimates](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/estimate) +- [Invoices](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/invoice) +- [Items](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/item) +- [JournalEntries](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/journalentry) +- [Payments](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/payment) +- [PaymentMethods](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/paymentmethod) +- [Purchases](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/purchase) +- [PurchaseOrders](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/purchaseorder) +- [RefundReceipts](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/refundreceipt) +- [SalesReceipts](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/salesreceipt) +- [TaxAgencies](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/taxagency) +- [TaxCodes](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/taxcode) +- [TaxRates](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/taxrate) +- [Terms](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/term) +- [TimeActivities](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/timeactivity) +- [Transfers](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/transfer) +- [VendorCredits](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/vendorcredit) +- [Vendors](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/vendor) + +### Data type mapping + +| Integration Type | Airbyte Type | Notes | +| :--------------- | :----------- | :---- | +| `string` | `string` | | +| `number` | `number` | | +| `array` | `array` | | +| `object` | `object` | | + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :---------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental Sync | Yes | | +| SSL connection | Yes | | +| Namespaces | No | | + +## Getting started + +1. Create an [Intuit Developer account](https://developer.intuit.com/app/developer/qbo/docs/get-started) +2. Create an app +3. Obtain credentials + +### Requirements + +- Client ID +- Client Secret +- Realm ID +- Refresh token + +The easiest way to get these credentials is by using Quickbook's [OAuth 2.0 playground](https://developer.intuit.com/app/developer/qbo/docs/develop/authentication-and-authorization/oauth-2.0-playground) + +**Important note:** The refresh token expires every 100 days. You will need to manually revisit the Oauth playground to obtain a refresh token every 100 days, or your syncs will expire. We plan on offering full Oauth support soon so you don't need to redo this process manually. + +## CHANGELOG + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------- | +| `0.1.5` | 2022-02-17 | [10346](https://github.com/airbytehq/airbyte/pull/10346) | Update label `Quickbooks` -> `QuickBooks` | +| `0.1.4` | 2021-12-20 | [8960](https://github.com/airbytehq/airbyte/pull/8960) | Update connector fields title/description | +| `0.1.3` | 2021-08-10 | [4986](https://github.com/airbytehq/airbyte/pull/4986) | Using number data type for decimal fields instead string | +| `0.1.2` | 2021-07-06 | [4539](https://github.com/airbytehq/airbyte/pull/4539) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | diff --git a/docs/integrations/sources/quickbooks.md b/docs/integrations/sources/quickbooks.md deleted file mode 100644 index 863481bec6c20..0000000000000 --- a/docs/integrations/sources/quickbooks.md +++ /dev/null @@ -1,85 +0,0 @@ -# QuickBooks - -## Overview - -The QuickBooks source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. - -This source wraps the [Singer QuickBooks Tap](https://github.com/singer-io/tap-quickbooks). - -### Output schema - -This Source is capable of syncing the following [Streams](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/most-commonly-used/account): - -* [Accounts](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/account) -* [BillPayments](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/billpayment) -* [Budgets](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/budget) -* [Bills](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/bill) -* [Classes](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/class) -* [CreditMemos](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/creditmemo) -* [Customers](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/customer) -* [Departments](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/department) -* [Deposits](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/deposit) -* [Employees](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/employee) -* [Estimates](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/estimate) -* [Invoices](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/invoice) -* [Items](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/item) -* [JournalEntries](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/journalentry) -* [Payments](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/payment) -* [PaymentMethods](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/paymentmethod) -* [Purchases](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/purchase) -* [PurchaseOrders](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/purchaseorder) -* [RefundReceipts](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/refundreceipt) -* [SalesReceipts](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/salesreceipt) -* [TaxAgencies](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/taxagency) -* [TaxCodes](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/taxcode) -* [TaxRates](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/taxrate) -* [Terms](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/term) -* [TimeActivities](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/timeactivity) -* [Transfers](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/transfer) -* [VendorCredits](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/vendorcredit) -* [Vendors](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/vendor) - -### Data type mapping - -| Integration Type | Airbyte Type | Notes | -| :--- | :--- | :--- | -| `string` | `string` | | -| `number` | `number` | | -| `array` | `array` | | -| `object` | `object` | | - -### Features - -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental Sync | Yes | | -| SSL connection | Yes | | -| Namespaces | No | | - -## Getting started - -1. Create an [Intuit Developer account](https://developer.intuit.com/app/developer/qbo/docs/get-started) -2. Create an app -3. Obtain credentials - -### Requirements - -* Client ID -* Client Secret -* Realm ID -* Refresh token - -The easiest way to get these credentials is by using Quickbook's [OAuth 2.0 playground](https://developer.intuit.com/app/developer/qbo/docs/develop/authentication-and-authorization/oauth-2.0-playground) - -**Important note:** The refresh token expires every 100 days. You will need to manually revisit the Oauth playground to obtain a refresh token every 100 days, or your syncs will expire. We plan on offering full Oauth support soon so you don't need to redo this process manually. - -## CHANGELOG - -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| `0.1.5` | 2022-02-17 | [10346](https://github.com/airbytehq/airbyte/pull/10346) | Update label `Quickbooks` -> `QuickBooks` | -| `0.1.4` | 2021-12-20 | [8960](https://github.com/airbytehq/airbyte/pull/8960) | Update connector fields title/description | -| `0.1.3` | 2021-08-10 | [4986](https://github.com/airbytehq/airbyte/pull/4986) | Using number data type for decimal fields instead string | -| `0.1.2` | 2021-07-06 | [4539](https://github.com/airbytehq/airbyte/pull/4539) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | - diff --git a/docs/integrations/sources/surveysparrow.md b/docs/integrations/sources/survey-sparrow.md similarity index 100% rename from docs/integrations/sources/surveysparrow.md rename to docs/integrations/sources/survey-sparrow.md diff --git a/docs/integrations/sources/twiliotaskrouter.md b/docs/integrations/sources/twilio-taskrouter.md similarity index 79% rename from docs/integrations/sources/twiliotaskrouter.md rename to docs/integrations/sources/twilio-taskrouter.md index 8bd6ffa48267d..4c7d8b8497592 100644 --- a/docs/integrations/sources/twiliotaskrouter.md +++ b/docs/integrations/sources/twilio-taskrouter.md @@ -12,13 +12,14 @@ You can find [Taskrouter](https://console.twilio.com/develop/explore) in the Exp See [docs](https://www.twilio.com/docs/taskrouter/api) for more details. ## Setup guide + ## Step 1: Set up the Twilio Takrouter connector in Airbyte ### For Airbyte Cloud: 1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. -3. On the Set up the source page, enter the name for the Twilio connector and select **Twilio Taskrouter** from the type dropdown. +3. On the Set up the source page, enter the name for the Twilio connector and select **Twilio Taskrouter** from the type dropdown. 4. Enter your `account_sid`. 5. Enter your `auth_token`. 6. Click **Set up source**. @@ -44,12 +45,17 @@ The Twilio source connector supports the following [sync modes](https://docs.air ## Supported Streams -* [Workspaces](https://www.twilio.com/docs/taskrouter/api/workspace) -* [All Workspaces](https://www.twilio.com/docs/taskrouter/api/workspace) -* [Workers](https://www.twilio.com/docs/taskrouter/api/worker) +- [Workspaces](https://www.twilio.com/docs/taskrouter/api/workspace) +- [All Workspaces](https://www.twilio.com/docs/taskrouter/api/workspace) +- [Workers](https://www.twilio.com/docs/taskrouter/api/worker) ## Performance considerations -The Twilio Taskrouter connector will gracefully handle rate limits. +The Twilio Taskrouter connector will gracefully handle rate limits. For more information, see [the Twilio docs for rate limitations](https://support.twilio.com/hc/en-us/articles/360044308153-Twilio-API-response-Error-429-Too-Many-Requests). +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------- | +| 0.1.0 | 2022-11-18 | [18685](https://github.com/airbytehq/airbyte/pull/18685) | 🎉 New Source: Twilio Taskrouter API [low-code cdk] | diff --git a/docs/integrations/sources/us-census.md b/docs/integrations/sources/us-census.md index d81dcaa39a493..6a3b5d5b1a539 100644 --- a/docs/integrations/sources/us-census.md +++ b/docs/integrations/sources/us-census.md @@ -10,19 +10,19 @@ This source always outputs a single stream, `us_census_stream`. The output of th ### Features -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental Sync | No | -| SSL connection | Yes | -| Namespaces | No | +| Feature | Supported? | +| :---------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental Sync | No | +| SSL connection | Yes | +| Namespaces | No | ## Getting started ### Requirements -* US Census API key -* US Census dataset path & query parameters +- US Census API key +- US Census dataset path & query parameters ### Setup guide @@ -30,14 +30,14 @@ Visit the [US Census API page](https://api.census.gov/data/key_signup.html) to o In addition, to understand how to configure the dataset path and query parameters, follow the guide and examples in the [API documentation](https://www.census.gov/data/developers/data-sets.html). Some particularly helpful pages: -* [Available Datasets](https://www.census.gov/data/developers/guidance/api-user-guide.Available_Data.html) -* [Core Concepts](https://www.census.gov/data/developers/guidance/api-user-guide.Core_Concepts.html) -* [Example Queries](https://www.census.gov/data/developers/guidance/api-user-guide.Example_API_Queries.html) +- [Available Datasets](https://www.census.gov/data/developers/guidance/api-user-guide.Available_Data.html) +- [Core Concepts](https://www.census.gov/data/developers/guidance/api-user-guide.Core_Concepts.html) +- [Example Queries](https://www.census.gov/data/developers/guidance/api-user-guide.Example_API_Queries.html) ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.1.0 | 2021-07-20 | [4228](https://github.com/airbytehq/airbyte/pull/4228) | Initial release | - +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :----------------------------------------------------- | :------------------------------------------------ | +| 0.1.2 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Update fields in source-connectors specifications | +| 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | +| 0.1.0 | 2021-07-20 | [4228](https://github.com/airbytehq/airbyte/pull/4228) | Initial release | diff --git a/docs/integrations/sources/webflow.md b/docs/integrations/sources/webflow.md index 0a0d1adb7a6e4..cabeae06101b1 100644 --- a/docs/integrations/sources/webflow.md +++ b/docs/integrations/sources/webflow.md @@ -4,14 +4,15 @@ description: 'This connector extracts "collections" from Webflow' # Webflow -Webflow is a CMS system that is used for publishing websites and blogs. This connector returns data that is made available by [Webflow APIs](https://developers.webflow.com/). +Webflow is a CMS system that is used for publishing websites and blogs. This connector returns data that is made available by [Webflow APIs](https://developers.webflow.com/). -Webflow uses [Collections](https://developers.webflow.com/#collections) to store different kinds of information. A collection can be "Blog Posts", or "Blog Authors", etc. Collection names are not pre-defined, the number of collections is not known in advance, and the schema for each collection may be different. +Webflow uses [Collections](https://developers.webflow.com/#collections) to store different kinds of information. A collection can be "Blog Posts", or "Blog Authors", etc. Collection names are not pre-defined, the number of collections is not known in advance, and the schema for each collection may be different. -This connector dynamically figures our which collections are available, creates the schema for each collection based on data extracted from Webflow, and creates an [Airbyte Stream](https://docs.airbyte.com/connector-development/cdk-python/full-refresh-stream/) for each collection. +This connector dynamically figures our which collections are available, creates the schema for each collection based on data extracted from Webflow, and creates an [Airbyte Stream](https://docs.airbyte.com/connector-development/cdk-python/full-refresh-stream/) for each collection. # Webflow credentials -You should be able to create a Webflow `API key` (aka `API token`) as described in [Intro to the Webflow API](https://university.webflow.com/lesson/intro-to-the-webflow-api). + +You should be able to create a Webflow `API key` (aka `API token`) as described in [Intro to the Webflow API](https://university.webflow.com/lesson/intro-to-the-webflow-api). Once you have the `API Key`/`API token`, you can confirm a [list of available sites](https://developers.webflow.com/#sites) and get their `_id` by executing the following: @@ -27,15 +28,16 @@ Which should respond with something similar to: [{"_id":"","createdOn":"2021-03-26T15:46:04.032Z","name":"Airbyte","shortName":"airbyte-dev","lastPublished":"2022-06-09T12:55:52.533Z","previewUrl":"https://screenshots.webflow.com/sites/","timezone":"America/Los_Angeles","database":""}] ``` -You will need to provide the `Site id` and `API key` to the Webflow connector in order for it to pull data from your Webflow site. +You will need to provide the `Site id` and `API key` to the Webflow connector in order for it to pull data from your Webflow site. # Related tutorial -If you are interested in learning more about the Webflow API and implementation details of this connector, you may wish to consult the [tutorial about how to build a connector to extract data from the Webflow API](https://airbyte.com/tutorials/extract-data-from-the-webflow-api). - -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.0 | 2022-06-22 | [13617](https://github.com/airbytehq/airbyte/pull/13617) | Initial release | -| 0.1.1 | 2022-06-22 | [13617](https://github.com/airbytehq/airbyte/pull/13617) | Update Spec Documentation URL | +If you are interested in learning more about the Webflow API and implementation details of this connector, you may wish to consult the [tutorial about how to build a connector to extract data from the Webflow API](https://airbyte.com/tutorials/extract-data-from-the-webflow-api). +## Changelog +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------- | +| 0.1.2 | 2022-07-14 | [14689](https://github.com/airbytehq/airbyte/pull/14689) | Webflow add ids to streams | +| 0.1.1 | 2022-06-22 | [13617](https://github.com/airbytehq/airbyte/pull/13617) | Update Spec Documentation URL | +| 0.1.0 | 2022-06-22 | [13617](https://github.com/airbytehq/airbyte/pull/13617) | Initial release | diff --git a/docs/integrations/sources/woocommerce.md b/docs/integrations/sources/woocommerce.md index dcf4ef97a77a8..d65b4223391e4 100644 --- a/docs/integrations/sources/woocommerce.md +++ b/docs/integrations/sources/woocommerce.md @@ -6,9 +6,9 @@ This page contains the setup guide and reference information for the WooCommerce To set up the WooCommerce source connector with Airbyte, you must be using: -* WooCommerce 3.5+ -* WordPress 4.4+ -* Pretty permalinks in `Settings > Permalinks` so that the custom endpoints are supported. +- WooCommerce 3.5+ +- WordPress 4.4+ +- Pretty permalinks in `Settings > Permalinks` so that the custom endpoints are supported. e.g. `/%year%/%monthnum%/%day%/%postname%/` You will need to generate new API key with read permissions and use `Customer key` and `Customer Secret`. @@ -49,45 +49,45 @@ You will need to generate new API key with read permissions and use `Customer ke The WooCommerce source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): -* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/glossary#full-refresh-sync) -* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append) -* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) -* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history) +- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/glossary#full-refresh-sync) +- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append) +- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) +- [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history) ## Supported Streams -* [Coupons](https://woocommerce.github.io/woocommerce-rest-api-docs/#coupons) \(Incremental\) -* [Customers](https://woocommerce.github.io/woocommerce-rest-api-docs/#customers) \(Incremental\) -* [orders](https://woocommerce.github.io/woocommerce-rest-api-docs/#orders) \(Incremental\) -* [Order notes](https://woocommerce.github.io/woocommerce-rest-api-docs/#order-notes) -* [Payment gateways](https://woocommerce.github.io/woocommerce-rest-api-docs/#payment-gateways) -* [Product attribute terms](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-attribute-terms) -* [Product attributes](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-attributes) -* [Product categories](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-categories) -* [Product reviews](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-reviews) \(Incremental\) -* [Product shipping classes](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-shipping-classes) -* [Product tags](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-tags) -* [Product variations](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-variations) -* [Products](https://woocommerce.github.io/woocommerce-rest-api-docs/#products) \(Incremental\) -* [Refunds](https://woocommerce.github.io/woocommerce-rest-api-docs/#refunds) -* [Shipping methods](https://woocommerce.github.io/woocommerce-rest-api-docs/#shipping-methods) -* [Shipping zone locations](https://woocommerce.github.io/woocommerce-rest-api-docs/#shipping-zone-locations) -* [Shipping zone methods](https://woocommerce.github.io/woocommerce-rest-api-docs/#shipping-zone-methods) -* [Shipping zones](https://woocommerce.github.io/woocommerce-rest-api-docs/#shipping-zones) -* [System status tools](https://woocommerce.github.io/woocommerce-rest-api-docs/#system-status-tools) -* [Tax classes](https://woocommerce.github.io/woocommerce-rest-api-docs/#tax-classes) -* [Tax rates](https://woocommerce.github.io/woocommerce-rest-api-docs/#tax-rates) +- [Coupons](https://woocommerce.github.io/woocommerce-rest-api-docs/#coupons) \(Incremental\) +- [Customers](https://woocommerce.github.io/woocommerce-rest-api-docs/#customers) \(Incremental\) +- [orders](https://woocommerce.github.io/woocommerce-rest-api-docs/#orders) \(Incremental\) +- [Order notes](https://woocommerce.github.io/woocommerce-rest-api-docs/#order-notes) +- [Payment gateways](https://woocommerce.github.io/woocommerce-rest-api-docs/#payment-gateways) +- [Product attribute terms](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-attribute-terms) +- [Product attributes](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-attributes) +- [Product categories](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-categories) +- [Product reviews](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-reviews) \(Incremental\) +- [Product shipping classes](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-shipping-classes) +- [Product tags](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-tags) +- [Product variations](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-variations) +- [Products](https://woocommerce.github.io/woocommerce-rest-api-docs/#products) \(Incremental\) +- [Refunds](https://woocommerce.github.io/woocommerce-rest-api-docs/#refunds) +- [Shipping methods](https://woocommerce.github.io/woocommerce-rest-api-docs/#shipping-methods) +- [Shipping zone locations](https://woocommerce.github.io/woocommerce-rest-api-docs/#shipping-zone-locations) +- [Shipping zone methods](https://woocommerce.github.io/woocommerce-rest-api-docs/#shipping-zone-methods) +- [Shipping zones](https://woocommerce.github.io/woocommerce-rest-api-docs/#shipping-zones) +- [System status tools](https://woocommerce.github.io/woocommerce-rest-api-docs/#system-status-tools) +- [Tax classes](https://woocommerce.github.io/woocommerce-rest-api-docs/#tax-classes) +- [Tax rates](https://woocommerce.github.io/woocommerce-rest-api-docs/#tax-rates) ## Connector-specific features & highlights Useful links: -* [WooCommerce Rest API Docs](https://woocommerce.github.io/woocommerce-rest-api-docs/#introduction) +- [WooCommerce Rest API Docs](https://woocommerce.github.io/woocommerce-rest-api-docs/#introduction) ## Data type map | Integration Type | Airbyte Type | Notes | -|:-----------------|:-------------|:------| +| :--------------- | :----------- | :---- | | `string` | `string` | | | `integer` | `integer` | | | `number` | `number` | | @@ -98,7 +98,7 @@ Useful links: ## Changelog | Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------| +| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------- | | 0.2.0 | 2022-11-30 | [19903](https://github.com/airbytehq/airbyte/pull/19903) | Migrate to low-code; Certification to Beta | | 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | | 0.1.0 | 2021-09-09 | [5955](https://github.com/airbytehq/airbyte/pull/5955) | Initial Release. Source WooCommerce | diff --git a/docs/integrations/sources/xero.md b/docs/integrations/sources/xero.md index 1aa3a11ede065..7fd6946db95eb 100644 --- a/docs/integrations/sources/xero.md +++ b/docs/integrations/sources/xero.md @@ -4,7 +4,7 @@ This is a setup guide for the Xero source connector which ingests data from the ## Prerequisites -First of all you should create an application in [Xero development center](https://developer.xero.com/app/manage/). The only supported integration type is to use [Xero Custom Connections](https://developer.xero.com/documentation/guides/oauth2/custom-connections/developer) so you should choose it on creating your Xero App. +First of all you should create an application in [Xero development center](https://developer.xero.com/app/manage/). The only supported integration type is to use [Xero Custom Connections](https://developer.xero.com/documentation/guides/oauth2/custom-connections/developer) so you should choose it on creating your Xero App. After creating an application, on configuration screen, authorize user for your Xero Organisation. Also, issue new Client Secret and remember it - it will be required for setting up Xero connector in your Airbyte instance. ## Supported streams @@ -56,3 +56,7 @@ The source connector supports the following [sync modes](https://docs.airbyte.co - Incremental ## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------------- | +| 0.1.0 | 2021-11-11 | [18666](https://github.com/airbytehq/airbyte/pull/18666) | 🎉 New Source - Xero [python cdk] | diff --git a/docs/integrations/sources/xkcd.md b/docs/integrations/sources/xkcd.md index 9088fa705eb85..64457c703bcbd 100644 --- a/docs/integrations/sources/xkcd.md +++ b/docs/integrations/sources/xkcd.md @@ -1,8 +1,8 @@ -# XKCD +# XKCD This page guides you through the process of setting up the xkcd source connector. -## Prerequisites +## Prerequisites XKCD is an open API, so no credentials are needed to set up the surce. @@ -20,6 +20,7 @@ XKCD does not perform rate limiting. ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.1.0 | 2022-10-17 | [18049](https://github.com/airbytehq/airbyte/pull/18049) | Initial version/release of the connector. +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------- | +| 0.1.1 | 2022-10-24 | [18386](https://github.com/airbytehq/airbyte/pull/18386) | Readded xkcd to source def yaml | +| 0.1.0 | 2022-10-17 | [18049](https://github.com/airbytehq/airbyte/pull/18049) | Initial version/release of the connector. | diff --git a/docs/integrations/sources/yahoo-finance-price.md b/docs/integrations/sources/yahoo-finance-price.md new file mode 100644 index 0000000000000..8162eb58e3eac --- /dev/null +++ b/docs/integrations/sources/yahoo-finance-price.md @@ -0,0 +1,9 @@ +# Yahoo Finance Price + +The Airbyte Source for [Yahoo Finance Price](https://finance.yahoo.com/) + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------- | +| 0.1.3 | 2022-03-23 | [10563](https://github.com/airbytehq/airbyte/pull/10563) | 🎉 Source Yahoo Finance Price | diff --git a/docs/integrations/sources/yandex-metrica.md b/docs/integrations/sources/yandex-metrica.md index abc103528f9a4..025a2f5b38cb3 100644 --- a/docs/integrations/sources/yandex-metrica.md +++ b/docs/integrations/sources/yandex-metrica.md @@ -3,32 +3,44 @@ This page guides you through the process of setting up the Yandex Metrica source connector. ## Prerequisites -* Counter ID -* OAuth2 Token + +- Counter ID +- OAuth2 Token ## Setup Yandex Metrica + 1. [Create a Yandex Metrica account](https://metrica.yandex.com/) if you don't already have one. 2. Head to [Management page](https://metrica.yandex.com/list) and add new tag or choose an existing one. 3. At the top of the dasboard you will see 8 digit number to the right of your website name. This is your **Counter ID**. 4. Create a new app or choose an existing one from [My apps page](https://oauth.yandex.com/). - * Which platform is the app required for?: **Web services** - * Callback URL: https://oauth.yandex.com/verification_code - * What data do you need?: **Yandex.Metrica**. Read permission will suffice. -5. Choose your app from [the list](https://oauth.yandex.com/). - * To create your API key you will need to grab your **ClientID**, - * Now to get the API key craft a GET request to an endpoint *https://oauth.yandex.com/authorizE?response_type=token&client_id=\* - * You will receive a response with your **API key**. Save it. + - Which platform is the app required for?: **Web services** + - Callback URL: https://oauth.yandex.com/verification_code + - What data do you need?: **Yandex.Metrica**. Read permission will suffice. +5. Choose your app from [the list](https://oauth.yandex.com/). + - To create your API key you will need to grab your **ClientID**, + - Now to get the API key craft a GET request to an endpoint *https://oauth.yandex.com/authorizE?response_type=token&client_id=\* + - You will receive a response with your **API key**. Save it. ## Supported sync modes + The Yandex Metrica source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): - - Full Refresh - - Incremental - * After the first sync the connector will set the state for next sync. The **start date** will be set to last syncs **end date**. The **end date** will be set to 1 day before today. + +- Full Refresh +- Incremental + - After the first sync the connector will set the state for next sync. The **start date** will be set to last syncs **end date**. The **end date** will be set to 1 day before today. ## Supported Streams -* [Views](https://yandex.com/dev/metrika/doc/api2/logs/fields/hits.html) (Incremental). -* [Sessions](https://yandex.com/dev/metrika/doc/api2/logs/fields/visits.html) (Incremental). + +- [Views](https://yandex.com/dev/metrika/doc/api2/logs/fields/hits.html) (Incremental). +- [Sessions](https://yandex.com/dev/metrika/doc/api2/logs/fields/visits.html) (Incremental). ## Notes + - We recommend syncing data once a day. Because of the Yandex Metrica API limitation it is only possible to extract records up to yesterdays date. Todays records will only be available tomorrow. -- Because of the way API works some syncs may take a long time to finish. Timeout period is 2 hours. \ No newline at end of file +- Because of the way API works some syncs may take a long time to finish. Timeout period is 2 hours. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------- | +| 0.1.0 | 2022-09-09 | [15061](https://github.com/airbytehq/airbyte/pull/15061) | 🎉 New Source: Yandex metrica | diff --git a/docs/integrations/sources/younium.md b/docs/integrations/sources/younium.md index 7aa192d09dffb..60d510a0e700c 100644 --- a/docs/integrations/sources/younium.md +++ b/docs/integrations/sources/younium.md @@ -7,30 +7,38 @@ This page contains the setup guide and reference information for the Younium sou This Younium source uses the [Younium API](https://developer.younium.com/). ## Setup guide + ### Step 1: Set up Younium #### For Airbyte OSS: + 1. Navigate to the Airbyte Open Source dashboard -2. Enter a name for your source +2. Enter a name for your source 3. Enter your Younium `username` 4. Enter your Younium `password` -5. Enter your Younium `legal_entity`. You can find the legal entity name in your account setting if you log in to the [Younium web platform](https://app.younium.com/) -7. Click **Set up source** +5. Enter your Younium `legal_entity`. You can find the legal entity name in your account setting if you log in to the [Younium web platform](https://app.younium.com/) +6. Click **Set up source** ## Supported sync modes The Younium source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): -| Feature | Supported? | -| :--- | :-- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | No | -| Replicate Incremental Deletes | No | -| SSL connection | Yes | -| Namespaces | No | +| Feature | Supported? | +| :---------------------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | No | +| Replicate Incremental Deletes | No | +| SSL connection | Yes | +| Namespaces | No | ## Supported Streams -* [Subscriptions](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Subscriptions) -* [Products](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Products) -* [Invoices](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Invoices) \ No newline at end of file +- [Subscriptions](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Subscriptions) +- [Products](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Products) +- [Invoices](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Invoices) + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------------- | +| 0.1.0 | 2022-11-09 | [18758](https://github.com/airbytehq/airbyte/pull/18758) | 🎉 New Source: Younium [python cdk] | diff --git a/docs/integrations/sources/youtube-analytics-business.md b/docs/integrations/sources/youtube-analytics-business.md new file mode 100644 index 0000000000000..45bcbe4974418 --- /dev/null +++ b/docs/integrations/sources/youtube-analytics-business.md @@ -0,0 +1,9 @@ +# YouTube Analytics Business + +The Airbyte Source for YouTube Analytics Business + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------- | +| 0.1.0 | 2022-11-09 | [19223](https://github.com/airbytehq/airbyte/pull/19223) | 🎉 New Source: Demo - Youtube Analytics Business | diff --git a/docs/understanding-airbyte/connections/incremental-append.md b/docs/understanding-airbyte/connections/incremental-append.md index d7f84d5c6bcb0..a779d0b1d13ce 100644 --- a/docs/understanding-airbyte/connections/incremental-append.md +++ b/docs/understanding-airbyte/connections/incremental-append.md @@ -22,47 +22,47 @@ As mentioned above, the delta from a sync will be _appended_ to the existing dat Assume that `updated_at` is our `cursor_field`. Let's say the following data already exists into our data warehouse. -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | false | 1754 | -| Marie Antoinette | false | 1755 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | false | 1754 | +| Marie Antoinette | false | 1755 | In the next sync, the delta contains the following record: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVII | false | 1785 | +| name | deceased | updated_at | +| :--------- | :------- | :--------- | +| Louis XVII | false | 1785 | At the end of this incremental sync, the data warehouse would now contain: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | false | 1754 | -| Marie Antoinette | false | 1755 | -| Louis XVII | false | 1785 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | false | 1754 | +| Marie Antoinette | false | 1755 | +| Louis XVII | false | 1785 | ### Updating a Record Let's assume that our warehouse contains all the data that it did at the end of the previous section. Now, unfortunately the king and queen lose their heads. Let's see that delta: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | true | 1793 | -| Marie Antoinette | true | 1793 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | true | 1793 | +| Marie Antoinette | true | 1793 | The output we expect to see in the warehouse is as follows: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | false | 1754 | -| Marie Antoinette | false | 1755 | -| Louis XVII | false | 1785 | -| Louis XVI | true | 1793 | -| Marie Antoinette | true | 1793 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | false | 1754 | +| Marie Antoinette | false | 1755 | +| Louis XVII | false | 1785 | +| Louis XVI | true | 1793 | +| Marie Antoinette | true | 1793 | ## Source-Defined Cursor -Some sources are able to determine the cursor that they use without any user input. For example, in the [exchange rates source](../../integrations/sources/exchangeratesapi.md), the source knows that the date field should be used to determine the last record that was synced. In these cases, simply select the incremental option in the UI. +Some sources are able to determine the cursor that they use without any user input. For example, in the [exchange rates source](../../integrations/sources/exchange-rates.md), the source knows that the date field should be used to determine the last record that was synced. In these cases, simply select the incremental option in the UI. ![](../../.gitbook/assets/incremental_source_defined.png) @@ -102,23 +102,23 @@ SELECT * FROM table WHERE cursor_field >= 'last_sync_max_cursor_field_value' Let's say the following data already exists into our data warehouse. -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | false | 1754 | -| Marie Antoinette | false | 1755 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | false | 1754 | +| Marie Antoinette | false | 1755 | At the start of the next sync, the source data contains the following new record: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | true | 1754 | +| name | deceased | updated_at | +| :-------- | :------- | :--------- | +| Louis XVI | true | 1754 | At the end of the second incremental sync, the data warehouse would still contain data from the first sync because the delta record did not provide a valid value for the cursor field \(the cursor field is not greater than last sync's max value, `1754 < 1755`\), so it is not emitted by the source as a new or modified record. -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | false | 1754 | -| Marie Antoinette | false | 1755 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | false | 1754 | +| Marie Antoinette | false | 1755 | Similarly, if multiple modifications are made during the same day to the same records. If the frequency of the sync is not granular enough \(for example, set for every 24h\), then intermediate modifications to the data are not going to be detected and emitted. Only the state of data at the time the sync runs will be reflected in the destination. diff --git a/docs/understanding-airbyte/connections/incremental-deduped-history.md b/docs/understanding-airbyte/connections/incremental-deduped-history.md index f0b954e7340d5..be02105b8bfef 100644 --- a/docs/understanding-airbyte/connections/incremental-deduped-history.md +++ b/docs/understanding-airbyte/connections/incremental-deduped-history.md @@ -35,57 +35,57 @@ As mentioned above, the delta from a sync will be _appended_ to the existing his Assume that `updated_at` is our `cursor_field` and `name` is the `primary_key`. Let's say the following data already exists into our data warehouse. -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | false | 1754 | -| Marie Antoinette | false | 1755 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | false | 1754 | +| Marie Antoinette | false | 1755 | In the next sync, the delta contains the following record: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVII | false | 1785 | +| name | deceased | updated_at | +| :--------- | :------- | :--------- | +| Louis XVII | false | 1785 | At the end of this incremental sync, the data warehouse would now contain: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | false | 1754 | -| Marie Antoinette | false | 1755 | -| Louis XVII | false | 1785 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | false | 1754 | +| Marie Antoinette | false | 1755 | +| Louis XVII | false | 1785 | ### Updating a Record Let's assume that our warehouse contains all the data that it did at the end of the previous section. Now, unfortunately the king and queen lose their heads. Let's see that delta: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | true | 1793 | -| Marie Antoinette | true | 1793 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | true | 1793 | +| Marie Antoinette | true | 1793 | The output we expect to see in the warehouse is as follows: In the history table: -| name | deceased | updated\_at | start\_at | end\_at | -| :--- | :--- | :--- | :--- | :--- | -| Louis XVI | false | 1754 | 1754 | 1793 | -| Louis XVI | true | 1793 | 1793 | NULL | -| Louis XVII | false | 1785 | 1785 | NULL | -| Marie Antoinette | false | 1755 | 1755 | 1793 | -| Marie Antoinette | true | 1793 | 1793 | NULL | +| name | deceased | updated_at | start_at | end_at | +| :--------------- | :------- | :--------- | :------- | :----- | +| Louis XVI | false | 1754 | 1754 | 1793 | +| Louis XVI | true | 1793 | 1793 | NULL | +| Louis XVII | false | 1785 | 1785 | NULL | +| Marie Antoinette | false | 1755 | 1755 | 1793 | +| Marie Antoinette | true | 1793 | 1793 | NULL | In the final de-duplicated table: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | true | 1793 | -| Louis XVII | false | 1785 | -| Marie Antoinette | true | 1793 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | true | 1793 | +| Louis XVII | false | 1785 | +| Marie Antoinette | true | 1793 | ## Source-Defined Cursor -Some sources are able to determine the cursor that they use without any user input. For example, in the [exchange rates source](../../integrations/sources/exchangeratesapi.md), the source knows that the date field should be used to determine the last record that was synced. In these cases, simply select the incremental option in the UI. +Some sources are able to determine the cursor that they use without any user input. For example, in the [exchange rates source](../../integrations/sources/exchange-rates.md), the source knows that the date field should be used to determine the last record that was synced. In these cases, simply select the incremental option in the UI. ![](../../.gitbook/assets/incremental_source_defined.png) @@ -127,23 +127,23 @@ select * from table where cursor_field > 'last_sync_max_cursor_field_value' Let's say the following data already exists into our data warehouse. -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | false | 1754 | -| Marie Antoinette | false | 1755 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | false | 1754 | +| Marie Antoinette | false | 1755 | At the start of the next sync, the source data contains the following new record: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | true | 1754 | +| name | deceased | updated_at | +| :-------- | :------- | :--------- | +| Louis XVI | true | 1754 | At the end of the second incremental sync, the data warehouse would still contain data from the first sync because the delta record did not provide a valid value for the cursor field \(the cursor field is not greater than last sync's max value, `1754 < 1755`\), so it is not emitted by the source as a new or modified record. -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | false | 1754 | -| Marie Antoinette | false | 1755 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | false | 1754 | +| Marie Antoinette | false | 1755 | Similarly, if multiple modifications are made during the same day to the same records. If the frequency of the sync is not granular enough \(for example, set for every 24h\), then intermediate modifications to the data are not going to be detected and emitted. Only the state of data at the time the sync runs will be reflected in the destination. diff --git a/pyproject.toml b/pyproject.toml index 17544d0afc311..e584fd9ba9c9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ extend-exclude = [ "models", ".eggs", "airbyte-cdk/python/airbyte_cdk/models/__init__.py", + "airbyte-cdk/python/airbyte_cdk/sources/declarative/models/__init__.py", ".tox", "airbyte_api_client", "**/generated/*", diff --git a/tools/bin/ci_check_dependency.py b/tools/bin/ci_check_dependency.py index 23a55077aab48..a57ae036d382e 100644 --- a/tools/bin/ci_check_dependency.py +++ b/tools/bin/ci_check_dependency.py @@ -17,9 +17,20 @@ "/integration_tests/", "/unit_tests/", # Common "acceptance-test-config.yml", "acceptance-test-docker.sh", ".md", ".dockerignore", ".gitignore", "requirements.txt"] +IGNORED_SOURCES = [ + re.compile("^source-e2e-test-cloud$"), + re.compile("^source-mongodb$"), + re.compile("^source-python-http-tutorial$"), + re.compile("^source-relational-db$"), + re.compile("^source-stock-ticker-api-tutorial$"), + re.compile("source-jdbc$"), + re.compile("^source-scaffold-.*$"), + re.compile(".*-secure$"), +] IGNORED_DESTINATIONS = [ re.compile(".*-strict-encrypt$"), re.compile("^destination-dev-null$"), + re.compile("^destination-scaffold-destination-python$"), re.compile("^destination-jdbc$") ] COMMENT_TEMPLATE_PATH = ".github/comment_templates/connector_dependency_template.md" @@ -130,6 +141,8 @@ def get_connector_changelog_status(connector: str, version) -> str: type, name = connector.replace("-strict-encrypt", "").replace("-denormalized", "").split("-", 1) doc_path = f"{DOC_PATH}{type}s/{name}.md" + if any(regex.match(connector) for regex in IGNORED_SOURCES): + return "🔵
(ignored)" if any(regex.match(connector) for regex in IGNORED_DESTINATIONS): return "🔵
(ignored)" if not os.path.exists(doc_path): @@ -160,7 +173,9 @@ def as_markdown_table_rows(connectors: List[str], definitions) -> str: version_status = get_connector_version_status(connector, version) changelog_status = get_connector_changelog_status(connector, version) definition = next((x for x in definitions if x["dockerRepository"].endswith(connector)), None) - if any(regex.match(connector) for regex in IGNORED_DESTINATIONS): + if any(regex.match(connector) for regex in IGNORED_SOURCES): + publish_status = "🔵
(ignored)" + elif any(regex.match(connector) for regex in IGNORED_DESTINATIONS): publish_status = "🔵
(ignored)" elif definition is None: publish_status = "⚠
(not in seed)"