From 1a6a1c67ed5706235b16c4a39b0093b208831b19 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 20 Dec 2022 17:05:02 +0200 Subject: [PATCH 1/7] Bump helm chart version reference to 0.43.6 (#20534) Co-authored-by: marcosmarxm --- charts/airbyte-bootloader/Chart.yaml | 2 +- .../Chart.yaml | 2 +- charts/airbyte-metrics/Chart.yaml | 2 +- charts/airbyte-pod-sweeper/Chart.yaml | 2 +- charts/airbyte-server/Chart.yaml | 2 +- charts/airbyte-temporal/Chart.yaml | 2 +- charts/airbyte-webapp/Chart.yaml | 2 +- charts/airbyte-worker/Chart.yaml | 2 +- charts/airbyte/Chart.lock | 22 +++++++++---------- charts/airbyte/Chart.yaml | 20 ++++++++--------- 10 files changed, 29 insertions(+), 29 deletions(-) diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index 0cb5b588a050..3b8434955777 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.43.3" +version: "0.43.6" # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml index c64f4eca2910..7c4adab12c49 100644 --- a/charts/airbyte-connector-builder-server/Chart.yaml +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.43.3" +version: "0.43.6" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index 5ccf6b44c7a6..6271c65e831c 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.43.3" +version: "0.43.6" # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index c7a04c748892..860e2d61ae46 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.43.3" +version: "0.43.6" # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index 68b3190f789f..1f527d8ebd90 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.43.3" +version: "0.43.6" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index 55ef02d98341..9d4386e765e8 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.43.3" +version: "0.43.6" # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index c28bb59ddd4e..73d1e47a107c 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.43.3" +version: "0.43.6" # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index 7d984701a838..b02c76aff509 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.43.3" +version: "0.43.6" # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock index 2805c6493767..e6dcb88b3cbd 100644 --- a/charts/airbyte/Chart.lock +++ b/charts/airbyte/Chart.lock @@ -4,30 +4,30 @@ dependencies: version: 1.17.1 - name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.43.3 + version: 0.43.6 - name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.43.3 + version: 0.43.6 - name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.43.3 + version: 0.43.6 - name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.43.3 + version: 0.43.6 - name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.43.3 + version: 0.43.6 - name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.43.3 + version: 0.43.6 - name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.43.3 + version: 0.43.6 - name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.43.3 + version: 0.43.6 - name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.43.3 -digest: sha256:04b9338c23554bf1fcf64b4e296ba262bd24be24253cad9b7c8567b404f67ddb -generated: "2022-12-13T00:15:39.706036573Z" + version: 0.43.6 +digest: sha256:4c97218bd5a68ef3b9d1bc28b047d6d7fa02d30a7c42596d11b0d45f028c1eaa +generated: "2022-12-15T19:17:27.989410129Z" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index aa60439c5204..37472c054b56 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.43.3 +version: 0.43.6 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -32,36 +32,36 @@ dependencies: - condition: airbyte-bootloader.enabled name: airbyte-bootloader repository: "https://airbytehq.github.io/helm-charts/" - version: 0.43.3 + version: 0.43.6 - condition: temporal.enabled name: temporal repository: "https://airbytehq.github.io/helm-charts/" - version: 0.43.3 + version: 0.43.6 - condition: webapp.enabled name: webapp repository: "https://airbytehq.github.io/helm-charts/" - version: 0.43.3 + version: 0.43.6 - condition: server.enabled name: server repository: "https://airbytehq.github.io/helm-charts/" - version: 0.43.3 + version: 0.43.6 - condition: worker.enabled name: worker repository: "https://airbytehq.github.io/helm-charts/" - version: 0.43.3 + version: 0.43.6 - condition: pod-sweeper.enabled name: pod-sweeper repository: "https://airbytehq.github.io/helm-charts/" - version: 0.43.3 + version: 0.43.6 - condition: metrics.enabled name: metrics repository: "https://airbytehq.github.io/helm-charts/" - version: 0.43.3 + version: 0.43.6 - condition: cron.enabled name: cron repository: "https://airbytehq.github.io/helm-charts/" - version: 0.43.3 + version: 0.43.6 - condition: connector-builder-server.enabled name: connector-builder-server repository: "https://airbytehq.github.io/helm-charts/" - version: 0.43.3 + version: 0.43.6 From b17df69dc56bf8751d0bfcfdfe94b1c7c257f850 Mon Sep 17 00:00:00 2001 From: Brian Lai <51336873+brianjlai@users.noreply.github.com> Date: Tue, 20 Dec 2022 16:16:14 -0500 Subject: [PATCH 2/7] fix a bug where the OAuthAuthenticator was misnamed (#20700) * fix a bug where the OAuthAuthenticator was misnamed * roll airbyte-cdk version * add some missing component types to the existing factory type registry * fix test --- airbyte-cdk/python/CHANGELOG.md | 3 +++ .../declarative/declarative_component_schema.yaml | 8 ++++---- .../sources/declarative/parsers/class_types_registry.py | 9 ++++++++- .../parsers/manifest_component_transformer.py | 2 +- airbyte-cdk/python/setup.py | 2 +- .../parsers/test_manifest_component_transformer.py | 2 +- 6 files changed, 18 insertions(+), 8 deletions(-) diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 727ab95d0e2a..739ca1d8c5f6 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.16.2 +Fix the naming of OAuthAuthenticator + ## 0.16.1 Include declarative_component_schema.yaml in the publish to PyPi diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 40a9e9e6bc27..1eef180598d0 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -363,8 +363,8 @@ definitions: $options: type: object additionalProperties: true - DeclarativeOauth2Authenticator: - description: Authenticator for requests using Oauth authentication + OAuthAuthenticator: + description: Authenticator for requests using OAuth 2.0 authentication type: object required: - type @@ -375,7 +375,7 @@ definitions: properties: type: type: string - enum: [DeclarativeOauth2Authenticator] + enum: [OAuthAuthenticator] client_id: type: string client_secret: @@ -580,7 +580,7 @@ definitions: - "$ref": "#/definitions/BasicHttpAuthenticator" - "$ref": "#/definitions/BearerAuthenticator" - "$ref": "#/definitions/CustomAuthenticator" - - "$ref": "#/definitions/DeclarativeOauth2Authenticator" + - "$ref": "#/definitions/OAuthAuthenticator" - "$ref": "#/definitions/NoAuth" - "$ref": "#/definitions/SessionTokenAuthenticator" error_handler: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py index a6ea7c2afac9..8db72d071810 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py @@ -19,6 +19,8 @@ from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.requesters import RequestOption +from airbyte_cdk.sources.declarative.requesters.error_handlers import HttpResponseFilter from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.constant_backoff_strategy import ConstantBackoffStrategy from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.exponential_backoff_strategy import ( ExponentialBackoffStrategy, @@ -37,6 +39,7 @@ from airbyte_cdk.sources.declarative.requesters.paginators.strategies.cursor_pagination_strategy import CursorPaginationStrategy from airbyte_cdk.sources.declarative.requesters.paginators.strategies.offset_increment import OffsetIncrement from airbyte_cdk.sources.declarative.requesters.paginators.strategies.page_increment import PageIncrement +from airbyte_cdk.sources.declarative.requesters.request_options import InterpolatedRequestOptionsProvider from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever from airbyte_cdk.sources.declarative.schema.inline_schema_loader import InlineSchemaLoader from airbyte_cdk.sources.declarative.schema.json_file_schema_loader import JsonFileSchemaLoader @@ -45,7 +48,7 @@ from airbyte_cdk.sources.declarative.stream_slicers.datetime_stream_slicer import DatetimeStreamSlicer from airbyte_cdk.sources.declarative.stream_slicers.list_stream_slicer import ListStreamSlicer from airbyte_cdk.sources.declarative.stream_slicers.single_slice import SingleSlice -from airbyte_cdk.sources.declarative.stream_slicers.substream_slicer import SubstreamSlicer +from airbyte_cdk.sources.declarative.stream_slicers.substream_slicer import ParentStreamConfig, SubstreamSlicer from airbyte_cdk.sources.declarative.transformations import RemoveFields from airbyte_cdk.sources.declarative.transformations.add_fields import AddFields @@ -69,8 +72,10 @@ "DpathExtractor": DpathExtractor, "ExponentialBackoffStrategy": ExponentialBackoffStrategy, "HttpRequester": HttpRequester, + "HttpResponseFilter": HttpResponseFilter, "InlineSchemaLoader": InlineSchemaLoader, "InterpolatedBoolean": InterpolatedBoolean, + "InterpolatedRequestOptionsProvider": InterpolatedRequestOptionsProvider, "InterpolatedString": InterpolatedString, "JsonSchema": JsonFileSchemaLoader, # todo remove after hacktoberfest and update connectors to use JsonFileSchemaLoader "JsonFileSchemaLoader": JsonFileSchemaLoader, @@ -81,7 +86,9 @@ "OAuthAuthenticator": DeclarativeOauth2Authenticator, "OffsetIncrement": OffsetIncrement, "PageIncrement": PageIncrement, + "ParentStreamConfig": ParentStreamConfig, "RecordSelector": RecordSelector, + "RequestOption": RequestOption, "RemoveFields": RemoveFields, "SimpleRetriever": SimpleRetriever, "SingleSlice": SingleSlice, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py index 42ed89899972..8be72eb77828 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py @@ -26,7 +26,7 @@ "DeclarativeSource.streams": "DeclarativeStream", # DeclarativeStream "DeclarativeStream.retriever": "SimpleRetriever", - "DeclarativeStream.schema_loader": "DefaultSchemaLoader", + "DeclarativeStream.schema_loader": "JsonFileSchemaLoader", # DefaultErrorHandler "DefaultErrorHandler.response_filters": "HttpResponseFilter", # DefaultPaginator diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 572207ff4437..d3ac09c8c966 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.16.1", + version="0.16.2", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py index f724e98013a6..f77e30b68478 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py @@ -17,7 +17,7 @@ { "type": "DeclarativeStream", "retriever": {"type": "SimpleRetriever"}, - "schema_loader": {"type": "DefaultSchemaLoader"}, + "schema_loader": {"type": "JsonFileSchemaLoader"}, } ], }, From 1fdaa1d0748e2b415e90f0567322edd1e4f4751e Mon Sep 17 00:00:00 2001 From: Maxime Carbonneau-Leclerc Date: Tue, 20 Dec 2022 17:53:35 -0500 Subject: [PATCH 3/7] [ISSUE #20044] generate pydantic models from handwritten schema (#20475) * handwritten low code manifest example components * add MinMaxDatetime to jsonschema * add a basic gradle command to generate manifest components * Add auth components to handwritten component schema - ApiKeyAuthenticator - BasicHttpAuthenticator - BearerAuthenticator - DeclarativeOauth2Authenticator - NoAuth * Respect optional properties in DeclarativeOauth2Authenticator * Fix `Dict[str, Any]` mapping in auth components * add default error handler composite error handler and http response filter components * [low code component schema] adding backoff strategies to schema * [low code component schema] fix float types * [low code component schema] add RecordFilter * Remove `config` from auth components * [low code component schema] add Interpolation (with pending question on 'type' not being defined) * Add CartesianProductStreamSlicer & DatetimeStreamSlicer * Add ListStreamSlicer, and fix nesting of DatetimeStreamSlicer * [low code component schema] add InterpolatedRequestOptionsProvider * Add slicer components, and fix a couple of components after reviewing output * [low code component schema] adding transformations and adding type to interpolators * adding spec and a few small tweaks * Add DefaultSchemaLoader * [low code component schema] attempt on custom class * Add descriptions for auth components * add RequestOption * remove interpolated objects from the schema in favor of strings only * a few schema fixes and adding some custom pagination and stream slicer * [low code component schema] fix CustomBackoffStrategy * Add CustomRecordExtractor * add some description and add additional properties * insert a transformer to hydrate default manifest components and perform validation against the handwritten schema * [low code component schema] validating existing schemas * [low code component schema] clean validation script * add manifest transformer tests and a few tweaks to the schema * Revert "[low code component schema] clean validation script" This reverts commit 2408f41cf6e84e9bf9111da832e08a96292fb5b1. * Revert "[low code component schema] validating existing schemas" This reverts commit 9d3997781562faecd4046372816afe35c3b31332. * [low code component schema] integrate validation script to gradle * [low code component schema] updating validation script permissions * remove a few model gen spike files and clean up comments * default types should take parent type into account and a few schema changes * [ISSUE #20044] generate pydantic models from handwritten schema * [ISSUE #20044] code review * [ISSUE #20044] re-generating declarative component schema files Co-authored-by: brianjlai Co-authored-by: Catherine Noll --- .pre-commit-config.yaml | 3 +- airbyte-cdk/python/README.md | 7 + .../low_code_component_schema.yaml | 918 ++++++++++++++++++ .../sources/declarative/models/__init__.py | 3 + .../models/declarative_component_schema.py | 490 ++++++++++ .../bin/generate-component-manifest-files.sh | 27 + airbyte-cdk/python/build.gradle | 11 + build.gradle | 6 + pyproject.toml | 1 + 9 files changed, 1465 insertions(+), 1 deletion(-) create mode 100644 airbyte-cdk/python/airbyte_cdk/sources/declarative/low_code_component_schema.yaml create mode 100644 airbyte-cdk/python/airbyte_cdk/sources/declarative/models/__init__.py create mode 100644 airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py create mode 100755 airbyte-cdk/python/bin/generate-component-manifest-files.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f82f2b5ba91f..c1577406f64b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,8 @@ repos: rev: v0.8.8 hooks: - id: licenseheaders - args: ["--tmpl=LICENSE_SHORT", "--ext=py", "-f"] + args: + ["--tmpl=LICENSE_SHORT", "--ext=py", "-x=**/models/__init__.py", "-f"] - repo: https://github.com/psf/black rev: 22.3.0 hooks: diff --git a/airbyte-cdk/python/README.md b/airbyte-cdk/python/README.md index d318c7cb81f8..a660ef8dba85 100644 --- a/airbyte-cdk/python/README.md +++ b/airbyte-cdk/python/README.md @@ -66,6 +66,13 @@ pip install -e ".[dev]" # [dev] installs development-only dependencies * Perform static type checks using `mypy airbyte_cdk`. `MyPy` configuration is in `.mypy.ini`. * The `type_check_and_test.sh` script bundles both type checking and testing in one convenient command. Feel free to use it! +##### Autogenerated files +If the iteration you are working on includes changes to the models, you might want to regenerate them. In order to do that, you can run: +```commandline +SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace +``` +This will generate the files based on the schemas, add the license information and format the code. If you want to only do the former and rely on pre-commit to the others, you can run the appropriate generation command i.e. `./gradlew generateProtocolClassFiles` or `./gradlew generateComponentManifestClassFiles`. + #### Testing All tests are located in the `unit_tests` directory. Run `pytest --cov=airbyte_cdk unit_tests/` to run them. This also presents a test coverage report. diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/low_code_component_schema.yaml b/airbyte-cdk/python/airbyte_cdk/sources/declarative/low_code_component_schema.yaml new file mode 100644 index 000000000000..02fdc9d3b24b --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/low_code_component_schema.yaml @@ -0,0 +1,918 @@ +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/airbyte_cdk/sources/declarative/handwritten_manifest.yaml +title: LowCodeComponentManifest +type: object +description: Low Code connector components +version: 1.0.0 +required: + - check + - streams + - version +properties: + check: + "$ref": "#/definitions/CheckStream" + streams: + type: array + items: + "$ref": "#/definitions/DeclarativeStream" + version: + type: string + spec: + "$ref": "#/definitions/Spec" +definitions: + AddedFieldDefinition: + description: Defines the field to add on a record + type: object + required: + - type + - path + - value + properties: + type: + enum: [AddedFieldDefinition] + path: + type: array + items: + type: string + value: + type: string + AddFields: + description: Transformation which adds field to an output record. The path of the added field can be nested. + type: object + required: + - type + - fields + properties: + type: + enum: [AddFields] + fields: + type: array + items: + - "$ref": "#/definitions/AddedFieldDefinition" + $options: + type: object + additionalProperties: true + ApiKeyAuthenticator: + description: Authenticator for requests requiring an api token + type: object + required: + - type + - api_token + properties: + type: + enum: [ApiKeyAuthenticator] + api_token: + type: string + header: + type: string + BasicHttpAuthenticator: + description: Authenticator for requests authenticated with a username and optional password + type: object + required: + - type + - username + properties: + type: + enum: [BasicHttpAuthenticator] + username: + type: string + password: + type: string + BearerAuthenticator: + description: Authenticator for requests authenticated with a Bearer token + type: object + required: + - type + - api_token + properties: + type: + enum: [BearerAuthenticator] + api_token: + type: string + CartesianProductStreamSlicer: + description: Stream slicer that iterates over the cartesian product of input stream slicers + type: object + required: + - type + - stream_slicers + properties: + type: + enum: [CartesianProductStreamSlicer] + stream_slicers: + type: array + items: + anyOf: + - "$ref": "#/definitions/CustomStreamSlicer" + - "$ref": "#/definitions/DatetimeStreamSlicer" + - "$ref": "#/definitions/ListStreamSlicer" + - "$ref": "#/definitions/SingleSlice" + - "$ref": "#/definitions/SubstreamSlicer" + $options: + type: object + additionalProperties: true + CheckStream: + description: Checks the connections by trying to read records from one or many of the streams selected by the developer + type: object + required: + - type + - stream_names + properties: + type: + enum: [CheckStream] + stream_names: + type: array + items: + type: string + CompositeErrorHandler: + description: Error handler that sequentially iterates over a list of error handlers + type: object + required: + - error_handlers + properties: + type: + enum: [CompositeErrorHandler] + error_handlers: + type: array + items: + anyOf: + - "$ref": "#/definitions/CompositeErrorHandler" + - "$ref": "#/definitions/DefaultErrorHandler" + $options: + type: object + additionalProperties: true + ConstantBackoffStrategy: + description: Backoff strategy with a constant backoff interval + type: object + required: + - type + - backoff_time_in_seconds + properties: + type: + enum: [ConstantBackoffStrategy] + backoff_time_in_seconds: + anyOf: + - type: number + - type: string + $options: + type: object + additionalProperties: true + CursorPagination: + description: Pagination strategy that evaluates an interpolated string to define the next page token + type: object + required: + - type + - cursor_value + properties: + type: + enum: [CursorPagination] + cursor_value: + type: string + page_size: + type: integer + stop_condition: + type: string + decoder: + "$ref": "#/definitions/JsonDecoder" + $options: + type: object + additionalProperties: true + CustomAuthenticator: + description: Authenticator component whose behavior is derived from a custom code implementation of the connector + type: object + additionalProperties: true + required: + - type + - class_name + properties: + type: + enum: [CustomAuthenticator] + class_name: + type: string + additionalProperties: true + $options: + type: object + additionalProperties: true + CustomBackoffStrategy: + description: Backoff strategy component whose behavior is derived from a custom code implementation of the connector + type: object + additionalProperties: true + required: + - type + - class_name + properties: + type: + enum: [CustomBackoffStrategy] + class_name: + type: string + $options: + type: object + additionalProperties: true + CustomErrorHandler: + description: Error handler component whose behavior is derived from a custom code implementation of the connector + type: object + additionalProperties: true + required: + - type + - class_name + properties: + type: + enum: [CustomErrorHandler] + class_name: + type: string + $options: + type: object + additionalProperties: true + CustomPaginationStrategy: + description: Pagination strategy component whose behavior is derived from a custom code implementation of the connector + type: object + additionalProperties: true + required: + - type + - class_name + properties: + type: + enum: [CustomPaginationStrategy] + class_name: + type: string + $options: + type: object + additionalProperties: true + CustomRecordExtractor: + description: Record extractor component whose behavior is derived from a custom code implementation of the connector + type: object + additionalProperties: true + required: + - type + - class_name + properties: + type: + enum: [CustomRecordExtractor] + class_name: + type: string + $options: + type: object + additionalProperties: true + CustomStreamSlicer: + description: Stream slicer component whose behavior is derived from a custom code implementation of the connector + type: object + additionalProperties: true + required: + - type + - class_name + properties: + type: + enum: [CustomStreamSlicer] + class_name: + type: string + $options: + type: object + additionalProperties: true + DatetimeStreamSlicer: + description: Stream slicer that slices the stream over a datetime range + type: object + required: + - type + - cursor_field + - end_datetime + - datetime_format + - start_datetime + - step + properties: + type: + enum: [DatetimeStreamSlicer] + cursor_field: + type: string + datetime_format: + type: string + end_datetime: + anyOf: + - type: string + - "$ref": "#/definitions/MinMaxDatetime" + start_datetime: + anyOf: + - type: string + - "$ref": "#/definitions/MinMaxDatetime" + step: + type: string + end_time_option: + "$ref": "#/definitions/RequestOption" + lookback_window: + type: string + start_time_option: + "$ref": "#/definitions/RequestOption" + stream_state_field_end: + type: string + stream_state_field_start: + type: string + $options: + type: object + additionalProperties: true + DeclarativeOauth2Authenticator: + description: Authenticator for requests using Oauth authentication + type: object + required: + - type + - client_id + - client_secret + - refresh_token + - token_refresh_endpoint + properties: + type: + enum: [DeclarativeOauth2Authenticator] + client_id: + type: string + client_secret: + type: string + refresh_token: + type: string + token_refresh_endpoint: + type: string + access_token_name: + type: string + expires_in_name: + type: string + grant_type: + type: string + refresh_request_body: + type: object + additionalProperties: true + scopes: + type: array + items: + type: string + token_expiry_date: + type: string + DeclarativeStream: + description: A stream whose behavior is described by a set of declarative low code components + type: object + additionalProperties: true + required: + - type + - retriever + properties: + type: + enum: [DeclarativeStream] + retriever: + "$ref": "#/definitions/SimpleRetriever" + checkpoint_interval: + type: integer + name: + type: string + default: "" + primary_key: + anyOf: + - type: string + - type: array + items: + type: string + - type: array + items: + type: array + items: + type: string + default: "" + schema_loader: + anyOf: + - "$ref": "#/definitions/DefaultSchemaLoader" + - "$ref": "#/definitions/JsonFileSchemaLoader" + stream_cursor_field: + anyOf: + - type: string + - type: array + items: + - type: string + transformations: + type: array + items: + anyOf: + - "$ref": "#/definitions/AddFields" + - "$ref": "#/definitions/RemoveFields" + $options: + type: object + additional_properties: true + DefaultErrorHandler: + description: The default error handler. Default behavior includes only retrying server errors (HTTP 5XX) and too many requests (HTTP 429) with an exponential backoff + type: object + properties: + type: + enum: [DefaultErrorHandler] + backoff_strategies: + type: array + items: + anyOf: + - "$ref": "#/definitions/ConstantBackoffStrategy" + - "$ref": "#/definitions/CustomBackoffStrategy" + - "$ref": "#/definitions/ExponentialBackoffStrategy" + - "$ref": "#/definitions/WaitTimeFromHeaderBackoffStrategy" + - "$ref": "#/definitions/WaitUntilTimeFromHeaderBackoffStrategy" + max_retries: + type: integer + default: 5 + response_filters: + type: array + items: + "$ref": "#/definitions/HttpResponseFilter" + $options: + type: object + additional_properties: true + DefaultPaginator: + description: Default pagination implementation to request pages of results with a fixed size until the pagination strategy no longer returns a next_page_token + type: object + required: + - type + - pagination_strategy + - url_base + properties: + type: + enum: [DefaultPaginator] + pagination_strategy: + anyOf: + - "$ref": "#/definitions/CursorPagination" + - "$ref": "#/definitions/CustomPaginationStrategy" + - "$ref": "#/definitions/OffsetIncrement" + - "$ref": "#/definitions/PageIncrement" + url_base: + type: string + decoder: + "$ref": "#/definitions/JsonDecoder" + page_size_option: + "$ref": "#/definitions/RequestOption" + page_token_option: + "$ref": "#/definitions/RequestOption" + $options: + type: object + additionalProperties: true + DefaultSchemaLoader: + description: Loads a schema from the default location or returns an empty schema for streams that have not defined their schema file yet. + type: object + required: + - type + properties: + type: + enum: [DefaultSchemaLoader] + $options: + type: object + additionalProperties: true + DpathExtractor: + description: Record extractor that searches a decoded response over a path defined as an array of fields + type: object + required: + - type + - field_pointer + properties: + type: + enum: [DpathExtractor] + field_pointer: + type: array + items: + - type: string + decoder: + "$ref": "#/definitions/JsonDecoder" + $options: + type: object + additionalProperties: true + ExponentialBackoffStrategy: + description: Backoff strategy with an exponential backoff interval + type: object + required: + - type + properties: + type: + enum: [ExponentialBackoffStrategy] + factor: + anyOf: + - type: number + - type: string + default: 5 + $options: + type: object + additionalProperties: true + HttpRequester: + description: Default implementation of a requester + type: object + required: + - type + - name + - path + - url_base + properties: + type: + enum: [HttpRequester] + name: + type: string + path: + type: string + url_base: + type: string + authenticator: + anyOf: + - "$ref": "#/definitions/ApiKeyAuthenticator" + - "$ref": "#/definitions/BasicHttpAuthenticator" + - "$ref": "#/definitions/BearerAuthenticator" + - "$ref": "#/definitions/CustomAuthenticator" + - "$ref": "#/definitions/DeclarativeOauth2Authenticator" + - "$ref": "#/definitions/NoAuth" + error_handler: + anyOf: + - "$ref": "#/definitions/DefaultErrorHandler" + - "$ref": "#/definitions/CustomErrorHandler" + - "$ref": "#/definitions/CompositeErrorHandler" + http_method: + anyOf: + - type: string + - type: string + enum: + - GET + - POST + default: GET + request_options_provider: + "$ref": "#/definitions/InterpolatedRequestOptionsProvider" + $options: + type: object + additionalProperties: true + HttpResponseFilter: + description: A filter that is used to select on properties of the HTTP response received + type: object + required: + - action + properties: + type: + enum: [HttpResponseFilter] + action: + type: string + enum: + - SUCCESS + - FAIL + - RETRY + - IGNORE + error_message: + type: string + error_message_contains: + type: string + http_codes: + type: array + items: + type: integer + uniqueItems: true + predicate: + type: string + $options: + type: object + additionalProperties: true + InterpolatedRequestOptionsProvider: + description: Defines the request options to set on an outgoing HTTP request by evaluating `InterpolatedMapping`s + type: object + required: + - type + properties: + type: + enum: [InterpolatedRequestOptionsProvider] + request_body_data: + anyOf: + - type: string + - type: object + additionalProperties: + type: string + request_body_json: + anyOf: + - type: string + - type: object + additionalProperties: + type: string + request_headers: + anyOf: + - type: string + - type: object + additionalProperties: + type: string + request_parameters: + anyOf: + - type: string + - type: object + additionalProperties: + type: string + $options: + type: object + additionalProperties: true + JsonFileSchemaLoader: + description: Loads the schema from a json file + type: object + required: + - type + properties: + type: + enum: [JsonFileSchemaLoader] + file_path: + type: string + $options: + type: object + additionalProperties: true + JsonDecoder: + type: object + required: + - type + properties: + type: + enum: [JsonDecoder] + ListStreamSlicer: + description: Stream slicer that iterates over the values of a list + type: object + required: + - type + - cursor_field + - slice_values + properties: + type: + enum: [ListStreamSlicer] + cursor_field: + type: string + slice_values: + anyOf: + - type: string + - type: array + items: + type: string + request_option: + "$ref": "#/definitions/RequestOption" + $options: + type: object + additionalProperties: true + MinMaxDatetime: + description: Compares the provided date against optional minimum or maximum times. The max_datetime serves as the ceiling and will be returned when datetime exceeds it. The min_datetime serves as the floor + type: object + required: + - type + - datetime + properties: + type: + enum: [MinMaxDatetime] + datetime: + type: string + datetime_format: + type: string + default: "" + max_datetime: + type: string + min_datetime: + type: string + $options: + type: object + additionalProperties: true + NoAuth: + description: Authenticator for requests requiring no authentication + type: object + required: + - type + properties: + type: + enum: [NoAuth] + $options: + type: object + additionalProperties: true + NoPagination: + description: Pagination implementation that never returns a next page + type: object + required: + - type + properties: + type: + enum: [NoPagination] + OffsetIncrement: + description: Pagination strategy that returns the number of records reads so far and returns it as the next page token + type: object + required: + - type + - page_size + properties: + type: + enum: [OffsetIncrement] + page_size: + type: integer + $options: + type: object + additionalProperties: true + PageIncrement: + description: Pagination strategy that returns the number of pages reads so far and returns it as the next page token + type: object + required: + - type + - page_size + properties: + type: + enum: [PageIncrement] + page_size: + type: integer + start_from_page: + type: integer + default: 0 + $options: + type: object + additionalProperties: true + ParentStreamConfig: + description: Describes how to create a stream slice from a parent stream + type: object + required: + - type + - parent_key + - stream + - stream_slice_field + properties: + type: + enum: [ParentStreamConfig] + parent_key: + type: string + stream: + "$ref": "#/definitions/DeclarativeStream" + stream_slice_field: + type: string + request_option: + "$ref": "#/definitions/RequestOption" + $options: + type: object + additionalProperties: true + RecordFilter: + description: Filter applied on a list of Records + type: object + required: + - type + properties: + type: + enum: [RecordFilter] + backoff_time_in_seconds: + type: string + default: "" + $options: + type: object + additionalProperties: true + RecordSelector: + description: Responsible for translating an HTTP response into a list of records by extracting records from the response and optionally filtering records based on a heuristic. + type: object + required: + - type + - extractor + properties: + type: + enum: [RecordSelector] + extractor: + anyOf: + - "$ref": "#/definitions/CustomRecordExtractor" + - "$ref": "#/definitions/DpathExtractor" + record_filter: + "$ref": "#/definitions/RecordFilter" + $options: + type: object + additionalProperties: true + RemoveFields: + description: A transformation which removes fields from a record. The fields removed are designated using FieldPointers. During transformation, if a field or any of its parents does not exist in the record, no error is thrown. + type: object + required: + - type + - field_pointers + properties: + type: + enum: [RemoveFields] + field_pointers: + type: array + items: + items: + type: string + RequestOption: + description: Describes an option to set on a request + type: object + required: + - type + - inject_into + properties: + type: + enum: [RequestOption] + inject_into: + enum: + - request_parameter + - header + - path + - body_data + - body_json + field_name: + type: string + SimpleRetriever: + description: Retrieves records by synchronously sending requests to fetch records. The retriever acts as an orchestrator between the requester, the record selector, the paginator, and the stream slicer. + type: object + required: + - type + - record_selector + - requester + properties: + type: + enum: [SimpleRetriever] + record_selector: + "$ref": "#/definitions/RecordSelector" + requester: + "$ref": "#/definitions/HttpRequester" + name: + type: string + default: "" + paginator: + anyOf: + - "$ref": "#/definitions/DefaultPaginator" + - "$ref": "#/definitions/NoPagination" + primary_key: + anyOf: + - type: string + - type: array + items: + type: string + - type: array + items: + type: array + items: + type: string + default: "" + stream_slicer: + anyOf: + - "$ref": "#/definitions/CartesianProductStreamSlicer" + - "$ref": "#/definitions/CustomStreamSlicer" + - "$ref": "#/definitions/DatetimeStreamSlicer" + - "$ref": "#/definitions/ListStreamSlicer" + - "$ref": "#/definitions/SingleSlice" + - "$ref": "#/definitions/SubstreamSlicer" + $options: + type: object + additionalProperties: true + SingleSlice: + description: Stream slicer returning only a single stream slice + type: object + required: + - type + properties: + type: + enum: [SingleSlice] + $options: + type: object + additionalProperties: true + Spec: + description: A connection specification made up of information about the connector and how it can be configured + type: object + required: + - type + - connection_specification + - documentation_url + properties: + type: + enum: [Spec] + connection_specification: + type: object + additionalProperties: true + documentation_url: + type: string + SubstreamSlicer: + description: Stream slicer that iterates over the parent's stream slices and records and emits slices by interpolating the slice_definition mapping + type: object + required: + - type + - parent_stream_configs + properties: + type: + enum: [SubstreamSlicer] + parent_stream_configs: + type: array + items: + "$ref": "#/definitions/ParentStreamConfig" + $options: + type: object + additionalProperties: true + WaitTimeFromHeaderBackoffStrategy: + description: Extract wait time from http header + type: object + required: + - type + - header + properties: + type: + enum: [WaitTimeFromHeaderBackoffStrategy] + header: + type: string + regex: + type: string + $options: + type: object + additionalProperties: true + WaitUntilTimeFromHeaderBackoffStrategy: + description: Extract time at which we can retry the request from response header and wait for the difference between now and that time + type: object + required: + - type + - header + properties: + type: + enum: [WaitUntilTimeFromHeaderBackoffStrategy] + header: + type: string + min_wait: + anyOf: + - type: number + - type: string + regex: + type: string + $options: + type: object + additionalProperties: true diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/__init__.py new file mode 100644 index 000000000000..986458e0779a --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/__init__.py @@ -0,0 +1,3 @@ +# generated by generate-component-manifest-files +from .declarative_component_schema import * +from .low_code_component_schema import * diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py new file mode 100644 index 000000000000..aff43c54c4a5 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -0,0 +1,490 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +# generated by datamodel-codegen: +# filename: declarative_component_schema.yaml + +from __future__ import annotations + +from enum import Enum +from typing import Any, Dict, List, Optional, Union + +from pydantic import BaseModel, Extra, Field +from typing_extensions import Literal + + +class AddedFieldDefinition(BaseModel): + type: Literal["AddedFieldDefinition"] + path: List[str] + value: str + + +class AddFields(BaseModel): + type: Literal["AddFields"] + fields: List[AddedFieldDefinition] + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class ApiKeyAuthenticator(BaseModel): + type: Literal["ApiKeyAuthenticator"] + api_token: str + header: Optional[str] = None + + +class BasicHttpAuthenticator(BaseModel): + type: Literal["BasicHttpAuthenticator"] + username: str + password: Optional[str] = None + + +class BearerAuthenticator(BaseModel): + type: Literal["BearerAuthenticator"] + api_token: str + + +class CheckStream(BaseModel): + type: Literal["CheckStream"] + stream_names: List[str] + + +class ConstantBackoffStrategy(BaseModel): + type: Literal["ConstantBackoffStrategy"] + backoff_time_in_seconds: Union[float, str] + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CustomAuthenticator(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["CustomAuthenticator"] + class_name: str + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CustomBackoffStrategy(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["CustomBackoffStrategy"] + class_name: str + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CustomErrorHandler(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["CustomErrorHandler"] + class_name: str + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CustomPaginationStrategy(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["CustomPaginationStrategy"] + class_name: str + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CustomRecordExtractor(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["CustomRecordExtractor"] + class_name: str + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CustomRetriever(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["CustomRetriever"] + class_name: str + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CustomStreamSlicer(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["CustomStreamSlicer"] + class_name: str + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CustomTransformation(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["CustomTransformation"] + class_name: str + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class DeclarativeOauth2Authenticator(BaseModel): + type: Literal["DeclarativeOauth2Authenticator"] + client_id: str + client_secret: str + refresh_token: str + token_refresh_endpoint: str + access_token_name: Optional[str] = None + expires_in_name: Optional[str] = None + grant_type: Optional[str] = None + refresh_request_body: Optional[Dict[str, Any]] = None + scopes: Optional[List[str]] = None + token_expiry_date: Optional[str] = None + + +class DefaultSchemaLoader(BaseModel): + type: Literal["DefaultSchemaLoader"] + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class ExponentialBackoffStrategy(BaseModel): + type: Literal["ExponentialBackoffStrategy"] + factor: Optional[Union[float, str]] = 5 + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class HttpMethodEnum(Enum): + GET = "GET" + POST = "POST" + + +class Action(Enum): + SUCCESS = "SUCCESS" + FAIL = "FAIL" + RETRY = "RETRY" + IGNORE = "IGNORE" + + +class HttpResponseFilter(BaseModel): + type: Literal["HttpResponseFilter"] + action: Action + error_message: Optional[str] = None + error_message_contains: Optional[str] = None + http_codes: Optional[List[int]] = None + predicate: Optional[str] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class InlineSchemaLoader(BaseModel): + type: Literal["InlineSchemaLoader"] + schema_: Optional[Dict[str, Any]] = Field(None, alias="schema") + + +class InterpolatedRequestOptionsProvider(BaseModel): + type: Literal["InterpolatedRequestOptionsProvider"] + request_body_data: Optional[Union[str, Dict[str, str]]] = None + request_body_json: Optional[Union[str, Dict[str, str]]] = None + request_headers: Optional[Union[str, Dict[str, str]]] = None + request_parameters: Optional[Union[str, Dict[str, str]]] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class Type(Enum): + JsonFileSchemaLoader = "JsonFileSchemaLoader" + JsonSchema = "JsonSchema" + + +class JsonFileSchemaLoader(BaseModel): + type: Type + file_path: Optional[str] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class JsonDecoder(BaseModel): + type: Literal["JsonDecoder"] + + +class MinMaxDatetime(BaseModel): + type: Literal["MinMaxDatetime"] + datetime: str + datetime_format: Optional[str] = "" + max_datetime: Optional[str] = None + min_datetime: Optional[str] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class NoAuth(BaseModel): + type: Literal["NoAuth"] + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class NoPagination(BaseModel): + type: Literal["NoPagination"] + + +class OffsetIncrement(BaseModel): + type: Literal["OffsetIncrement"] + page_size: Union[float, str] + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class PageIncrement(BaseModel): + type: Literal["PageIncrement"] + page_size: int + start_from_page: Optional[int] = 0 + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class PrimaryKey(BaseModel): + __root__: Union[str, List[str], List[List[str]]] = Field(..., description="The stream field to be used to distinguish unique rows") + + +class RecordFilter(BaseModel): + type: Literal["RecordFilter"] + backoff_time_in_seconds: Optional[str] = "" + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class RemoveFields(BaseModel): + type: Literal["RemoveFields"] + field_pointers: List[List[str]] + + +class InjectInto(Enum): + request_parameter = "request_parameter" + header = "header" + path = "path" + body_data = "body_data" + body_json = "body_json" + + +class RequestOption(BaseModel): + type: Literal["RequestOption"] + inject_into: InjectInto + field_name: Optional[str] = None + + +class Schemas(BaseModel): + pass + + class Config: + extra = Extra.allow + + +class SessionTokenAuthenticator(BaseModel): + type: Literal["SessionTokenAuthenticator"] + api_url: str + header: str + login_url: str + session_token: str + session_token_response_key: str + username: str + validate_session_url: str + password: Optional[str] = "" + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class SingleSlice(BaseModel): + type: Literal["SingleSlice"] + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class Spec(BaseModel): + type: Literal["Spec"] + connection_specification: Dict[str, Any] + documentation_url: str + + +class WaitTimeFromHeader(BaseModel): + type: Literal["WaitTimeFromHeader"] + header: str + regex: Optional[str] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class WaitUntilTimeFromHeader(BaseModel): + type: Literal["WaitUntilTimeFromHeader"] + header: str + min_wait: Optional[Union[float, str]] = None + regex: Optional[str] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CursorPagination(BaseModel): + type: Literal["CursorPagination"] + cursor_value: str + page_size: Optional[int] = None + stop_condition: Optional[str] = None + decoder: Optional[JsonDecoder] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class DatetimeStreamSlicer(BaseModel): + type: Literal["DatetimeStreamSlicer"] + cursor_field: str + datetime_format: str + end_datetime: Union[str, MinMaxDatetime] + start_datetime: Union[str, MinMaxDatetime] + step: str + end_time_option: Optional[RequestOption] = None + lookback_window: Optional[str] = None + start_time_option: Optional[RequestOption] = None + stream_state_field_end: Optional[str] = None + stream_state_field_start: Optional[str] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class DefaultErrorHandler(BaseModel): + type: Literal["DefaultErrorHandler"] + backoff_strategies: Optional[ + List[ + Union[ + ConstantBackoffStrategy, + CustomBackoffStrategy, + ExponentialBackoffStrategy, + WaitTimeFromHeader, + WaitUntilTimeFromHeader, + ] + ] + ] = None + max_retries: Optional[int] = 5 + response_filters: Optional[List[HttpResponseFilter]] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class DefaultPaginator(BaseModel): + type: Literal["DefaultPaginator"] + pagination_strategy: Union[CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement] + url_base: str + decoder: Optional[JsonDecoder] = None + page_size_option: Optional[RequestOption] = None + page_token_option: Optional[RequestOption] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class DpathExtractor(BaseModel): + type: Literal["DpathExtractor"] + field_pointer: List[str] + decoder: Optional[JsonDecoder] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class ListStreamSlicer(BaseModel): + type: Literal["ListStreamSlicer"] + cursor_field: str + slice_values: Union[str, List[str]] + request_option: Optional[RequestOption] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class RecordSelector(BaseModel): + type: Literal["RecordSelector"] + extractor: Union[CustomRecordExtractor, DpathExtractor] + record_filter: Optional[RecordFilter] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class CompositeErrorHandler(BaseModel): + type: Literal["CompositeErrorHandler"] + error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class HttpRequester(BaseModel): + type: Literal["HttpRequester"] + name: str + path: str + url_base: str + authenticator: Optional[ + Union[ + ApiKeyAuthenticator, + BasicHttpAuthenticator, + BearerAuthenticator, + CustomAuthenticator, + DeclarativeOauth2Authenticator, + NoAuth, + SessionTokenAuthenticator, + ] + ] = None + error_handler: Optional[Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler]] = None + http_method: Optional[Union[str, HttpMethodEnum]] = "GET" + request_options_provider: Optional[InterpolatedRequestOptionsProvider] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class DeclarativeSource(BaseModel): + type: Literal["DeclarativeSource"] + check: CheckStream + streams: List[DeclarativeStream] + version: str + schemas: Optional[Schemas] = None + spec: Optional[Spec] = None + + +class CartesianProductStreamSlicer(BaseModel): + type: Literal["CartesianProductStreamSlicer"] + stream_slicers: List[ + Union[ + CustomStreamSlicer, + DatetimeStreamSlicer, + ListStreamSlicer, + SingleSlice, + SubstreamSlicer, + ] + ] + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class DeclarativeStream(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["DeclarativeStream"] + retriever: Union[CustomRetriever, SimpleRetriever] + checkpoint_interval: Optional[int] = None + name: Optional[str] = "" + primary_key: Optional[Union[str, List[str], List[List[str]]]] = "" + schema_loader: Optional[Union[DefaultSchemaLoader, InlineSchemaLoader, JsonFileSchemaLoader]] = None + stream_cursor_field: Optional[Union[str, List[str]]] = None + transformations: Optional[List[Union[AddFields, CustomTransformation, RemoveFields]]] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class ParentStreamConfig(BaseModel): + type: Literal["ParentStreamConfig"] + parent_key: str + stream: DeclarativeStream + stream_slice_field: str + request_option: Optional[RequestOption] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class SimpleRetriever(BaseModel): + type: Literal["SimpleRetriever"] + record_selector: RecordSelector + requester: HttpRequester + name: Optional[str] = "" + paginator: Optional[Union[DefaultPaginator, NoPagination]] = None + primary_key: Optional[PrimaryKey] = None + stream_slicer: Optional[ + Union[ + CartesianProductStreamSlicer, + CustomStreamSlicer, + DatetimeStreamSlicer, + ListStreamSlicer, + SingleSlice, + SubstreamSlicer, + ] + ] = None + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +class SubstreamSlicer(BaseModel): + type: Literal["SubstreamSlicer"] + parent_stream_configs: List[ParentStreamConfig] + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + +CompositeErrorHandler.update_forward_refs() +DeclarativeSource.update_forward_refs() +CartesianProductStreamSlicer.update_forward_refs() +DeclarativeStream.update_forward_refs() +SimpleRetriever.update_forward_refs() diff --git a/airbyte-cdk/python/bin/generate-component-manifest-files.sh b/airbyte-cdk/python/bin/generate-component-manifest-files.sh new file mode 100755 index 000000000000..31d5066dce38 --- /dev/null +++ b/airbyte-cdk/python/bin/generate-component-manifest-files.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -e + +[ -z "$ROOT_DIR" ] && exit 1 + +YAML_DIR=airbyte-cdk/python/airbyte_cdk/sources/declarative +OUTPUT_DIR=airbyte-cdk/python/airbyte_cdk/sources/declarative/models + +function main() { + rm -rf "$ROOT_DIR/$OUTPUT_DIR"/*.py + echo "# generated by generate-component-manifest-files" > "$ROOT_DIR/$OUTPUT_DIR"/__init__.py + + for f in "$ROOT_DIR/$YAML_DIR"/*.yaml; do + filename_wo_ext=$(basename "$f" | cut -d . -f 1) + echo "from .$filename_wo_ext import *" >> "$ROOT_DIR/$OUTPUT_DIR"/__init__.py + + docker run --user "$(id -u):$(id -g)" -v "$ROOT_DIR":/airbyte airbyte/code-generator:dev \ + --input "/airbyte/$YAML_DIR/$filename_wo_ext.yaml" \ + --output "/airbyte/$OUTPUT_DIR/$filename_wo_ext.py" \ + --use-title-as-name \ + --disable-timestamp \ + --enum-field-as-literal one + done +} + +main "$@" diff --git a/airbyte-cdk/python/build.gradle b/airbyte-cdk/python/build.gradle index c5cf8cc3af33..addc32e7c80f 100644 --- a/airbyte-cdk/python/build.gradle +++ b/airbyte-cdk/python/build.gradle @@ -13,6 +13,12 @@ task generateProtocolClassFiles(type: Exec) { dependsOn ':tools:code-generator:airbyteDocker' } +task generateComponentManifestClassFiles(type: Exec) { + environment 'ROOT_DIR', rootDir.absolutePath + commandLine 'bin/generate-component-manifest-files.sh' + dependsOn ':tools:code-generator:airbyteDocker' +} + task validateSourceYamlManifest(type: Exec) { environment 'ROOT_DIR', rootDir.absolutePath commandLine 'bin/validate-yaml-schema.sh' @@ -22,3 +28,8 @@ blackFormat.dependsOn generateProtocolClassFiles isortFormat.dependsOn generateProtocolClassFiles flakeCheck.dependsOn generateProtocolClassFiles installReqs.dependsOn generateProtocolClassFiles + +blackFormat.dependsOn generateComponentManifestClassFiles +isortFormat.dependsOn generateComponentManifestClassFiles +flakeCheck.dependsOn generateComponentManifestClassFiles +installReqs.dependsOn generateComponentManifestClassFiles diff --git a/build.gradle b/build.gradle index 8c30547fc84c..5d7c2213598f 100644 --- a/build.gradle +++ b/build.gradle @@ -485,6 +485,7 @@ subprojects { subproj -> task('generate') { dependsOn subprojects.collect { it.getTasksByName('generateProtocolClassFiles', true) } + dependsOn subprojects.collect { it.getTasksByName('generateComponentManifestClassFiles', true) } dependsOn subprojects.collect { it.getTasksByName('generateJsonSchema2Pojo', true) } } @@ -527,6 +528,11 @@ subprojects { if (generateFilesTask != null) { licenseTask.dependsOn generateFilesTask } + + def generateManifestFilesTask = project.tasks.findByName('generateComponentManifestClassFiles') + if (generateManifestFilesTask != null) { + licenseTask.dependsOn generateManifestFilesTask + } } task listAllDependencies(type: DependencyReportTask) {} } diff --git a/pyproject.toml b/pyproject.toml index 17544d0afc31..e584fd9ba9c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ extend-exclude = [ "models", ".eggs", "airbyte-cdk/python/airbyte_cdk/models/__init__.py", + "airbyte-cdk/python/airbyte_cdk/sources/declarative/models/__init__.py", ".tox", "airbyte_api_client", "**/generated/*", From a55eb7df2d64ac8fc9c5166c94a9d4a0a1a88189 Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Tue, 20 Dec 2022 14:57:25 -0800 Subject: [PATCH 4/7] Fix CI Dependency Check Failures (#20666) * pardot * plaid * quickbooks * appfollow * appstore * cloudtrail * clickup * clockify * coda * Coinmarketcap * cooper * dixa * dv-360 * exchange-rates * file * gridly * Hellobaton * kustomer * mailersend * microsoft dataverse * n8n * PersistIq * Survey Sparrow * Twilio Taskrouter * YouTube Analytics Business * Younium * Yahoo Finance Price * Yandex Metrica * Xero * WooCommerce * XKCD * Webflow * US Census API * Qonto * Pivotal Tracker * KVDB * Firestore * Ignore even more connectors * test run * SFTP JSON * cleanup * move pardot changelog * update links * remove testing HACK * Update docs/integrations/sources/dixa.md Co-authored-by: Augustin * Update docs/integrations/sources/kustomer-singer.md Co-authored-by: Augustin * Update docs/integrations/sources/pardot.md Co-authored-by: Augustin * Update docs/integrations/sources/kustomer-singer.md Co-authored-by: Augustin * Update docs/integrations/sources/pardot.md Co-authored-by: Augustin Co-authored-by: Augustin --- .../connectors/source-pardot/pardot.md | 65 +++++++------- docs/integrations/destinations/firestore.md | 9 ++ docs/integrations/destinations/kvdb.md | 9 ++ docs/integrations/destinations/sftp-json.md | 16 ++-- docs/integrations/sources/appfollow.md | 2 +- .../{appstore.md => appstore-singer.md} | 0 docs/integrations/sources/aws-cloudtrail.md | 42 ++++----- docs/integrations/sources/clickup-api.md | 9 ++ docs/integrations/sources/clockify.md | 9 ++ docs/integrations/sources/coda.md | 28 +++--- docs/integrations/sources/coinmarketcap.md | 33 +++---- docs/integrations/sources/copper.md | 30 ++++--- docs/integrations/sources/dixa.md | 40 ++++----- .../sources/{dv360.md => dv-360.md} | 51 ++++++----- ...{exchangeratesapi.md => exchange-rates.md} | 38 ++++----- docs/integrations/sources/file.md | 54 ++++++------ docs/integrations/sources/gridly.md | 22 +++-- docs/integrations/sources/hellobaton.md | 54 ++++++------ docs/integrations/sources/kustomer-singer.md | 47 ++++++++++ docs/integrations/sources/kustomer.md | 47 ---------- docs/integrations/sources/mailersend.md | 5 +- .../sources/microsoft-dataverse.md | 16 ++-- docs/integrations/sources/n8n.md | 5 +- docs/integrations/sources/pardot.md | 7 +- docs/integrations/sources/persistiq.md | 25 +++--- docs/integrations/sources/pivotal-tracker.md | 51 +++++------ docs/integrations/sources/plaid.md | 2 + docs/integrations/sources/qonto.md | 9 ++ .../integrations/sources/quickbooks-singer.md | 84 ++++++++++++++++++ docs/integrations/sources/quickbooks.md | 85 ------------------- .../{surveysparrow.md => survey-sparrow.md} | 0 ...iliotaskrouter.md => twilio-taskrouter.md} | 16 ++-- docs/integrations/sources/us-census.md | 32 +++---- docs/integrations/sources/webflow.md | 24 +++--- docs/integrations/sources/woocommerce.md | 62 +++++++------- docs/integrations/sources/xero.md | 6 +- docs/integrations/sources/xkcd.md | 11 +-- .../sources/yahoo-finance-price.md | 9 ++ docs/integrations/sources/yandex-metrica.md | 42 +++++---- docs/integrations/sources/younium.md | 34 +++++--- .../sources/youtube-analytics-business.md | 9 ++ .../connections/incremental-append.md | 70 +++++++-------- .../incremental-deduped-history.md | 80 ++++++++--------- tools/bin/ci_check_dependency.py | 17 +++- 44 files changed, 726 insertions(+), 580 deletions(-) create mode 100644 docs/integrations/destinations/firestore.md create mode 100644 docs/integrations/destinations/kvdb.md rename docs/integrations/sources/{appstore.md => appstore-singer.md} (100%) create mode 100644 docs/integrations/sources/clickup-api.md create mode 100644 docs/integrations/sources/clockify.md rename docs/integrations/sources/{dv360.md => dv-360.md} (67%) rename docs/integrations/sources/{exchangeratesapi.md => exchange-rates.md} (85%) create mode 100644 docs/integrations/sources/kustomer-singer.md delete mode 100644 docs/integrations/sources/kustomer.md create mode 100644 docs/integrations/sources/qonto.md create mode 100644 docs/integrations/sources/quickbooks-singer.md delete mode 100644 docs/integrations/sources/quickbooks.md rename docs/integrations/sources/{surveysparrow.md => survey-sparrow.md} (100%) rename docs/integrations/sources/{twiliotaskrouter.md => twilio-taskrouter.md} (79%) create mode 100644 docs/integrations/sources/yahoo-finance-price.md create mode 100644 docs/integrations/sources/youtube-analytics-business.md diff --git a/airbyte-integrations/connectors/source-pardot/pardot.md b/airbyte-integrations/connectors/source-pardot/pardot.md index 72ae9ff15455..e877c48a103a 100644 --- a/airbyte-integrations/connectors/source-pardot/pardot.md +++ b/airbyte-integrations/connectors/source-pardot/pardot.md @@ -8,27 +8,27 @@ The Pardot supports full refresh syncs Several output streams are available from this source: -* [Campaigns](https://developer.salesforce.com/docs/marketing/pardot/guide/campaigns-v4.html) -* [EmailClicks](https://developer.salesforce.com/docs/marketing/pardot/guide/batch-email-clicks-v4.html) -* [ListMembership](https://developer.salesforce.com/docs/marketing/pardot/guide/list-memberships-v4.html) -* [Lists](https://developer.salesforce.com/docs/marketing/pardot/guide/lists-v4.html) -* [ProspectAccounts](https://developer.salesforce.com/docs/marketing/pardot/guide/prospect-accounts-v4.html) -* [Prospects](https://developer.salesforce.com/docs/marketing/pardot/guide/prospects-v4.html) -* [Users](https://developer.salesforce.com/docs/marketing/pardot/guide/users-v4.html) -* [VisitorActivities](https://developer.salesforce.com/docs/marketing/pardot/guide/visitor-activities-v4.html) -* [Visitors](https://developer.salesforce.com/docs/marketing/pardot/guide/visitors-v4.html) -* [Visits](https://developer.salesforce.com/docs/marketing/pardot/guide/visits-v4.html) +- [Campaigns](https://developer.salesforce.com/docs/marketing/pardot/guide/campaigns-v4.html) +- [EmailClicks](https://developer.salesforce.com/docs/marketing/pardot/guide/batch-email-clicks-v4.html) +- [ListMembership](https://developer.salesforce.com/docs/marketing/pardot/guide/list-memberships-v4.html) +- [Lists](https://developer.salesforce.com/docs/marketing/pardot/guide/lists-v4.html) +- [ProspectAccounts](https://developer.salesforce.com/docs/marketing/pardot/guide/prospect-accounts-v4.html) +- [Prospects](https://developer.salesforce.com/docs/marketing/pardot/guide/prospects-v4.html) +- [Users](https://developer.salesforce.com/docs/marketing/pardot/guide/users-v4.html) +- [VisitorActivities](https://developer.salesforce.com/docs/marketing/pardot/guide/visitor-activities-v4.html) +- [Visitors](https://developer.salesforce.com/docs/marketing/pardot/guide/visitors-v4.html) +- [Visits](https://developer.salesforce.com/docs/marketing/pardot/guide/visits-v4.html) If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose) ### Features -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental Sync | No | -| SSL connection | No | -| Namespaces | No | +| Feature | Supported? | +| :---------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental Sync | No | +| SSL connection | No | +| Namespaces | No | ### Performance considerations @@ -38,26 +38,19 @@ The Pardot connector should not run into Pardot API limitations under normal usa ### Requirements -* Pardot Account -* Pardot Business Unit ID -* Client ID -* Client Secret -* Refresh Token -* Start Date -* Is Sandbox environment? +- Pardot Account +- Pardot Business Unit ID +- Client ID +- Client Secret +- Refresh Token +- Start Date +- Is Sandbox environment? ### Setup guide -* `pardot_business_unit_id`: Pardot Business ID, can be found at Setup > Pardot > Pardot Account Setup -* `client_id`: The Consumer Key that can be found when viewing your app in Salesforce -* `client_secret`: The Consumer Secret that can be found when viewing your app in Salesforce -* `refresh_token`: Salesforce Refresh Token used for Airbyte to access your Salesforce account. If you don't know what this is, follow [this guide](https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b) to retrieve it. -* `start_date`: UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. Leave blank to skip this filter -* `is_sandbox`: Whether or not the the app is in a Salesforce sandbox. If you do not know what this, assume it is false. - -## Changelog - -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.0 | 2021-10-16 | [7091](https://github.com/airbytehq/airbyte/pull/7091) | 🎉 New Source: Pardot | - +- `pardot_business_unit_id`: Pardot Business ID, can be found at Setup > Pardot > Pardot Account Setup +- `client_id`: The Consumer Key that can be found when viewing your app in Salesforce +- `client_secret`: The Consumer Secret that can be found when viewing your app in Salesforce +- `refresh_token`: Salesforce Refresh Token used for Airbyte to access your Salesforce account. If you don't know what this is, follow [this guide](https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b) to retrieve it. +- `start_date`: UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. Leave blank to skip this filter +- `is_sandbox`: Whether or not the the app is in a Salesforce sandbox. If you do not know what this, assume it is false. diff --git a/docs/integrations/destinations/firestore.md b/docs/integrations/destinations/firestore.md new file mode 100644 index 000000000000..c82a9f12068e --- /dev/null +++ b/docs/integrations/destinations/firestore.md @@ -0,0 +1,9 @@ +# Firestore + +The Firestore destination for Airbyte + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :----------------------------------------------------- | :---------------------------- | +| 0.1.1 | 2021-11-21 | [8158](https://github.com/airbytehq/airbyte/pull/8158) | Publish Destination Firestore | diff --git a/docs/integrations/destinations/kvdb.md b/docs/integrations/destinations/kvdb.md new file mode 100644 index 000000000000..1d9a4341e8fc --- /dev/null +++ b/docs/integrations/destinations/kvdb.md @@ -0,0 +1,9 @@ +# KVDB + +The KVDB destination for Airbyte + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :----------------------------------------------------- | :---------------------------- | +| 0.1.0 | 2021-07-19 | [4786](https://github.com/airbytehq/airbyte/pull/4786) | Python Demo Destination: KVDB | diff --git a/docs/integrations/destinations/sftp-json.md b/docs/integrations/destinations/sftp-json.md index 6f3c90d0ae7f..8de20277a763 100644 --- a/docs/integrations/destinations/sftp-json.md +++ b/docs/integrations/destinations/sftp-json.md @@ -13,11 +13,11 @@ Each file will contain a collection of `json` objects which correspond directly #### Features -| Feature | Supported | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | Yes | -| Namespaces | No | +| Feature | Supported | +| :------------------------ | :-------- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | Yes | +| Namespaces | No | #### Performance considerations @@ -34,3 +34,9 @@ The `filename` **should not** have an extension in the configuration, as `.jsonl If `destination_path` is set to `/myfolder/files` and `filename` is set to `mydata`, the resulting file will be `/myfolder/files/mydata.jsonl`. These files can then be accessed by creating an SFTP connection to the server and navigating to the `destination_path`. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :----------------------------------------------------- | :---------------------------- | +| 0.1.0 | 2022-11-24 | [4924](https://github.com/airbytehq/airbyte/pull/4924) | 🎉 New Destination: SFTP JSON | diff --git a/docs/integrations/sources/appfollow.md b/docs/integrations/sources/appfollow.md index 21f56ccac73c..d8a19b02bcae 100644 --- a/docs/integrations/sources/appfollow.md +++ b/docs/integrations/sources/appfollow.md @@ -37,4 +37,4 @@ The Appfollow connector ideally should gracefully handle Appfollow API limitatio | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------- | -| 0.1.0 | 2022-08-11 | [14418](https://github.com/airbytehq/airbyte/pull/14418) | New Source: Appfollow | +| 0.1.1 | 2022-08-11 | [14418](https://github.com/airbytehq/airbyte/pull/14418) | New Source: Appfollow | diff --git a/docs/integrations/sources/appstore.md b/docs/integrations/sources/appstore-singer.md similarity index 100% rename from docs/integrations/sources/appstore.md rename to docs/integrations/sources/appstore-singer.md diff --git a/docs/integrations/sources/aws-cloudtrail.md b/docs/integrations/sources/aws-cloudtrail.md index eb711cd2ec68..6c44cd568c9f 100644 --- a/docs/integrations/sources/aws-cloudtrail.md +++ b/docs/integrations/sources/aws-cloudtrail.md @@ -10,26 +10,26 @@ This Source Connector is based on a [Boto3 CloudTrail](https://boto3.amazonaws.c This Source is capable of syncing the following core Streams: -* [Management Events](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cloudtrail.html#CloudTrail.Client.lookup_events) +- [Management Events](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cloudtrail.html#CloudTrail.Client.lookup_events) Insight events are not supported right now. Only Management events are available. ### Data type mapping | Integration Type | Airbyte Type | Notes | -| :--- | :--- | :--- | -| `string` | `string` | | -| `number` | `integer` | | -| `array` | `array` | | -| `object` | `object` | | +| :--------------- | :----------- | :---- | +| `string` | `string` | | +| `number` | `integer` | | +| `array` | `array` | | +| `object` | `object` | | ### Features -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental Sync | Yes | | -| Namespaces | No | | +| Feature | Supported?\(Yes/No\) | Notes | +| :---------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental Sync | Yes | | +| Namespaces | No | | ### Performance considerations @@ -39,9 +39,9 @@ The rate of lookup requests for `events` stream is limited to two per second, pe ### Requirements -* AWS Access key ID -* AWS Secret access key -* AWS region name +- AWS Access key ID +- AWS Secret access key +- AWS region name ### Setup guide @@ -49,10 +49,10 @@ Please, follow this [steps](https://docs.aws.amazon.com/powershell/latest/usergu ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.3 | 2021-12-23 | [8434](https://github.com/airbytehq/airbyte/pull/8434) | Update fields in source-connectors specifications | -| 0.1.2 | 2021-08-04 | [5152](https://github.com/airbytehq/airbyte/pull/5152) | Fix connector spec.json | -| 0.1.1 | 2021-07-06 | [4539](https://github.com/airbytehq/airbyte/pull/4539) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | -| 0.1.0 | 2021-06-23 | [4122](https://github.com/airbytehq/airbyte/pull/4122) | Initial release supporting the LookupEvent API | - +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------ | +| 0.1.4 | 2022-04-11 | [11763](https://github.com/airbytehq/airbyte/pull/11763) | Upgrade to Python 3.9 | +| 0.1.3 | 2021-12-23 | [8434](https://github.com/airbytehq/airbyte/pull/8434) | Update fields in source-connectors specifications | +| 0.1.2 | 2021-08-04 | [5152](https://github.com/airbytehq/airbyte/pull/5152) | Fix connector spec.json | +| 0.1.1 | 2021-07-06 | [4539](https://github.com/airbytehq/airbyte/pull/4539) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | +| 0.1.0 | 2021-06-23 | [4122](https://github.com/airbytehq/airbyte/pull/4122) | Initial release supporting the LookupEvent API | diff --git a/docs/integrations/sources/clickup-api.md b/docs/integrations/sources/clickup-api.md new file mode 100644 index 000000000000..3801d5708934 --- /dev/null +++ b/docs/integrations/sources/clickup-api.md @@ -0,0 +1,9 @@ +# ClickUP API + +The Airbyte Source for [ClickUP API](https://clickup.com/api/) + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :--------------------- | +| 0.1.0 | 2022-11-13 | [17770](https://github.com/airbytehq/airbyte/pull/17770) | 🎉 New Source: ClickUp | diff --git a/docs/integrations/sources/clockify.md b/docs/integrations/sources/clockify.md new file mode 100644 index 000000000000..5e5c386f5763 --- /dev/null +++ b/docs/integrations/sources/clockify.md @@ -0,0 +1,9 @@ +# Clockify + +The Airbyte Source for [Clockify](https://clockify.me) + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------- | +| 0.1.0 | 2022-10-26 | [17767](https://github.com/airbytehq/airbyte/pull/17767) | 🎉 New Connector: Clockify [python cdk] | diff --git a/docs/integrations/sources/coda.md b/docs/integrations/sources/coda.md index 301f22536f3b..b7c5438bd040 100755 --- a/docs/integrations/sources/coda.md +++ b/docs/integrations/sources/coda.md @@ -7,6 +7,7 @@ This page contains the setup guide and reference information for the Coda source You can find or create authentication tokens within [Coda](https://coda.io/account#apiSettings). ## Setup guide + ## Step 1: Set up the Coda connector in Airbyte ### For Airbyte Cloud: @@ -22,7 +23,7 @@ You can find or create authentication tokens within [Coda](https://coda.io/accou ### For Airbyte OSS: 1. Navigate to the Airbyte Open Source dashboard. -2. Set the name for your source. +2. Set the name for your source. 3. Enter your `auth_token` - Coda Authentication Token with the necessary permissions \(described below\). 4. Enter your `doc_id` - Document id for a specific document created on Coda. You can check it under [Advanced Settings](https://coda.io/account) by exporting data and copying the id in doc_manifest.json from the downloaded zip. @@ -33,7 +34,7 @@ You can find or create authentication tokens within [Coda](https://coda.io/accou The Coda source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): | Feature | Supported? | -| :---------------- |:-----------| +| :---------------- | :--------- | | Full Refresh Sync | Yes | | Incremental Sync | No | | SSL connection | No | @@ -41,22 +42,25 @@ The Coda source connector supports the following [sync modes](https://docs.airby ## Supported Streams -* [Docs](https://coda.io/developers/apis/v1#tag/Docs/operation/listDocs) -* [Permissions](https://coda.io/developers/apis/v1#tag/Permissions/operation/getPermissions) -* [Categories](https://coda.io/developers/apis/v1#tag/Publishing/operation/listCategories) -* [Pages](https://coda.io/developers/apis/v1#tag/Pages/operation/listPages) -* [Tables](https://coda.io/developers/apis/v1#tag/Tables/operation/listTables) -* [Formulas](https://coda.io/developers/apis/v1#tag/Formulas/operation/listFormulas) -* [Controls](https://coda.io/developers/apis/v1#tag/Controls/operation/listControls) - - +- [Docs](https://coda.io/developers/apis/v1#tag/Docs/operation/listDocs) +- [Permissions](https://coda.io/developers/apis/v1#tag/Permissions/operation/getPermissions) +- [Categories](https://coda.io/developers/apis/v1#tag/Publishing/operation/listCategories) +- [Pages](https://coda.io/developers/apis/v1#tag/Pages/operation/listPages) +- [Tables](https://coda.io/developers/apis/v1#tag/Tables/operation/listTables) +- [Formulas](https://coda.io/developers/apis/v1#tag/Formulas/operation/listFormulas) +- [Controls](https://coda.io/developers/apis/v1#tag/Controls/operation/listControls) ## Data type map | Integration Type | Airbyte Type | -|:-----------------| :----------- | +| :--------------- | :----------- | | `string` | `string` | | `integer` | `number` | | `array` | `array` | | `object` | `object` | +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------- | +| 0.1.0 | 2022-11-17 | [18675](https://github.com/airbytehq/airbyte/pull/18675) | 🎉 New source: Coda [python cdk] | diff --git a/docs/integrations/sources/coinmarketcap.md b/docs/integrations/sources/coinmarketcap.md index db336c0fa74e..ca07a8374b80 100644 --- a/docs/integrations/sources/coinmarketcap.md +++ b/docs/integrations/sources/coinmarketcap.md @@ -6,18 +6,18 @@ This source can sync data from the [Coinmarketcap API](https://coinmarketcap.com ## This Source Supports the Following Streams -* [categories](https://coinmarketcap.com/api/documentation/v1/#operation/getV1CryptocurrencyCategories) -* [listing](https://coinmarketcap.com/api/documentation/v1/#operation/getV1CryptocurrencyListingsLatest) -* [quotes](https://coinmarketcap.com/api/documentation/v1/#operation/getV2CryptocurrencyQuotesLatest) -* [fiat](https://coinmarketcap.com/api/documentation/v1/#tag/fiat) -* [exchange](https://coinmarketcap.com/api/documentation/v1/#tag/exchange) +- [categories](https://coinmarketcap.com/api/documentation/v1/#operation/getV1CryptocurrencyCategories) +- [listing](https://coinmarketcap.com/api/documentation/v1/#operation/getV1CryptocurrencyListingsLatest) +- [quotes](https://coinmarketcap.com/api/documentation/v1/#operation/getV2CryptocurrencyQuotesLatest) +- [fiat](https://coinmarketcap.com/api/documentation/v1/#tag/fiat) +- [exchange](https://coinmarketcap.com/api/documentation/v1/#tag/exchange) ### Features -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental Sync | No | | +| Feature | Supported?\(Yes/No\) | Notes | +| :---------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental Sync | No | | ### Performance considerations @@ -27,13 +27,14 @@ Coinmarketcap APIs are under rate limits for the number of API calls allowed per ### Requirements -* [API token](https://coinmarketcap.com/api/documentation/v1/#section/Authentication) -* Data Type: - * latest - * historical +- [API token](https://coinmarketcap.com/api/documentation/v1/#section/Authentication) +- Data Type: + - latest + - historical ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :-------------------------------------------------------- | :----------------------------------------- | -| 0.1.0 | 2022-10-29 | [#18565](https://github.com/airbytehq/airbyte/pull/18565) | 🎉 New Source: Coinmarketcap API [low-code CDK] | \ No newline at end of file +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :-------------------------------------------------------- | :---------------------------------------------- | +| 0.1.1 | 2022-11-01 | [#18790](https://github.com/airbytehq/airbyte/pull/18790) | Correct coinmarket spec | +| 0.1.0 | 2022-10-29 | [#18565](https://github.com/airbytehq/airbyte/pull/18565) | 🎉 New Source: Coinmarketcap API [low-code CDK] | diff --git a/docs/integrations/sources/copper.md b/docs/integrations/sources/copper.md index ee70d64531fc..939670690390 100644 --- a/docs/integrations/sources/copper.md +++ b/docs/integrations/sources/copper.md @@ -7,31 +7,37 @@ This page contains the setup guide and reference information for the Copper sour This Copper source uses the [Copper API](https://developer.copper.com/). ## Setup guide + ### Step 1: Set up Copper #### For Airbyte OSS: + 1. Navigate to the Airbyte Open Source dashboard -2. Enter a name for your source +2. Enter a name for your source 3. Enter your Copper `api_key`. This can be generated by logging into Copper -> Settings -> Integrations -> API Keys 4. Enter your Copper `user_email`. The email used to login to Copper -7. Click **Set up source** +5. Click **Set up source** ## Supported sync modes The Copper source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): -| Feature | Supported? | -| :--- | :-- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | No | -| Replicate Incremental Deletes | No | -| SSL connection | Yes | -| Namespaces | No | +| Feature | Supported? | +| :---------------------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | No | +| Replicate Incremental Deletes | No | +| SSL connection | Yes | +| Namespaces | No | ## Supported Streams -* [People](https://developer.copper.com/people/list-people-search.html) -* [Companies](https://developer.copper.com/companies/list-companies-search.html) -* [Projects](https://developer.copper.com/projects/list-projects-search.html) +- [People](https://developer.copper.com/people/list-people-search.html) +- [Companies](https://developer.copper.com/companies/list-companies-search.html) +- [Projects](https://developer.copper.com/projects/list-projects-search.html) +## Changelog +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :-------------------------------------------------------- | :--------------------------------- | +| 0.1.0 | 2022-11-17 | [#18848](https://github.com/airbytehq/airbyte/pull/18848) | 🎉 New Source: Copper [python cdk] | diff --git a/docs/integrations/sources/dixa.md b/docs/integrations/sources/dixa.md index 16a66c9f159c..c1a1b1c961bf 100644 --- a/docs/integrations/sources/dixa.md +++ b/docs/integrations/sources/dixa.md @@ -2,34 +2,34 @@ ## Sync overview -This source can sync data for the [Dixa conversation\_export API](https://support.dixa.help/en/articles/174-export-conversations-via-api). It supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. +This source can sync data for the [Dixa conversation_export API](https://support.dixa.help/en/articles/174-export-conversations-via-api). It supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. ### Output schema This Source is capable of syncing the following Streams: -* [Conversation export](https://support.dixa.help/en/articles/174-export-conversations-via-api) +- [Conversation export](https://support.dixa.help/en/articles/174-export-conversations-via-api) ### Data type mapping | Integration Type | Airbyte Type | Notes | -| :--- | :--- | :--- | -| `string` | `string` | | -| `int` | `integer` | | -| `timestamp` | `integer` | | -| `array` | `array` | | +| :--------------- | :----------- | :---- | +| `string` | `string` | | +| `int` | `integer` | | +| `timestamp` | `integer` | | +| `array` | `array` | | ### Features -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental Sync | Yes | | -| Namespaces | No | | +| Feature | Supported?\(Yes/No\) | Notes | +| :---------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental Sync | Yes | | +| Namespaces | No | | ### Performance considerations -The connector is limited by standard Dixa conversation\_export API [limits](https://support.dixa.help/en/articles/174-export-conversations-via-api). It should not run into limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. +The connector is limited by standard Dixa conversation_export API [limits](https://support.dixa.help/en/articles/174-export-conversations-via-api). It should not run into limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. When using the connector, keep in mind that increasing the `batch_size` parameter will decrease the number of requests sent to the API, but increase the response and processing time. @@ -37,7 +37,7 @@ When using the connector, keep in mind that increasing the `batch_size` paramete ### Requirements -* Dixa API token +- Dixa API token ### Setup guide @@ -49,9 +49,9 @@ When using the connector, keep in mind that increasing the `batch_size` paramete ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.2 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.1.1 | 2021-08-12 | [5367](https://github.com/airbytehq/airbyte/pull/5367) | Migrated to CI Sandbox, refactorred code structure for future support | -| 0.1.0 | 2021-07-07 | [4358](https://github.com/airbytehq/airbyte/pull/4358) | New source | - +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------- | +| 0.1.3 | 2022-07-07 | [14437](https://github.com/airbytehq/airbyte/pull/14437) | 🎉 Source Dixa: bump version 0.1.3 | +| 0.1.2 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | +| 0.1.1 | 2021-08-12 | [5367](https://github.com/airbytehq/airbyte/pull/5367) | Migrated to CI Sandbox, refactorred code structure for future support | +| 0.1.0 | 2021-07-07 | [4358](https://github.com/airbytehq/airbyte/pull/4358) | New source | diff --git a/docs/integrations/sources/dv360.md b/docs/integrations/sources/dv-360.md similarity index 67% rename from docs/integrations/sources/dv360.md rename to docs/integrations/sources/dv-360.md index c1f3b6e76d7c..9e4341f1d847 100644 --- a/docs/integrations/sources/dv360.md +++ b/docs/integrations/sources/dv-360.md @@ -8,21 +8,20 @@ DoubleClick Bid Manager API `v1.1` is the latest available and recommended versi ## Features -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental Sync | Yes | - +| Feature | Supported? | +| :---------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental Sync | Yes | ## Supported Tables This source is capable of syncing the following tables and their data: -* Audience Composition -* Floodlight -* Reach -* Standard -* Unique Reach Audience +- Audience Composition +- Floodlight +- Reach +- Standard +- Unique Reach Audience **Note**: It is recommended to first build the desired report in the UI to avoid any errors, since there are several limilations and requirements pertaining to reporting types, filters, dimensions, and metrics (such as valid combinations of metrics and dimensions). @@ -33,7 +32,7 @@ Available filters and metrics are provided in this [page](https://developers.goo ## Getting Started \(Airbyte-Cloud\) 1. Click `Authenticate your Display & Video 360 account` to sign in with Google and authorize your account. -2. Get the partner ID for your account. +2. Get the partner ID for your account. 3. Fill out a start date, and optionally, an end date and filters (check the [Queries documentation](https://developers.google.com/bid-manager/v1.1/queries)) . 4. You're done. @@ -43,28 +42,28 @@ Available filters and metrics are provided in this [page](https://developers.goo You can use the [setup tool](https://console.developers.google.com/start/api?id=doubleclickbidmanager&credential=client_key) to create credentials and enable the DBM API in the Google API Console. -* access_token -* refresh_token -* token_uri -* client_id -* client_secret -* start_date -* end_date -* partner_id -* filters +- access_token +- refresh_token +- token_uri +- client_id +- client_secret +- start_date +- end_date +- partner_id +- filters #### Setup guide -* [Getting Started guide](https://developers.google.com/bid-manager/guides/getting-started-api) +- [Getting Started guide](https://developers.google.com/bid-manager/guides/getting-started-api) -* [Using OAuth 2.0 to Access Google APIs](https://developers.google.com/identity/protocols/oauth2/web-server#enable-apis) +- [Using OAuth 2.0 to Access Google APIs](https://developers.google.com/identity/protocols/oauth2/web-server#enable-apis) ## Rate Limiting & Performance Considerations \(Airbyte Open Source\) This source is constrained by the limits set by the DBM API. You can read more about those limits in the [Display & Video 360 docs](https://developers.google.com/bid-manager/guides/scheduled-reports/best-practices#consider_reporting_quotas). -## CHANGELOG +## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.1 | 2022--| [11828](https://github.com/airbytehq/airbyte/pull/11828) | Release Native Display & Video 360 Connector | \ No newline at end of file +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------- | +| 0.1.0 | 2022-09-28 | [11828](https://github.com/airbytehq/airbyte/pull/11828) | Release Native Display & Video 360 Connector | diff --git a/docs/integrations/sources/exchangeratesapi.md b/docs/integrations/sources/exchange-rates.md similarity index 85% rename from docs/integrations/sources/exchangeratesapi.md rename to docs/integrations/sources/exchange-rates.md index 6b4f79a305a1..aeaefd189861 100644 --- a/docs/integrations/sources/exchangeratesapi.md +++ b/docs/integrations/sources/exchange-rates.md @@ -12,8 +12,8 @@ It contains one stream: `exchange_rates` Each record in the stream contains many fields: -* The date of the record -* One field for every supported [currency](https://www.ecb.europa.eu/stats/policy_and_exchange_rates/euro_reference_exchange_rates/html/index.en.html) which contain the value of that currency on that date. +- The date of the record +- One field for every supported [currency](https://www.ecb.europa.eu/stats/policy_and_exchange_rates/euro_reference_exchange_rates/html/index.en.html) which contain the value of that currency on that date. #### Data type mapping @@ -21,17 +21,17 @@ Currencies are `number` and the date is a `string`. #### Features -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | Yes | -| Namespaces | No | +| Feature | Supported? | +| :------------------------ | :--------- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | Yes | +| Namespaces | No | ### Getting started ### Requirements -* API Access Key +- API Access Key ### Setup guide @@ -44,14 +44,14 @@ If you have `free` subscription plan \(you may check it [here](https://manage.ex ## Changelog -| Version | Date | Pull Request | Subject | -|:--------| :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------- | -| 1.2.7 | 2022-10-31 | [18726](https://github.com/airbytehq/airbyte/pull/18726) | Fix handling error during check connection | -| 1.2.6 | 2022-08-23 | [15884](https://github.com/airbytehq/airbyte/pull/15884) | Migrated to new API Layer endpoint | -| 0.2.6 | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Update connector to use a `spec.yaml` | -| 0.2.5 | 2021-11-12 | [7936](https://github.com/airbytehq/airbyte/pull/7936) | Add ignore_weekends boolean option | -| 0.2.4 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.2.3 | 2021-06-06 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for kubernetes support | -| 0.2.2 | 2021-05-28 | [3677](https://github.com/airbytehq/airbyte/pull/3677) | Adding clearer error message when a currency isn't supported. access_key field in spec.json was marked as sensitive | -| 0.2.0 | 2021-05-26 | [3566](https://github.com/airbytehq/airbyte/pull/3566) | Move from `api.ratesapi.io/` to `api.exchangeratesapi.io/`. Add required field `access_key` to `config.json`. | -| 0.1.0 | 2021-04-19 | [2942](https://github.com/airbytehq/airbyte/pull/2942) | Implement Exchange API using the CDK | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------ | +| 1.2.7 | 2022-10-31 | [18726](https://github.com/airbytehq/airbyte/pull/18726) | Fix handling error during check connection | +| 1.2.6 | 2022-08-23 | [15884](https://github.com/airbytehq/airbyte/pull/15884) | Migrated to new API Layer endpoint | +| 0.2.6 | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Update connector to use a `spec.yaml` | +| 0.2.5 | 2021-11-12 | [7936](https://github.com/airbytehq/airbyte/pull/7936) | Add ignore_weekends boolean option | +| 0.2.4 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | +| 0.2.3 | 2021-06-06 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for kubernetes support | +| 0.2.2 | 2021-05-28 | [3677](https://github.com/airbytehq/airbyte/pull/3677) | Adding clearer error message when a currency isn't supported. access_key field in spec.json was marked as sensitive | +| 0.2.0 | 2021-05-26 | [3566](https://github.com/airbytehq/airbyte/pull/3566) | Move from `api.ratesapi.io/` to `api.exchangeratesapi.io/`. Add required field `access_key` to `config.json`. | +| 0.1.0 | 2021-04-19 | [2942](https://github.com/airbytehq/airbyte/pull/2942) | Implement Exchange API using the CDK | diff --git a/docs/integrations/sources/file.md b/docs/integrations/sources/file.md index 780070d11014..8b5d56626cd2 100644 --- a/docs/integrations/sources/file.md +++ b/docs/integrations/sources/file.md @@ -54,12 +54,15 @@ This source produces a single table for the target file as it replicates only on ## Getting Started + **For Airbyte Cloud:** Setup through Airbyte Cloud will be exactly the same as the open-source setup, except for the fact that local files are disabled. + + **For Airbyte Open Source:** 1. Once the File Source is selected, you should define both the storage provider along its URL and format of the file. @@ -68,20 +71,20 @@ Setup through Airbyte Cloud will be exactly the same as the open-source setup, e #### Provider Specific Information -* In case of Google Drive, it is necesary to use the Download URL, the format for that is `https://drive.google.com/uc?export=download&id=[DRIVE_FILE_ID]` where `[DRIVE_FILE_ID]` is the string found in the Share URL here `https://drive.google.com/file/d/[DRIVE_FILE_ID]/view?usp=sharing` -* In case of GCS, it is necessary to provide the content of the service account keyfile to access private buckets. See settings of [BigQuery Destination](../destinations/bigquery.md) -* In case of AWS S3, the pair of `aws_access_key_id` and `aws_secret_access_key` is necessary to access private S3 buckets. -* In case of AzBlob, it is necessary to provide the `storage_account` in which the blob you want to access resides. Either `sas_token` [(info)](https://docs.microsoft.com/en-us/azure/storage/blobs/sas-service-create?tabs=dotnet) or `shared_key` [(info)](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?tabs=azure-portal) is necessary to access private blobs. +- In case of Google Drive, it is necesary to use the Download URL, the format for that is `https://drive.google.com/uc?export=download&id=[DRIVE_FILE_ID]` where `[DRIVE_FILE_ID]` is the string found in the Share URL here `https://drive.google.com/file/d/[DRIVE_FILE_ID]/view?usp=sharing` +- In case of GCS, it is necessary to provide the content of the service account keyfile to access private buckets. See settings of [BigQuery Destination](../destinations/bigquery.md) +- In case of AWS S3, the pair of `aws_access_key_id` and `aws_secret_access_key` is necessary to access private S3 buckets. +- In case of AzBlob, it is necessary to provide the `storage_account` in which the blob you want to access resides. Either `sas_token` [(info)](https://docs.microsoft.com/en-us/azure/storage/blobs/sas-service-create?tabs=dotnet) or `shared_key` [(info)](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?tabs=azure-portal) is necessary to access private blobs. ### Reader Options -The Reader in charge of loading the file format is currently based on [Pandas IO Tools](https://pandas.pydata.org/pandas-docs/stable/user\_guide/io.html). It is possible to customize how to load the file into a Pandas DataFrame as part of this Source Connector. This is doable in the `reader_options` that should be in JSON format and depends on the chosen file format. See pandas' documentation, depending on the format: +The Reader in charge of loading the file format is currently based on [Pandas IO Tools](https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html). It is possible to customize how to load the file into a Pandas DataFrame as part of this Source Connector. This is doable in the `reader_options` that should be in JSON format and depends on the chosen file format. See pandas' documentation, depending on the format: -For example, if the format `CSV` is selected, then options from the [read\_csv](https://pandas.pydata.org/pandas-docs/stable/user\_guide/io.html#io-read-csv-table) functions are available. +For example, if the format `CSV` is selected, then options from the [read_csv](https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#io-read-csv-table) functions are available. -* It is therefore possible to customize the `delimiter` (or `sep`) to in case of tab separated files. -* Header line can be ignored with `header=0` and customized with `names` -* etc +- It is therefore possible to customize the `delimiter` (or `sep`) to in case of tab separated files. +- Header line can be ignored with `header=0` and customized with `names` +- etc We would therefore provide in the `reader_options` the following json: @@ -89,7 +92,7 @@ We would therefore provide in the `reader_options` the following json: { "sep" : "\t", "header" : 0, "names": "column1, column2"} ``` -In case you select `JSON` format, then options from the [read\_json](https://pandas.pydata.org/pandas-docs/stable/user\_guide/io.html#io-json-reader) reader are available. +In case you select `JSON` format, then options from the [read_json](https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#io-json-reader) reader are available. For example, you can use the `{"orient" : "records"}` to change how orientation of data is loaded (if data is `[{column -> value}, … , {column -> value}]`) @@ -103,19 +106,19 @@ Normally, Airbyte tries to infer the data type from the source, but you can use Here are a list of examples of possible file inputs: -| Dataset Name | Storage | URL | Reader Impl | Service Account | Description | -| ------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | ----------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| epidemiology | HTTPS | [https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv](https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv) | | | [COVID-19 Public dataset](https://console.cloud.google.com/marketplace/product/bigquery-public-datasets/covid19-public-data-program?filter=solution-type:dataset\&id=7d6cc408-53c8-4485-a187-b8cb9a5c0b56) on BigQuery | -| hr\_and\_financials | GCS | gs://airbyte-vault/financial.csv | smart\_open or gcfs | {"type": "service\_account", "private\_key\_id": "XXXXXXXX", ...} | data from a private bucket, a service account is necessary | -| landsat\_index | GCS | gcp-public-data-landsat/index.csv.gz | smart\_open | | Using smart\_open, we don't need to specify the compression (note the gs:// is optional too, same for other providers) | +| Dataset Name | Storage | URL | Reader Impl | Service Account | Description | +| ----------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------ | -------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| epidemiology | HTTPS | [https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv](https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv) | | | [COVID-19 Public dataset](https://console.cloud.google.com/marketplace/product/bigquery-public-datasets/covid19-public-data-program?filter=solution-type:dataset&id=7d6cc408-53c8-4485-a187-b8cb9a5c0b56) on BigQuery | +| hr_and_financials | GCS | gs://airbyte-vault/financial.csv | smart_open or gcfs | {"type": "service_account", "private_key_id": "XXXXXXXX", ...} | data from a private bucket, a service account is necessary | +| landsat_index | GCS | gcp-public-data-landsat/index.csv.gz | smart_open | | Using smart_open, we don't need to specify the compression (note the gs:// is optional too, same for other providers) | Examples with reader options: -| Dataset Name | Storage | URL | Reader Impl | Reader Options | Description | -| -------------- | ------- | ----------------------------------------------- | ----------- | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | -| landsat\_index | GCS | gs://gcp-public-data-landsat/index.csv.gz | GCFS | {"compression": "gzip"} | Additional reader options to specify a compression option to `read_csv` | -| GDELT | S3 | s3://gdelt-open-data/events/20190914.export.csv | | {"sep": "\t", "header": null} | Here is TSV data separated by tabs without header row from [AWS Open Data](https://registry.opendata.aws/gdelt/) | -| server\_logs | local | /local/logs.log | | {"sep": ";"} | After making sure a local text file exists at `/tmp/airbyte_local/logs.log` with logs file from some server that are delimited by ';' delimiters | +| Dataset Name | Storage | URL | Reader Impl | Reader Options | Description | +| ------------- | ------- | ----------------------------------------------- | ----------- | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | +| landsat_index | GCS | gs://gcp-public-data-landsat/index.csv.gz | GCFS | {"compression": "gzip"} | Additional reader options to specify a compression option to `read_csv` | +| GDELT | S3 | s3://gdelt-open-data/events/20190914.export.csv | | {"sep": "\t", "header": null} | Here is TSV data separated by tabs without header row from [AWS Open Data](https://registry.opendata.aws/gdelt/) | +| server_logs | local | /local/logs.log | | {"sep": ";"} | After making sure a local text file exists at `/tmp/airbyte_local/logs.log` with logs file from some server that are delimited by ';' delimiters | Example for SFTP: @@ -127,15 +130,16 @@ Please see (or add) more at `airbyte-integrations/connectors/source-file/integra ## Performance Considerations and Notes -In order to read large files from a remote location, this connector uses the [smart\_open](https://pypi.org/project/smart-open/) library. However, it is possible to switch to either [GCSFS](https://gcsfs.readthedocs.io/en/latest/) or [S3FS](https://s3fs.readthedocs.io/en/latest/) implementations as it is natively supported by the `pandas` library. This choice is made possible through the optional `reader_impl` parameter. +In order to read large files from a remote location, this connector uses the [smart_open](https://pypi.org/project/smart-open/) library. However, it is possible to switch to either [GCSFS](https://gcsfs.readthedocs.io/en/latest/) or [S3FS](https://s3fs.readthedocs.io/en/latest/) implementations as it is natively supported by the `pandas` library. This choice is made possible through the optional `reader_impl` parameter. -* Note that for local filesystem, the file probably have to be stored somewhere in the `/tmp/airbyte_local` folder with the same limitations as the [CSV Destination](../destinations/local-csv.md) so the `URL` should also starts with `/local/`. -* The JSON implementation needs to be tweaked in order to produce more complex catalog and is still in an experimental state: Simple JSON schemas should work at this point but may not be well handled when there are multiple layers of nesting. +- Note that for local filesystem, the file probably have to be stored somewhere in the `/tmp/airbyte_local` folder with the same limitations as the [CSV Destination](../destinations/local-csv.md) so the `URL` should also starts with `/local/`. +- The JSON implementation needs to be tweaked in order to produce more complex catalog and is still in an experimental state: Simple JSON schemas should work at this point but may not be well handled when there are multiple layers of nesting. ## Changelog | Version | Date | Pull Request | Subject | -| ------- | ---------- | -------------------------------------------------------- | -------------------------------------------------------- | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------- | +| 0.2.31 | 2022-11-17 | [19567](https://github.com/airbytehq/airbyte/pull/19567) | Source File: bump 0.2.31 | | 0.2.30 | 2022-11-10 | [19222](https://github.com/airbytehq/airbyte/pull/19222) | Use AirbyteConnectionStatus for "check" command | | 0.2.29 | 2022-11-08 | [18587](https://github.com/airbytehq/airbyte/pull/18587) | Fix pandas read_csv header none issue. | | 0.2.28 | 2022-10-27 | [18428](https://github.com/airbytehq/airbyte/pull/18428) | Added retry logic for `Connection reset error - 104` | @@ -159,7 +163,7 @@ In order to read large files from a remote location, this connector uses the [sm | 0.2.7 | 2021-10-28 | [7387](https://github.com/airbytehq/airbyte/pull/7387) | Migrate source to CDK structure, add SAT testing. | | 0.2.6 | 2021-08-26 | [5613](https://github.com/airbytehq/airbyte/pull/5613) | Add support to xlsb format | | 0.2.5 | 2021-07-26 | [4953](https://github.com/airbytehq/airbyte/pull/4953) | Allow non-default port for SFTP type | -| 0.2.4 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add AIRBYTE\_ENTRYPOINT for Kubernetes support | +| 0.2.4 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add AIRBYTE_ENTRYPOINT for Kubernetes support | | 0.2.3 | 2021-06-01 | [3771](https://github.com/airbytehq/airbyte/pull/3771) | Add Azure Storage Blob Files option | | 0.2.2 | 2021-04-16 | [2883](https://github.com/airbytehq/airbyte/pull/2883) | Fix CSV discovery memory consumption | | 0.2.1 | 2021-04-03 | [2726](https://github.com/airbytehq/airbyte/pull/2726) | Fix base connector versioning | diff --git a/docs/integrations/sources/gridly.md b/docs/integrations/sources/gridly.md index 1f98968588b6..64cbd1a8622f 100644 --- a/docs/integrations/sources/gridly.md +++ b/docs/integrations/sources/gridly.md @@ -9,27 +9,31 @@ A Gridly account. ## Setup guide ### Get api Key + 1. To quickly get your API key, access your Gridly Dashboard, then select a Grid View and you can find the key in API quick start right panel. -![img.png](../../.gitbook/assets/gridly_api_key1.png) + ![img.png](../../.gitbook/assets/gridly_api_key1.png) 2. Owner and Administrators can go to Settings/API keys to create company-level API keys with scoped privileges and accesses. -![img.png](../../.gitbook/assets/gridly_api_key2.png) + ![img.png](../../.gitbook/assets/gridly_api_key2.png) ### Get grid id + The grid id is available in the url. Gridly support version control, by default the `grid id` is the same to the `branch id` when `Master` branch is selected. For fetching records on other branches, use `branch id` instead. ![img.png](../../.gitbook/assets/gridly_grid_id.png) - ## Supported sync modes -| Feature | Supported? | -| :---------------------------- |:-----------| -| Full Refresh Sync | Yes | -| Incremental Sync | No | - +| Feature | Supported? | +| :---------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental Sync | No | ## Supported Streams -* [Records](https://www.gridly.com/docs/api/#record) +- [Records](https://www.gridly.com/docs/api/#record) ## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------- | +| 0.1.1 | 2022-12-08 | [20048](https://github.com/airbytehq/airbyte/pull/20048) | Source Gridly: add icon and make grid_id parameter required | diff --git a/docs/integrations/sources/hellobaton.md b/docs/integrations/sources/hellobaton.md index 16d8d401d884..950ac1c0060b 100644 --- a/docs/integrations/sources/hellobaton.md +++ b/docs/integrations/sources/hellobaton.md @@ -6,37 +6,37 @@ This source can sync data from the [baton API](https://app.hellobaton.com/api/re ## This Source Supports the Following Streams -* activity -* companies -* milestones -* phases -* project_attachments -* projects -* task_attachemnts -* tasks -* templates -* time_entries -* users +- activity +- companies +- milestones +- phases +- project_attachments +- projects +- task_attachemnts +- tasks +- templates +- time_entries +- users Baton adds new streams fairly regularly please submit an issue or PR if this project doesn't support required streams for your use case. ### Data type mapping | Integration Type | Airbyte Type | Notes | -| :--- | :--- | :--- | -| `string` | `string` | | -| `integer` | `integer` | | -| `number` | `number` | | -| `array` | `array` | | -| `object` | `object` | | +| :--------------- | :----------- | :---- | +| `string` | `string` | | +| `integer` | `integer` | | +| `number` | `number` | | +| `array` | `array` | | +| `object` | `object` | | ### Features -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental Sync | No | | -| Namespaces | No | | +| Feature | Supported?\(Yes/No\) | Notes | +| :---------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental Sync | No | | +| Namespaces | No | | ### Performance considerations @@ -46,5 +46,11 @@ The connector is rate limited at 1000 requests per minute per api key. If you fi ### Requirements -* Baton account -* Baton api key \ No newline at end of file +- Baton account +- Baton api key + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :----------------------------------------------------- | :------------------------ | +| 0.1.0 | 2022-01-14 | [8461](https://github.com/airbytehq/airbyte/pull/8461) | 🎉 New Source: Hellobaton | diff --git a/docs/integrations/sources/kustomer-singer.md b/docs/integrations/sources/kustomer-singer.md new file mode 100644 index 000000000000..5fc12662e4e9 --- /dev/null +++ b/docs/integrations/sources/kustomer-singer.md @@ -0,0 +1,47 @@ +# Kustomer + +## Sync overview + +The Kustomer source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. + +This source can sync data for the [Kustomer API](https://developer.kustomer.com/kustomer-api-docs). + +This Source Connector is based on a [Singer tap](https://github.com/singer-io/tap-kustomer). + +### Output schema + +This Source is capable of syncing the following core Streams: + +- [Conversations](https://developer.kustomer.com/kustomer-api-docs/reference/conversations) +- [Customers](https://developer.kustomer.com/kustomer-api-docs/reference/customers) +- [KObjects](https://developer.kustomer.com/kustomer-api-docs/reference/kobjects-custom-objects) +- [Messages](https://developer.kustomer.com/kustomer-api-docs/reference/messages) +- [Notes](https://developer.kustomer.com/kustomer-api-docs/reference/notes) +- [Shortcuts](https://developer.kustomer.com/kustomer-api-docs/reference/shortcuts) +- [Tags](https://developer.kustomer.com/kustomer-api-docs/reference/tags-knowledge-base) +- [Teams](https://developer.kustomer.com/kustomer-api-docs/reference/teams) +- [Users](https://developer.kustomer.com/kustomer-api-docs/reference/users) + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :------------------------ | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | +| Namespaces | No | | + +### Performance considerations + +Kustomer has some [rate limit restrictions](https://developer.kustomer.com/kustomer-api-docs/reference/rate-limiting). + +## Requirements + +- **Kustomer API token**. See the [Kustomer docs](https://help.kustomer.com/api-keys-SJs5YTIWX) for information on how to obtain an API token. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :----------------------------------------------------- | :-------------------------------------------------------------------------------- | +| 0.1.2 | 2021-12-25 | [8578](https://github.com/airbytehq/airbyte/pull/8578) | Update fields in source-connectors specifications | +| 0.1.1 | 2021-12-22 | [8738](https://github.com/airbytehq/airbyte/pull/8738) | Deleted `user-agent`, `date_window_size`, `page_size_limit` from `spec.json` file | +| 0.1.0 | 2021-07-22 | [4550](https://github.com/airbytehq/airbyte/pull/4550) | Add Kustomer Source Connector | diff --git a/docs/integrations/sources/kustomer.md b/docs/integrations/sources/kustomer.md deleted file mode 100644 index cbd0471eebbe..000000000000 --- a/docs/integrations/sources/kustomer.md +++ /dev/null @@ -1,47 +0,0 @@ -# Kustomer - -## Sync overview - -The Kustomer source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. - -This source can sync data for the [Kustomer API](https://developer.kustomer.com/kustomer-api-docs). - -This Source Connector is based on a [Singer tap](https://github.com/singer-io/tap-kustomer). - -### Output schema - -This Source is capable of syncing the following core Streams: - -* [Conversations](https://developer.kustomer.com/kustomer-api-docs/reference/conversations) -* [Customers](https://developer.kustomer.com/kustomer-api-docs/reference/customers) -* [KObjects](https://developer.kustomer.com/kustomer-api-docs/reference/kobjects-custom-objects) -* [Messages](https://developer.kustomer.com/kustomer-api-docs/reference/messages) -* [Notes](https://developer.kustomer.com/kustomer-api-docs/reference/notes) -* [Shortcuts](https://developer.kustomer.com/kustomer-api-docs/reference/shortcuts) -* [Tags](https://developer.kustomer.com/kustomer-api-docs/reference/tags-knowledge-base) -* [Teams](https://developer.kustomer.com/kustomer-api-docs/reference/teams) -* [Users](https://developer.kustomer.com/kustomer-api-docs/reference/users) - -### Features - -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental - Append Sync | Yes | | -| Namespaces | No | | - -### Performance considerations - -Kustomer has some [rate limit restrictions](https://developer.kustomer.com/kustomer-api-docs/reference/rate-limiting). - -## Requirements - -* **Kustomer API token**. See the [Kustomer docs](https://help.kustomer.com/api-keys-SJs5YTIWX) for information on how to obtain an API token. - -## Changelog - -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.1 | 2021-12-13 | [8738](https://github.com/airbytehq/airbyte/pull/8738) | Deleted `user-agent`, `date_window_size`, `page_size_limit` from `spec.json` file | -| 0.1.0 | 2021-07-22 | [4550](https://github.com/airbytehq/airbyte/pull/4550) | Add Kustomer Source Connector | - diff --git a/docs/integrations/sources/mailersend.md b/docs/integrations/sources/mailersend.md index 5d4601745480..8c99fb4a7046 100644 --- a/docs/integrations/sources/mailersend.md +++ b/docs/integrations/sources/mailersend.md @@ -23,5 +23,6 @@ MailerSend has a default [rate limit](https://developers.mailersend.com/general. ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--- | :----------- | :------ | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------- | +| 0.1.0 | 2022-11-13 | [18669](https://github.com/airbytehq/airbyte/pull/18669) | 🎉 New Source: Mailersend [low-code CDK] | diff --git a/docs/integrations/sources/microsoft-dataverse.md b/docs/integrations/sources/microsoft-dataverse.md index 7aca712b3328..3a17d967e7cf 100644 --- a/docs/integrations/sources/microsoft-dataverse.md +++ b/docs/integrations/sources/microsoft-dataverse.md @@ -14,7 +14,7 @@ This source will automatically discover the schema of the Entities of your Datav ### Data type mapping | Integration Type | Airbyte Type | Notes | -|:-------------------|:--------------------------|:----------------------| +| :----------------- | :------------------------ | :-------------------- | | `String` | `string` | | | `UniqueIdentifier` | `string` | | | `DateTime` | `timestamp with timezone` | | @@ -33,7 +33,7 @@ Other types are defined as `string`. ### Features | Feature | Supported?\(Yes/No\) | Notes | -|:------------------------------|:---------------------|:-----------------------------------------------------------| +| :---------------------------- | :------------------- | :--------------------------------------------------------- | | Full Refresh Sync | Yes | | | Incremental Sync | Yes | | | CDC | Yes | Not all entities support it. Deleted data only have the ID | @@ -45,9 +45,9 @@ Other types are defined as `string`. ### Requirements -* Application \(client\) ID -* Directory \(tenant\) ID -* Client secrets +- Application \(client\) ID +- Directory \(tenant\) ID +- Client secrets ### Setup guide @@ -57,8 +57,8 @@ https://learn.microsoft.com/en-us/power-apps/developer/data-platform/authenticat The procedure to generate the credentials and setup the necessary permissions is well described in this post from Magnetism blog: https://blog.magnetismsolutions.com/blog/paulnieuwelaar/2021/9/21/setting-up-an-application-user-in-dynamics-365 - ## CHANGELOG -| Version | Date | Pull Request | Subject | -|:--------|:-----|:-------------|:--------| +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------- | +| 0.1.0 | 2022-11-14 | [18646](https://github.com/airbytehq/airbyte/pull/18646) | 🎉 New Source: Microsoft Dataverse [python cdk] | diff --git a/docs/integrations/sources/n8n.md b/docs/integrations/sources/n8n.md index d4e1404d68c5..d212fdbcb486 100644 --- a/docs/integrations/sources/n8n.md +++ b/docs/integrations/sources/n8n.md @@ -27,5 +27,6 @@ You need a n8n instance or use cloud version ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--- | :----------- | :------ | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------------- | +| 0.1.0 | 2022-11-08 | [18745](https://github.com/airbytehq/airbyte/pull/18745) | 🎉 New Source: N8n [low-code cdk] | diff --git a/docs/integrations/sources/pardot.md b/docs/integrations/sources/pardot.md index 2a02ae782d53..f8f304797a39 100644 --- a/docs/integrations/sources/pardot.md +++ b/docs/integrations/sources/pardot.md @@ -4,6 +4,7 @@ The Airbyte Source for [Salesforce Pardot](https://www.pardot.com/) ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :----------------------------------------------------- | :----------------- | -| 0.1.0 | 2021-11-09 | [7091](https://github.com/airbytehq/airbyte/pull/7091) | New Source: Pardot | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------- | +| 0.1.1 | 2022-12-16 | [19618](https://github.com/airbytehq/airbyte/pull/19618) | Fix `visitors` stream | +| 0.1.0 | 2021-11-19 | [7091](https://github.com/airbytehq/airbyte/pull/7091) | 🎉 New Source: Pardot | diff --git a/docs/integrations/sources/persistiq.md b/docs/integrations/sources/persistiq.md index fa466116718e..40e6ee64ea9a 100644 --- a/docs/integrations/sources/persistiq.md +++ b/docs/integrations/sources/persistiq.md @@ -6,22 +6,21 @@ The PersistIq source supports Full Refresh syncs only. This source syncs data for the [PersistIq API](https://apidocs.persistiq.com/#introduction). - ### Output schema This Source is capable of syncing the following streams: -* [Users](https://apidocs.persistiq.com/#users) -* [Leads](https://apidocs.persistiq.com/#leads) -* [Campaigns](https://apidocs.persistiq.com/#campaigns) +- [Users](https://apidocs.persistiq.com/#users) +- [Leads](https://apidocs.persistiq.com/#leads) +- [Campaigns](https://apidocs.persistiq.com/#campaigns) ### Features -| Feature | Supported?\(Yes/No\) -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | No | -| Namespaces | No | +| Feature | Supported?\(Yes/No\) | +| :------------------------ | :------------------- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | No | +| Namespaces | No | ### Performance considerations @@ -31,8 +30,14 @@ The PersistIq connector should not run into PersistIq API limitations under norm ### Requirements -* PersistIq API Key +- PersistIq API Key ### Setup guide Please read [How to find your API key](https://apidocs.persistiq.com/#introduction). + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :----------------------------------------------------- | :----------------------- | +| 0.1.0 | 2022-01-21 | [9515](https://github.com/airbytehq/airbyte/pull/9515) | 🎉 New Source: PersistIq | diff --git a/docs/integrations/sources/pivotal-tracker.md b/docs/integrations/sources/pivotal-tracker.md index 3b823e8c4754..d533f6c6b061 100644 --- a/docs/integrations/sources/pivotal-tracker.md +++ b/docs/integrations/sources/pivotal-tracker.md @@ -3,35 +3,36 @@ ## Overview The Pivotal Tracker source supports Full Refresh syncs. It supports pulling from : -* Activity -* Epics -* Labels -* Project Membership -* Projects -* Releases -* Stories + +- Activity +- Epics +- Labels +- Project Membership +- Projects +- Releases +- Stories ### Output schema Output streams: -* [Activity](https://www.pivotaltracker.com/help/api/rest/v5#Activity) -* [Epics](https://www.pivotaltracker.com/help/api/rest/v5#Epics) -* [Labels](https://www.pivotaltracker.com/help/api/rest/v5#Labels) -* [Project Membership](https://www.pivotaltracker.com/help/api/rest/v5#Project_Memberships) -* [Projects](https://www.pivotaltracker.com/help/api/rest/v5#Projects) -* [Releases](https://www.pivotaltracker.com/help/api/rest/v5#Releases) -* [Stories](https://www.pivotaltracker.com/help/api/rest/v5#Stories) +- [Activity](https://www.pivotaltracker.com/help/api/rest/v5#Activity) +- [Epics](https://www.pivotaltracker.com/help/api/rest/v5#Epics) +- [Labels](https://www.pivotaltracker.com/help/api/rest/v5#Labels) +- [Project Membership](https://www.pivotaltracker.com/help/api/rest/v5#Project_Memberships) +- [Projects](https://www.pivotaltracker.com/help/api/rest/v5#Projects) +- [Releases](https://www.pivotaltracker.com/help/api/rest/v5#Releases) +- [Stories](https://www.pivotaltracker.com/help/api/rest/v5#Stories) ### Features -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | Coming soon | +| Feature | Supported? | +| :---------------------------- | :---------- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | Coming soon | | Replicate Incremental Deletes | Coming soon | -| SSL connection | Yes | -| Namespaces | No | +| SSL connection | Yes | +| Namespaces | No | ### Performance considerations @@ -41,13 +42,15 @@ The Pivotal Trakcer connector should not run into Stripe API limitations under n ### Requirements -* Pivotal Trakcer API Token +- Pivotal Trakcer API Token ### Setup guide to create the API Token Access your profile [here](https://www.pivotaltracker.com/profile) go down and click in **Create New Token**. Use this to pull data from Pivotal Tracker. -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.0 | 2022-04-04 | [11060](https://github.com/airbytehq/airbyte/pull/11060) | Initial Release | +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------- | +| 0.1.0 | 2022-04-04 | [11060](https://github.com/airbytehq/airbyte/pull/11060) | Initial Release | diff --git a/docs/integrations/sources/plaid.md b/docs/integrations/sources/plaid.md index f9cc856147a1..889ffb39ea84 100644 --- a/docs/integrations/sources/plaid.md +++ b/docs/integrations/sources/plaid.md @@ -66,6 +66,8 @@ This guide will walk through how to create the credentials you need to run this - We should now have everything we need to configure this source in the UI. +## Changelog + | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------ | | 0.3.2 | 2022-08-02 | [15231](https://github.com/airbytehq/airbyte/pull/15231) | Added min_last_updated_datetime support for Capital One items | diff --git a/docs/integrations/sources/qonto.md b/docs/integrations/sources/qonto.md new file mode 100644 index 000000000000..18536cfaf890 --- /dev/null +++ b/docs/integrations/sources/qonto.md @@ -0,0 +1,9 @@ +# Qonto + +The Airbyte Source for [Qonto](https://qonto.com) + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------------- | +| 0.1.0 | 2022-11-14 | [17452](https://github.com/airbytehq/airbyte/pull/17452) | 🎉 New Source: Qonto [python cdk] | diff --git a/docs/integrations/sources/quickbooks-singer.md b/docs/integrations/sources/quickbooks-singer.md new file mode 100644 index 000000000000..540ea748cc73 --- /dev/null +++ b/docs/integrations/sources/quickbooks-singer.md @@ -0,0 +1,84 @@ +# QuickBooks + +## Overview + +The QuickBooks source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. + +This source wraps the [Singer QuickBooks Tap](https://github.com/singer-io/tap-quickbooks). + +### Output schema + +This Source is capable of syncing the following [Streams](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/most-commonly-used/account): + +- [Accounts](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/account) +- [BillPayments](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/billpayment) +- [Budgets](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/budget) +- [Bills](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/bill) +- [Classes](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/class) +- [CreditMemos](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/creditmemo) +- [Customers](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/customer) +- [Departments](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/department) +- [Deposits](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/deposit) +- [Employees](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/employee) +- [Estimates](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/estimate) +- [Invoices](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/invoice) +- [Items](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/item) +- [JournalEntries](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/journalentry) +- [Payments](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/payment) +- [PaymentMethods](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/paymentmethod) +- [Purchases](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/purchase) +- [PurchaseOrders](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/purchaseorder) +- [RefundReceipts](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/refundreceipt) +- [SalesReceipts](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/salesreceipt) +- [TaxAgencies](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/taxagency) +- [TaxCodes](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/taxcode) +- [TaxRates](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/taxrate) +- [Terms](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/term) +- [TimeActivities](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/timeactivity) +- [Transfers](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/transfer) +- [VendorCredits](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/vendorcredit) +- [Vendors](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/vendor) + +### Data type mapping + +| Integration Type | Airbyte Type | Notes | +| :--------------- | :----------- | :---- | +| `string` | `string` | | +| `number` | `number` | | +| `array` | `array` | | +| `object` | `object` | | + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :---------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental Sync | Yes | | +| SSL connection | Yes | | +| Namespaces | No | | + +## Getting started + +1. Create an [Intuit Developer account](https://developer.intuit.com/app/developer/qbo/docs/get-started) +2. Create an app +3. Obtain credentials + +### Requirements + +- Client ID +- Client Secret +- Realm ID +- Refresh token + +The easiest way to get these credentials is by using Quickbook's [OAuth 2.0 playground](https://developer.intuit.com/app/developer/qbo/docs/develop/authentication-and-authorization/oauth-2.0-playground) + +**Important note:** The refresh token expires every 100 days. You will need to manually revisit the Oauth playground to obtain a refresh token every 100 days, or your syncs will expire. We plan on offering full Oauth support soon so you don't need to redo this process manually. + +## CHANGELOG + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------- | +| `0.1.5` | 2022-02-17 | [10346](https://github.com/airbytehq/airbyte/pull/10346) | Update label `Quickbooks` -> `QuickBooks` | +| `0.1.4` | 2021-12-20 | [8960](https://github.com/airbytehq/airbyte/pull/8960) | Update connector fields title/description | +| `0.1.3` | 2021-08-10 | [4986](https://github.com/airbytehq/airbyte/pull/4986) | Using number data type for decimal fields instead string | +| `0.1.2` | 2021-07-06 | [4539](https://github.com/airbytehq/airbyte/pull/4539) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | diff --git a/docs/integrations/sources/quickbooks.md b/docs/integrations/sources/quickbooks.md deleted file mode 100644 index 863481bec6c2..000000000000 --- a/docs/integrations/sources/quickbooks.md +++ /dev/null @@ -1,85 +0,0 @@ -# QuickBooks - -## Overview - -The QuickBooks source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. - -This source wraps the [Singer QuickBooks Tap](https://github.com/singer-io/tap-quickbooks). - -### Output schema - -This Source is capable of syncing the following [Streams](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/most-commonly-used/account): - -* [Accounts](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/account) -* [BillPayments](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/billpayment) -* [Budgets](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/budget) -* [Bills](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/bill) -* [Classes](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/class) -* [CreditMemos](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/creditmemo) -* [Customers](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/customer) -* [Departments](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/department) -* [Deposits](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/deposit) -* [Employees](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/employee) -* [Estimates](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/estimate) -* [Invoices](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/invoice) -* [Items](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/item) -* [JournalEntries](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/journalentry) -* [Payments](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/payment) -* [PaymentMethods](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/paymentmethod) -* [Purchases](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/purchase) -* [PurchaseOrders](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/purchaseorder) -* [RefundReceipts](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/refundreceipt) -* [SalesReceipts](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/salesreceipt) -* [TaxAgencies](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/taxagency) -* [TaxCodes](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/taxcode) -* [TaxRates](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/taxrate) -* [Terms](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/term) -* [TimeActivities](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/timeactivity) -* [Transfers](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/transfer) -* [VendorCredits](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/vendorcredit) -* [Vendors](https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/vendor) - -### Data type mapping - -| Integration Type | Airbyte Type | Notes | -| :--- | :--- | :--- | -| `string` | `string` | | -| `number` | `number` | | -| `array` | `array` | | -| `object` | `object` | | - -### Features - -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental Sync | Yes | | -| SSL connection | Yes | | -| Namespaces | No | | - -## Getting started - -1. Create an [Intuit Developer account](https://developer.intuit.com/app/developer/qbo/docs/get-started) -2. Create an app -3. Obtain credentials - -### Requirements - -* Client ID -* Client Secret -* Realm ID -* Refresh token - -The easiest way to get these credentials is by using Quickbook's [OAuth 2.0 playground](https://developer.intuit.com/app/developer/qbo/docs/develop/authentication-and-authorization/oauth-2.0-playground) - -**Important note:** The refresh token expires every 100 days. You will need to manually revisit the Oauth playground to obtain a refresh token every 100 days, or your syncs will expire. We plan on offering full Oauth support soon so you don't need to redo this process manually. - -## CHANGELOG - -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| `0.1.5` | 2022-02-17 | [10346](https://github.com/airbytehq/airbyte/pull/10346) | Update label `Quickbooks` -> `QuickBooks` | -| `0.1.4` | 2021-12-20 | [8960](https://github.com/airbytehq/airbyte/pull/8960) | Update connector fields title/description | -| `0.1.3` | 2021-08-10 | [4986](https://github.com/airbytehq/airbyte/pull/4986) | Using number data type for decimal fields instead string | -| `0.1.2` | 2021-07-06 | [4539](https://github.com/airbytehq/airbyte/pull/4539) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | - diff --git a/docs/integrations/sources/surveysparrow.md b/docs/integrations/sources/survey-sparrow.md similarity index 100% rename from docs/integrations/sources/surveysparrow.md rename to docs/integrations/sources/survey-sparrow.md diff --git a/docs/integrations/sources/twiliotaskrouter.md b/docs/integrations/sources/twilio-taskrouter.md similarity index 79% rename from docs/integrations/sources/twiliotaskrouter.md rename to docs/integrations/sources/twilio-taskrouter.md index 8bd6ffa48267..4c7d8b849759 100644 --- a/docs/integrations/sources/twiliotaskrouter.md +++ b/docs/integrations/sources/twilio-taskrouter.md @@ -12,13 +12,14 @@ You can find [Taskrouter](https://console.twilio.com/develop/explore) in the Exp See [docs](https://www.twilio.com/docs/taskrouter/api) for more details. ## Setup guide + ## Step 1: Set up the Twilio Takrouter connector in Airbyte ### For Airbyte Cloud: 1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. -3. On the Set up the source page, enter the name for the Twilio connector and select **Twilio Taskrouter** from the type dropdown. +3. On the Set up the source page, enter the name for the Twilio connector and select **Twilio Taskrouter** from the type dropdown. 4. Enter your `account_sid`. 5. Enter your `auth_token`. 6. Click **Set up source**. @@ -44,12 +45,17 @@ The Twilio source connector supports the following [sync modes](https://docs.air ## Supported Streams -* [Workspaces](https://www.twilio.com/docs/taskrouter/api/workspace) -* [All Workspaces](https://www.twilio.com/docs/taskrouter/api/workspace) -* [Workers](https://www.twilio.com/docs/taskrouter/api/worker) +- [Workspaces](https://www.twilio.com/docs/taskrouter/api/workspace) +- [All Workspaces](https://www.twilio.com/docs/taskrouter/api/workspace) +- [Workers](https://www.twilio.com/docs/taskrouter/api/worker) ## Performance considerations -The Twilio Taskrouter connector will gracefully handle rate limits. +The Twilio Taskrouter connector will gracefully handle rate limits. For more information, see [the Twilio docs for rate limitations](https://support.twilio.com/hc/en-us/articles/360044308153-Twilio-API-response-Error-429-Too-Many-Requests). +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------- | +| 0.1.0 | 2022-11-18 | [18685](https://github.com/airbytehq/airbyte/pull/18685) | 🎉 New Source: Twilio Taskrouter API [low-code cdk] | diff --git a/docs/integrations/sources/us-census.md b/docs/integrations/sources/us-census.md index d81dcaa39a49..6a3b5d5b1a53 100644 --- a/docs/integrations/sources/us-census.md +++ b/docs/integrations/sources/us-census.md @@ -10,19 +10,19 @@ This source always outputs a single stream, `us_census_stream`. The output of th ### Features -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental Sync | No | -| SSL connection | Yes | -| Namespaces | No | +| Feature | Supported? | +| :---------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental Sync | No | +| SSL connection | Yes | +| Namespaces | No | ## Getting started ### Requirements -* US Census API key -* US Census dataset path & query parameters +- US Census API key +- US Census dataset path & query parameters ### Setup guide @@ -30,14 +30,14 @@ Visit the [US Census API page](https://api.census.gov/data/key_signup.html) to o In addition, to understand how to configure the dataset path and query parameters, follow the guide and examples in the [API documentation](https://www.census.gov/data/developers/data-sets.html). Some particularly helpful pages: -* [Available Datasets](https://www.census.gov/data/developers/guidance/api-user-guide.Available_Data.html) -* [Core Concepts](https://www.census.gov/data/developers/guidance/api-user-guide.Core_Concepts.html) -* [Example Queries](https://www.census.gov/data/developers/guidance/api-user-guide.Example_API_Queries.html) +- [Available Datasets](https://www.census.gov/data/developers/guidance/api-user-guide.Available_Data.html) +- [Core Concepts](https://www.census.gov/data/developers/guidance/api-user-guide.Core_Concepts.html) +- [Example Queries](https://www.census.gov/data/developers/guidance/api-user-guide.Example_API_Queries.html) ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.1.0 | 2021-07-20 | [4228](https://github.com/airbytehq/airbyte/pull/4228) | Initial release | - +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :----------------------------------------------------- | :------------------------------------------------ | +| 0.1.2 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Update fields in source-connectors specifications | +| 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | +| 0.1.0 | 2021-07-20 | [4228](https://github.com/airbytehq/airbyte/pull/4228) | Initial release | diff --git a/docs/integrations/sources/webflow.md b/docs/integrations/sources/webflow.md index 0a0d1adb7a6e..cabeae06101b 100644 --- a/docs/integrations/sources/webflow.md +++ b/docs/integrations/sources/webflow.md @@ -4,14 +4,15 @@ description: 'This connector extracts "collections" from Webflow' # Webflow -Webflow is a CMS system that is used for publishing websites and blogs. This connector returns data that is made available by [Webflow APIs](https://developers.webflow.com/). +Webflow is a CMS system that is used for publishing websites and blogs. This connector returns data that is made available by [Webflow APIs](https://developers.webflow.com/). -Webflow uses [Collections](https://developers.webflow.com/#collections) to store different kinds of information. A collection can be "Blog Posts", or "Blog Authors", etc. Collection names are not pre-defined, the number of collections is not known in advance, and the schema for each collection may be different. +Webflow uses [Collections](https://developers.webflow.com/#collections) to store different kinds of information. A collection can be "Blog Posts", or "Blog Authors", etc. Collection names are not pre-defined, the number of collections is not known in advance, and the schema for each collection may be different. -This connector dynamically figures our which collections are available, creates the schema for each collection based on data extracted from Webflow, and creates an [Airbyte Stream](https://docs.airbyte.com/connector-development/cdk-python/full-refresh-stream/) for each collection. +This connector dynamically figures our which collections are available, creates the schema for each collection based on data extracted from Webflow, and creates an [Airbyte Stream](https://docs.airbyte.com/connector-development/cdk-python/full-refresh-stream/) for each collection. # Webflow credentials -You should be able to create a Webflow `API key` (aka `API token`) as described in [Intro to the Webflow API](https://university.webflow.com/lesson/intro-to-the-webflow-api). + +You should be able to create a Webflow `API key` (aka `API token`) as described in [Intro to the Webflow API](https://university.webflow.com/lesson/intro-to-the-webflow-api). Once you have the `API Key`/`API token`, you can confirm a [list of available sites](https://developers.webflow.com/#sites) and get their `_id` by executing the following: @@ -27,15 +28,16 @@ Which should respond with something similar to: [{"_id":"","createdOn":"2021-03-26T15:46:04.032Z","name":"Airbyte","shortName":"airbyte-dev","lastPublished":"2022-06-09T12:55:52.533Z","previewUrl":"https://screenshots.webflow.com/sites/","timezone":"America/Los_Angeles","database":""}] ``` -You will need to provide the `Site id` and `API key` to the Webflow connector in order for it to pull data from your Webflow site. +You will need to provide the `Site id` and `API key` to the Webflow connector in order for it to pull data from your Webflow site. # Related tutorial -If you are interested in learning more about the Webflow API and implementation details of this connector, you may wish to consult the [tutorial about how to build a connector to extract data from the Webflow API](https://airbyte.com/tutorials/extract-data-from-the-webflow-api). - -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.0 | 2022-06-22 | [13617](https://github.com/airbytehq/airbyte/pull/13617) | Initial release | -| 0.1.1 | 2022-06-22 | [13617](https://github.com/airbytehq/airbyte/pull/13617) | Update Spec Documentation URL | +If you are interested in learning more about the Webflow API and implementation details of this connector, you may wish to consult the [tutorial about how to build a connector to extract data from the Webflow API](https://airbyte.com/tutorials/extract-data-from-the-webflow-api). +## Changelog +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------- | +| 0.1.2 | 2022-07-14 | [14689](https://github.com/airbytehq/airbyte/pull/14689) | Webflow add ids to streams | +| 0.1.1 | 2022-06-22 | [13617](https://github.com/airbytehq/airbyte/pull/13617) | Update Spec Documentation URL | +| 0.1.0 | 2022-06-22 | [13617](https://github.com/airbytehq/airbyte/pull/13617) | Initial release | diff --git a/docs/integrations/sources/woocommerce.md b/docs/integrations/sources/woocommerce.md index dcf4ef97a77a..d65b4223391e 100644 --- a/docs/integrations/sources/woocommerce.md +++ b/docs/integrations/sources/woocommerce.md @@ -6,9 +6,9 @@ This page contains the setup guide and reference information for the WooCommerce To set up the WooCommerce source connector with Airbyte, you must be using: -* WooCommerce 3.5+ -* WordPress 4.4+ -* Pretty permalinks in `Settings > Permalinks` so that the custom endpoints are supported. +- WooCommerce 3.5+ +- WordPress 4.4+ +- Pretty permalinks in `Settings > Permalinks` so that the custom endpoints are supported. e.g. `/%year%/%monthnum%/%day%/%postname%/` You will need to generate new API key with read permissions and use `Customer key` and `Customer Secret`. @@ -49,45 +49,45 @@ You will need to generate new API key with read permissions and use `Customer ke The WooCommerce source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): -* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/glossary#full-refresh-sync) -* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append) -* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) -* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history) +- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/glossary#full-refresh-sync) +- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append) +- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) +- [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history) ## Supported Streams -* [Coupons](https://woocommerce.github.io/woocommerce-rest-api-docs/#coupons) \(Incremental\) -* [Customers](https://woocommerce.github.io/woocommerce-rest-api-docs/#customers) \(Incremental\) -* [orders](https://woocommerce.github.io/woocommerce-rest-api-docs/#orders) \(Incremental\) -* [Order notes](https://woocommerce.github.io/woocommerce-rest-api-docs/#order-notes) -* [Payment gateways](https://woocommerce.github.io/woocommerce-rest-api-docs/#payment-gateways) -* [Product attribute terms](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-attribute-terms) -* [Product attributes](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-attributes) -* [Product categories](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-categories) -* [Product reviews](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-reviews) \(Incremental\) -* [Product shipping classes](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-shipping-classes) -* [Product tags](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-tags) -* [Product variations](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-variations) -* [Products](https://woocommerce.github.io/woocommerce-rest-api-docs/#products) \(Incremental\) -* [Refunds](https://woocommerce.github.io/woocommerce-rest-api-docs/#refunds) -* [Shipping methods](https://woocommerce.github.io/woocommerce-rest-api-docs/#shipping-methods) -* [Shipping zone locations](https://woocommerce.github.io/woocommerce-rest-api-docs/#shipping-zone-locations) -* [Shipping zone methods](https://woocommerce.github.io/woocommerce-rest-api-docs/#shipping-zone-methods) -* [Shipping zones](https://woocommerce.github.io/woocommerce-rest-api-docs/#shipping-zones) -* [System status tools](https://woocommerce.github.io/woocommerce-rest-api-docs/#system-status-tools) -* [Tax classes](https://woocommerce.github.io/woocommerce-rest-api-docs/#tax-classes) -* [Tax rates](https://woocommerce.github.io/woocommerce-rest-api-docs/#tax-rates) +- [Coupons](https://woocommerce.github.io/woocommerce-rest-api-docs/#coupons) \(Incremental\) +- [Customers](https://woocommerce.github.io/woocommerce-rest-api-docs/#customers) \(Incremental\) +- [orders](https://woocommerce.github.io/woocommerce-rest-api-docs/#orders) \(Incremental\) +- [Order notes](https://woocommerce.github.io/woocommerce-rest-api-docs/#order-notes) +- [Payment gateways](https://woocommerce.github.io/woocommerce-rest-api-docs/#payment-gateways) +- [Product attribute terms](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-attribute-terms) +- [Product attributes](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-attributes) +- [Product categories](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-categories) +- [Product reviews](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-reviews) \(Incremental\) +- [Product shipping classes](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-shipping-classes) +- [Product tags](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-tags) +- [Product variations](https://woocommerce.github.io/woocommerce-rest-api-docs/#product-variations) +- [Products](https://woocommerce.github.io/woocommerce-rest-api-docs/#products) \(Incremental\) +- [Refunds](https://woocommerce.github.io/woocommerce-rest-api-docs/#refunds) +- [Shipping methods](https://woocommerce.github.io/woocommerce-rest-api-docs/#shipping-methods) +- [Shipping zone locations](https://woocommerce.github.io/woocommerce-rest-api-docs/#shipping-zone-locations) +- [Shipping zone methods](https://woocommerce.github.io/woocommerce-rest-api-docs/#shipping-zone-methods) +- [Shipping zones](https://woocommerce.github.io/woocommerce-rest-api-docs/#shipping-zones) +- [System status tools](https://woocommerce.github.io/woocommerce-rest-api-docs/#system-status-tools) +- [Tax classes](https://woocommerce.github.io/woocommerce-rest-api-docs/#tax-classes) +- [Tax rates](https://woocommerce.github.io/woocommerce-rest-api-docs/#tax-rates) ## Connector-specific features & highlights Useful links: -* [WooCommerce Rest API Docs](https://woocommerce.github.io/woocommerce-rest-api-docs/#introduction) +- [WooCommerce Rest API Docs](https://woocommerce.github.io/woocommerce-rest-api-docs/#introduction) ## Data type map | Integration Type | Airbyte Type | Notes | -|:-----------------|:-------------|:------| +| :--------------- | :----------- | :---- | | `string` | `string` | | | `integer` | `integer` | | | `number` | `number` | | @@ -98,7 +98,7 @@ Useful links: ## Changelog | Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------| +| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------- | | 0.2.0 | 2022-11-30 | [19903](https://github.com/airbytehq/airbyte/pull/19903) | Migrate to low-code; Certification to Beta | | 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | | 0.1.0 | 2021-09-09 | [5955](https://github.com/airbytehq/airbyte/pull/5955) | Initial Release. Source WooCommerce | diff --git a/docs/integrations/sources/xero.md b/docs/integrations/sources/xero.md index 1aa3a11ede06..7fd6946db95e 100644 --- a/docs/integrations/sources/xero.md +++ b/docs/integrations/sources/xero.md @@ -4,7 +4,7 @@ This is a setup guide for the Xero source connector which ingests data from the ## Prerequisites -First of all you should create an application in [Xero development center](https://developer.xero.com/app/manage/). The only supported integration type is to use [Xero Custom Connections](https://developer.xero.com/documentation/guides/oauth2/custom-connections/developer) so you should choose it on creating your Xero App. +First of all you should create an application in [Xero development center](https://developer.xero.com/app/manage/). The only supported integration type is to use [Xero Custom Connections](https://developer.xero.com/documentation/guides/oauth2/custom-connections/developer) so you should choose it on creating your Xero App. After creating an application, on configuration screen, authorize user for your Xero Organisation. Also, issue new Client Secret and remember it - it will be required for setting up Xero connector in your Airbyte instance. ## Supported streams @@ -56,3 +56,7 @@ The source connector supports the following [sync modes](https://docs.airbyte.co - Incremental ## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------------- | +| 0.1.0 | 2021-11-11 | [18666](https://github.com/airbytehq/airbyte/pull/18666) | 🎉 New Source - Xero [python cdk] | diff --git a/docs/integrations/sources/xkcd.md b/docs/integrations/sources/xkcd.md index 9088fa705eb8..64457c703bcb 100644 --- a/docs/integrations/sources/xkcd.md +++ b/docs/integrations/sources/xkcd.md @@ -1,8 +1,8 @@ -# XKCD +# XKCD This page guides you through the process of setting up the xkcd source connector. -## Prerequisites +## Prerequisites XKCD is an open API, so no credentials are needed to set up the surce. @@ -20,6 +20,7 @@ XKCD does not perform rate limiting. ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.1.0 | 2022-10-17 | [18049](https://github.com/airbytehq/airbyte/pull/18049) | Initial version/release of the connector. +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------- | +| 0.1.1 | 2022-10-24 | [18386](https://github.com/airbytehq/airbyte/pull/18386) | Readded xkcd to source def yaml | +| 0.1.0 | 2022-10-17 | [18049](https://github.com/airbytehq/airbyte/pull/18049) | Initial version/release of the connector. | diff --git a/docs/integrations/sources/yahoo-finance-price.md b/docs/integrations/sources/yahoo-finance-price.md new file mode 100644 index 000000000000..8162eb58e3ea --- /dev/null +++ b/docs/integrations/sources/yahoo-finance-price.md @@ -0,0 +1,9 @@ +# Yahoo Finance Price + +The Airbyte Source for [Yahoo Finance Price](https://finance.yahoo.com/) + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------- | +| 0.1.3 | 2022-03-23 | [10563](https://github.com/airbytehq/airbyte/pull/10563) | 🎉 Source Yahoo Finance Price | diff --git a/docs/integrations/sources/yandex-metrica.md b/docs/integrations/sources/yandex-metrica.md index abc103528f9a..025a2f5b38cb 100644 --- a/docs/integrations/sources/yandex-metrica.md +++ b/docs/integrations/sources/yandex-metrica.md @@ -3,32 +3,44 @@ This page guides you through the process of setting up the Yandex Metrica source connector. ## Prerequisites -* Counter ID -* OAuth2 Token + +- Counter ID +- OAuth2 Token ## Setup Yandex Metrica + 1. [Create a Yandex Metrica account](https://metrica.yandex.com/) if you don't already have one. 2. Head to [Management page](https://metrica.yandex.com/list) and add new tag or choose an existing one. 3. At the top of the dasboard you will see 8 digit number to the right of your website name. This is your **Counter ID**. 4. Create a new app or choose an existing one from [My apps page](https://oauth.yandex.com/). - * Which platform is the app required for?: **Web services** - * Callback URL: https://oauth.yandex.com/verification_code - * What data do you need?: **Yandex.Metrica**. Read permission will suffice. -5. Choose your app from [the list](https://oauth.yandex.com/). - * To create your API key you will need to grab your **ClientID**, - * Now to get the API key craft a GET request to an endpoint *https://oauth.yandex.com/authorizE?response_type=token&client_id=\* - * You will receive a response with your **API key**. Save it. + - Which platform is the app required for?: **Web services** + - Callback URL: https://oauth.yandex.com/verification_code + - What data do you need?: **Yandex.Metrica**. Read permission will suffice. +5. Choose your app from [the list](https://oauth.yandex.com/). + - To create your API key you will need to grab your **ClientID**, + - Now to get the API key craft a GET request to an endpoint *https://oauth.yandex.com/authorizE?response_type=token&client_id=\* + - You will receive a response with your **API key**. Save it. ## Supported sync modes + The Yandex Metrica source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): - - Full Refresh - - Incremental - * After the first sync the connector will set the state for next sync. The **start date** will be set to last syncs **end date**. The **end date** will be set to 1 day before today. + +- Full Refresh +- Incremental + - After the first sync the connector will set the state for next sync. The **start date** will be set to last syncs **end date**. The **end date** will be set to 1 day before today. ## Supported Streams -* [Views](https://yandex.com/dev/metrika/doc/api2/logs/fields/hits.html) (Incremental). -* [Sessions](https://yandex.com/dev/metrika/doc/api2/logs/fields/visits.html) (Incremental). + +- [Views](https://yandex.com/dev/metrika/doc/api2/logs/fields/hits.html) (Incremental). +- [Sessions](https://yandex.com/dev/metrika/doc/api2/logs/fields/visits.html) (Incremental). ## Notes + - We recommend syncing data once a day. Because of the Yandex Metrica API limitation it is only possible to extract records up to yesterdays date. Todays records will only be available tomorrow. -- Because of the way API works some syncs may take a long time to finish. Timeout period is 2 hours. \ No newline at end of file +- Because of the way API works some syncs may take a long time to finish. Timeout period is 2 hours. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------- | +| 0.1.0 | 2022-09-09 | [15061](https://github.com/airbytehq/airbyte/pull/15061) | 🎉 New Source: Yandex metrica | diff --git a/docs/integrations/sources/younium.md b/docs/integrations/sources/younium.md index 7aa192d09dff..60d510a0e700 100644 --- a/docs/integrations/sources/younium.md +++ b/docs/integrations/sources/younium.md @@ -7,30 +7,38 @@ This page contains the setup guide and reference information for the Younium sou This Younium source uses the [Younium API](https://developer.younium.com/). ## Setup guide + ### Step 1: Set up Younium #### For Airbyte OSS: + 1. Navigate to the Airbyte Open Source dashboard -2. Enter a name for your source +2. Enter a name for your source 3. Enter your Younium `username` 4. Enter your Younium `password` -5. Enter your Younium `legal_entity`. You can find the legal entity name in your account setting if you log in to the [Younium web platform](https://app.younium.com/) -7. Click **Set up source** +5. Enter your Younium `legal_entity`. You can find the legal entity name in your account setting if you log in to the [Younium web platform](https://app.younium.com/) +6. Click **Set up source** ## Supported sync modes The Younium source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): -| Feature | Supported? | -| :--- | :-- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | No | -| Replicate Incremental Deletes | No | -| SSL connection | Yes | -| Namespaces | No | +| Feature | Supported? | +| :---------------------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | No | +| Replicate Incremental Deletes | No | +| SSL connection | Yes | +| Namespaces | No | ## Supported Streams -* [Subscriptions](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Subscriptions) -* [Products](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Products) -* [Invoices](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Invoices) \ No newline at end of file +- [Subscriptions](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Subscriptions) +- [Products](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Products) +- [Invoices](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Invoices) + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------------- | +| 0.1.0 | 2022-11-09 | [18758](https://github.com/airbytehq/airbyte/pull/18758) | 🎉 New Source: Younium [python cdk] | diff --git a/docs/integrations/sources/youtube-analytics-business.md b/docs/integrations/sources/youtube-analytics-business.md new file mode 100644 index 000000000000..45bcbe497441 --- /dev/null +++ b/docs/integrations/sources/youtube-analytics-business.md @@ -0,0 +1,9 @@ +# YouTube Analytics Business + +The Airbyte Source for YouTube Analytics Business + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------- | +| 0.1.0 | 2022-11-09 | [19223](https://github.com/airbytehq/airbyte/pull/19223) | 🎉 New Source: Demo - Youtube Analytics Business | diff --git a/docs/understanding-airbyte/connections/incremental-append.md b/docs/understanding-airbyte/connections/incremental-append.md index d7f84d5c6bcb..a779d0b1d13c 100644 --- a/docs/understanding-airbyte/connections/incremental-append.md +++ b/docs/understanding-airbyte/connections/incremental-append.md @@ -22,47 +22,47 @@ As mentioned above, the delta from a sync will be _appended_ to the existing dat Assume that `updated_at` is our `cursor_field`. Let's say the following data already exists into our data warehouse. -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | false | 1754 | -| Marie Antoinette | false | 1755 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | false | 1754 | +| Marie Antoinette | false | 1755 | In the next sync, the delta contains the following record: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVII | false | 1785 | +| name | deceased | updated_at | +| :--------- | :------- | :--------- | +| Louis XVII | false | 1785 | At the end of this incremental sync, the data warehouse would now contain: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | false | 1754 | -| Marie Antoinette | false | 1755 | -| Louis XVII | false | 1785 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | false | 1754 | +| Marie Antoinette | false | 1755 | +| Louis XVII | false | 1785 | ### Updating a Record Let's assume that our warehouse contains all the data that it did at the end of the previous section. Now, unfortunately the king and queen lose their heads. Let's see that delta: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | true | 1793 | -| Marie Antoinette | true | 1793 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | true | 1793 | +| Marie Antoinette | true | 1793 | The output we expect to see in the warehouse is as follows: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | false | 1754 | -| Marie Antoinette | false | 1755 | -| Louis XVII | false | 1785 | -| Louis XVI | true | 1793 | -| Marie Antoinette | true | 1793 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | false | 1754 | +| Marie Antoinette | false | 1755 | +| Louis XVII | false | 1785 | +| Louis XVI | true | 1793 | +| Marie Antoinette | true | 1793 | ## Source-Defined Cursor -Some sources are able to determine the cursor that they use without any user input. For example, in the [exchange rates source](../../integrations/sources/exchangeratesapi.md), the source knows that the date field should be used to determine the last record that was synced. In these cases, simply select the incremental option in the UI. +Some sources are able to determine the cursor that they use without any user input. For example, in the [exchange rates source](../../integrations/sources/exchange-rates.md), the source knows that the date field should be used to determine the last record that was synced. In these cases, simply select the incremental option in the UI. ![](../../.gitbook/assets/incremental_source_defined.png) @@ -102,23 +102,23 @@ SELECT * FROM table WHERE cursor_field >= 'last_sync_max_cursor_field_value' Let's say the following data already exists into our data warehouse. -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | false | 1754 | -| Marie Antoinette | false | 1755 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | false | 1754 | +| Marie Antoinette | false | 1755 | At the start of the next sync, the source data contains the following new record: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | true | 1754 | +| name | deceased | updated_at | +| :-------- | :------- | :--------- | +| Louis XVI | true | 1754 | At the end of the second incremental sync, the data warehouse would still contain data from the first sync because the delta record did not provide a valid value for the cursor field \(the cursor field is not greater than last sync's max value, `1754 < 1755`\), so it is not emitted by the source as a new or modified record. -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | false | 1754 | -| Marie Antoinette | false | 1755 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | false | 1754 | +| Marie Antoinette | false | 1755 | Similarly, if multiple modifications are made during the same day to the same records. If the frequency of the sync is not granular enough \(for example, set for every 24h\), then intermediate modifications to the data are not going to be detected and emitted. Only the state of data at the time the sync runs will be reflected in the destination. diff --git a/docs/understanding-airbyte/connections/incremental-deduped-history.md b/docs/understanding-airbyte/connections/incremental-deduped-history.md index f0b954e7340d..be02105b8bfe 100644 --- a/docs/understanding-airbyte/connections/incremental-deduped-history.md +++ b/docs/understanding-airbyte/connections/incremental-deduped-history.md @@ -35,57 +35,57 @@ As mentioned above, the delta from a sync will be _appended_ to the existing his Assume that `updated_at` is our `cursor_field` and `name` is the `primary_key`. Let's say the following data already exists into our data warehouse. -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | false | 1754 | -| Marie Antoinette | false | 1755 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | false | 1754 | +| Marie Antoinette | false | 1755 | In the next sync, the delta contains the following record: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVII | false | 1785 | +| name | deceased | updated_at | +| :--------- | :------- | :--------- | +| Louis XVII | false | 1785 | At the end of this incremental sync, the data warehouse would now contain: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | false | 1754 | -| Marie Antoinette | false | 1755 | -| Louis XVII | false | 1785 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | false | 1754 | +| Marie Antoinette | false | 1755 | +| Louis XVII | false | 1785 | ### Updating a Record Let's assume that our warehouse contains all the data that it did at the end of the previous section. Now, unfortunately the king and queen lose their heads. Let's see that delta: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | true | 1793 | -| Marie Antoinette | true | 1793 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | true | 1793 | +| Marie Antoinette | true | 1793 | The output we expect to see in the warehouse is as follows: In the history table: -| name | deceased | updated\_at | start\_at | end\_at | -| :--- | :--- | :--- | :--- | :--- | -| Louis XVI | false | 1754 | 1754 | 1793 | -| Louis XVI | true | 1793 | 1793 | NULL | -| Louis XVII | false | 1785 | 1785 | NULL | -| Marie Antoinette | false | 1755 | 1755 | 1793 | -| Marie Antoinette | true | 1793 | 1793 | NULL | +| name | deceased | updated_at | start_at | end_at | +| :--------------- | :------- | :--------- | :------- | :----- | +| Louis XVI | false | 1754 | 1754 | 1793 | +| Louis XVI | true | 1793 | 1793 | NULL | +| Louis XVII | false | 1785 | 1785 | NULL | +| Marie Antoinette | false | 1755 | 1755 | 1793 | +| Marie Antoinette | true | 1793 | 1793 | NULL | In the final de-duplicated table: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | true | 1793 | -| Louis XVII | false | 1785 | -| Marie Antoinette | true | 1793 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | true | 1793 | +| Louis XVII | false | 1785 | +| Marie Antoinette | true | 1793 | ## Source-Defined Cursor -Some sources are able to determine the cursor that they use without any user input. For example, in the [exchange rates source](../../integrations/sources/exchangeratesapi.md), the source knows that the date field should be used to determine the last record that was synced. In these cases, simply select the incremental option in the UI. +Some sources are able to determine the cursor that they use without any user input. For example, in the [exchange rates source](../../integrations/sources/exchange-rates.md), the source knows that the date field should be used to determine the last record that was synced. In these cases, simply select the incremental option in the UI. ![](../../.gitbook/assets/incremental_source_defined.png) @@ -127,23 +127,23 @@ select * from table where cursor_field > 'last_sync_max_cursor_field_value' Let's say the following data already exists into our data warehouse. -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | false | 1754 | -| Marie Antoinette | false | 1755 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | false | 1754 | +| Marie Antoinette | false | 1755 | At the start of the next sync, the source data contains the following new record: -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | true | 1754 | +| name | deceased | updated_at | +| :-------- | :------- | :--------- | +| Louis XVI | true | 1754 | At the end of the second incremental sync, the data warehouse would still contain data from the first sync because the delta record did not provide a valid value for the cursor field \(the cursor field is not greater than last sync's max value, `1754 < 1755`\), so it is not emitted by the source as a new or modified record. -| name | deceased | updated\_at | -| :--- | :--- | :--- | -| Louis XVI | false | 1754 | -| Marie Antoinette | false | 1755 | +| name | deceased | updated_at | +| :--------------- | :------- | :--------- | +| Louis XVI | false | 1754 | +| Marie Antoinette | false | 1755 | Similarly, if multiple modifications are made during the same day to the same records. If the frequency of the sync is not granular enough \(for example, set for every 24h\), then intermediate modifications to the data are not going to be detected and emitted. Only the state of data at the time the sync runs will be reflected in the destination. diff --git a/tools/bin/ci_check_dependency.py b/tools/bin/ci_check_dependency.py index 23a55077aab4..a57ae036d382 100644 --- a/tools/bin/ci_check_dependency.py +++ b/tools/bin/ci_check_dependency.py @@ -17,9 +17,20 @@ "/integration_tests/", "/unit_tests/", # Common "acceptance-test-config.yml", "acceptance-test-docker.sh", ".md", ".dockerignore", ".gitignore", "requirements.txt"] +IGNORED_SOURCES = [ + re.compile("^source-e2e-test-cloud$"), + re.compile("^source-mongodb$"), + re.compile("^source-python-http-tutorial$"), + re.compile("^source-relational-db$"), + re.compile("^source-stock-ticker-api-tutorial$"), + re.compile("source-jdbc$"), + re.compile("^source-scaffold-.*$"), + re.compile(".*-secure$"), +] IGNORED_DESTINATIONS = [ re.compile(".*-strict-encrypt$"), re.compile("^destination-dev-null$"), + re.compile("^destination-scaffold-destination-python$"), re.compile("^destination-jdbc$") ] COMMENT_TEMPLATE_PATH = ".github/comment_templates/connector_dependency_template.md" @@ -130,6 +141,8 @@ def get_connector_changelog_status(connector: str, version) -> str: type, name = connector.replace("-strict-encrypt", "").replace("-denormalized", "").split("-", 1) doc_path = f"{DOC_PATH}{type}s/{name}.md" + if any(regex.match(connector) for regex in IGNORED_SOURCES): + return "🔵
(ignored)" if any(regex.match(connector) for regex in IGNORED_DESTINATIONS): return "🔵
(ignored)" if not os.path.exists(doc_path): @@ -160,7 +173,9 @@ def as_markdown_table_rows(connectors: List[str], definitions) -> str: version_status = get_connector_version_status(connector, version) changelog_status = get_connector_changelog_status(connector, version) definition = next((x for x in definitions if x["dockerRepository"].endswith(connector)), None) - if any(regex.match(connector) for regex in IGNORED_DESTINATIONS): + if any(regex.match(connector) for regex in IGNORED_SOURCES): + publish_status = "🔵
(ignored)" + elif any(regex.match(connector) for regex in IGNORED_DESTINATIONS): publish_status = "🔵
(ignored)" elif definition is None: publish_status = "âš 
(not in seed)" From 401aa442c92bec4843f47ad8b1c14e2e918f6966 Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Tue, 20 Dec 2022 15:17:04 -0800 Subject: [PATCH 5/7] Fix failing test (#20634) * Fix failing test * Fix failing test * Fix failing test * Fix failing test * Fix failing test * Fix failing test * Fix failing test * Fix failing test * Fix failing test * comment * comment --- .../integration_tests/sources/Db2SourceDatatypeTest.java | 3 ++- .../source/oracle/OracleSourceDatatypeTest.java | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java index 5ea0a2dc6b9f..eb47985e15c1 100644 --- a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java @@ -16,6 +16,7 @@ import io.airbyte.integrations.standardtest.source.TestDataHolder; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.JsonSchemaType; +import java.math.BigDecimal; import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.testcontainers.containers.Db2Container; @@ -126,7 +127,7 @@ protected void initTests() { .airbyteType(JsonSchemaType.NUMBER) .fullSourceDataType("DECIMAL(31, 0)") .addInsertValues("null", "1", "DECIMAL((-1 + 10E+29), 31, 0)", "DECIMAL((1 - 10E+29), 31, 0)") - .addExpectedValues(null, "1", "1.0E30", "-1.0E30") + .addExpectedValues(null, "1", "%.0f".formatted(Double.valueOf("1.0E30")), "%.0f".formatted(Double.valueOf("-1.0E30"))) .build()); addDataTypeTestData( TestDataHolder.builder() diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java index 7d8541597a85..c80b0998f91d 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java @@ -146,7 +146,10 @@ protected void initTests() { .sourceType("NUMBER") .airbyteType(JsonSchemaType.NUMBER) .addInsertValues("null", "1", "123.45", "power(10, -130)", "9.99999999999999999999 * power(10, 125)") - .addExpectedValues(null, "1", "123.45", String.valueOf(Math.pow(10, -130)), String.valueOf(9.99999999999999999999 * Math.pow(10, 125))) + /* The 999990000… below is the plain string representation of 9.999 * power(10, 125) */ + /* because normalization expects a plain integer strings whereas `Math.pow(10, 125)` returns a scientific notation */ + .addExpectedValues(null, "1", "123.45", String.valueOf(Math.pow(10, -130)), + "999999999999999999999000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") .build()); addDataTypeTestData( @@ -155,7 +158,7 @@ protected void initTests() { .airbyteType(JsonSchemaType.NUMBER) .fullSourceDataType("NUMBER(6,-2)") .addInsertValues("123.89") - .addExpectedValues("100.0") + .addExpectedValues("100") .build()); addDataTypeTestData( @@ -164,7 +167,7 @@ protected void initTests() { .airbyteType(JsonSchemaType.NUMBER) .fullSourceDataType("FLOAT(5)") .addInsertValues("1.34", "126.45") - .addExpectedValues("1.3", "130.0") + .addExpectedValues("1.3", "130") .build()); addDataTypeTestData( From 16e890aee1c9456f7dd22c61abd12214c02d131e Mon Sep 17 00:00:00 2001 From: Michael Siega <109092231+mfsiega-airbyte@users.noreply.github.com> Date: Wed, 21 Dec 2022 02:45:03 +0100 Subject: [PATCH 6/7] add noise to cron scheduling (#20665) * add noise to cron scheduling * fix tests * add a workspace id to test cron scheduling jitter * add unit test and fix scheduling noise for crons --- .../activities/ConfigFetchActivityImpl.java | 52 ++++++++- .../activities/ConfigFetchActivityTest.java | 103 +++++++++++++----- 2 files changed, 125 insertions(+), 30 deletions(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityImpl.java index 514a56eba4f9..794b501d8b62 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityImpl.java @@ -7,6 +7,7 @@ import static io.airbyte.metrics.lib.ApmTraceConstants.ACTIVITY_TRACE_OPERATION_NAME; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.CONNECTION_ID_KEY; +import com.google.common.annotations.VisibleForTesting; import datadog.trace.api.Trace; import io.airbyte.commons.temporal.config.WorkerMode; import io.airbyte.commons.temporal.exception.RetryableException; @@ -19,6 +20,7 @@ import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.persistence.job.WorkspaceHelper; import io.airbyte.persistence.job.models.Job; import io.airbyte.validation.json.JsonValidationException; import io.micronaut.context.annotation.Requires; @@ -32,23 +34,35 @@ import java.util.Date; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.TimeZone; import java.util.UUID; import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import org.joda.time.DateTimeZone; import org.quartz.CronExpression; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Slf4j @Singleton @Requires(env = WorkerMode.CONTROL_PLANE) public class ConfigFetchActivityImpl implements ConfigFetchActivity { + private static final Logger LOGGER = LoggerFactory.getLogger(ConfigFetchActivityImpl.class); private final static long MS_PER_SECOND = 1000L; private final static long MIN_CRON_INTERVAL_SECONDS = 60; + private static final Set SCHEDULING_NOISE_WORKSPACE_IDS = Set.of( + // Testing + UUID.fromString("0ace5e1f-4787-43df-8919-456f5f4d03d1"), + UUID.fromString("20810d92-41a4-4cfd-85db-fb50e77cf36b"), + // Prod + UUID.fromString("226edbc1-4a9c-4401-95a9-90435d667d9d")); + private static final long SCHEDULING_NOISE_CONSTANT = 15; private final ConfigRepository configRepository; private final JobPersistence jobPersistence; + private final WorkspaceHelper workspaceHelper; private final Integer syncJobMaxAttempts; private final Supplier currentSecondsSupplier; @@ -56,8 +70,18 @@ public ConfigFetchActivityImpl(final ConfigRepository configRepository, final JobPersistence jobPersistence, @Value("${airbyte.worker.sync.max-attempts}") final Integer syncJobMaxAttempts, @Named("currentSecondsSupplier") final Supplier currentSecondsSupplier) { + this(configRepository, jobPersistence, new WorkspaceHelper(configRepository, jobPersistence), syncJobMaxAttempts, currentSecondsSupplier); + } + + @VisibleForTesting + protected ConfigFetchActivityImpl(final ConfigRepository configRepository, + final JobPersistence jobPersistence, + final WorkspaceHelper workspaceHelper, + @Value("${airbyte.worker.sync.max-attempts}") final Integer syncJobMaxAttempts, + @Named("currentSecondsSupplier") final Supplier currentSecondsSupplier) { this.configRepository = configRepository; this.jobPersistence = jobPersistence; + this.workspaceHelper = workspaceHelper; this.syncJobMaxAttempts = syncJobMaxAttempts; this.currentSecondsSupplier = currentSecondsSupplier; } @@ -128,8 +152,10 @@ private ScheduleRetrieverOutput getTimeToWaitFromScheduleType(final StandardSync + MIN_CRON_INTERVAL_SECONDS : currentSecondsSupplier.get()) * MS_PER_SECOND); final Date nextRunStart = cronExpression.getNextValidTimeAfter(new Date(earliestNextRun)); - final Duration timeToWait = Duration.ofSeconds( + Duration timeToWait = Duration.ofSeconds( Math.max(0, nextRunStart.getTime() / MS_PER_SECOND - currentSecondsSupplier.get())); + + timeToWait = addSchedulingNoiseForAllowListedWorkspace(timeToWait, standardSync); return new ScheduleRetrieverOutput(timeToWait); } catch (final ParseException e) { throw (DateTimeException) new DateTimeException(e.getMessage()).initCause(e); @@ -137,6 +163,30 @@ private ScheduleRetrieverOutput getTimeToWaitFromScheduleType(final StandardSync } } + private Duration addSchedulingNoiseForAllowListedWorkspace(Duration timeToWait, StandardSync standardSync) { + final UUID workspaceId; + try { + workspaceId = workspaceHelper.getWorkspaceForConnectionId(standardSync.getConnectionId()); + } catch (JsonValidationException | ConfigNotFoundException e) { + // We tolerate exceptions and fail open by doing nothing. + return timeToWait; + } + if (!SCHEDULING_NOISE_WORKSPACE_IDS.contains(workspaceId)) { + // Only apply to a specific set of workspaces. + return timeToWait; + } + if (!standardSync.getScheduleType().equals(ScheduleType.CRON)) { + // Only apply noise to cron connections. + return timeToWait; + } + + // We really do want to add some scheduling noise for this connection. + final long minutesToWait = (long) (Math.random() * SCHEDULING_NOISE_CONSTANT); + LOGGER.debug("Adding {} minutes noise to wait", minutesToWait); + // Note: we add an extra second to make the unit tests pass in case `minutesToWait` was 0. + return timeToWait.plusMinutes(minutesToWait).plusSeconds(1); + } + /** * @param standardSync * @param connectionId diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityTest.java index 69aa56cd40b3..e2f1e103d7c0 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityTest.java @@ -4,6 +4,9 @@ package io.airbyte.workers.temporal.scheduling.activities; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + import io.airbyte.config.BasicSchedule; import io.airbyte.config.Cron; import io.airbyte.config.Schedule; @@ -14,11 +17,13 @@ import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.persistence.job.WorkspaceHelper; import io.airbyte.persistence.job.models.Job; import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.temporal.scheduling.activities.ConfigFetchActivity.ScheduleRetrieverInput; import io.airbyte.workers.temporal.scheduling.activities.ConfigFetchActivity.ScheduleRetrieverOutput; import java.io.IOException; +import java.time.Duration; import java.time.Instant; import java.util.Calendar; import java.util.Optional; @@ -31,7 +36,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) @@ -45,6 +49,9 @@ class ConfigFetchActivityTest { @Mock private JobPersistence mJobPersistence; + @Mock + private WorkspaceHelper mWorkspaceHelper; + @Mock private Job mJob; @@ -69,13 +76,14 @@ class ConfigFetchActivityTest { .withTimeUnit(BasicSchedule.TimeUnit.MINUTES) .withUnits(5L))); + public static final String UTC = "UTC"; private final static StandardSync standardSyncWithCronScheduleType = new StandardSync() .withScheduleType(ScheduleType.CRON) .withStatus(Status.ACTIVE) .withScheduleData(new ScheduleData() .withCron(new Cron() .withCronExpression("0 0 12 * * ?") - .withCronTimeZone("UTC"))); + .withCronTimeZone(UTC))); private final static StandardSync standardSyncWithScheduleDisable = new StandardSync() .withSchedule(new Schedule() @@ -93,7 +101,8 @@ class ConfigFetchActivityTest { @BeforeEach void setup() { configFetchActivity = - new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, SYNC_JOB_MAX_ATTEMPTS, () -> Instant.now().getEpochSecond()); + new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, mWorkspaceHelper, SYNC_JOB_MAX_ATTEMPTS, + () -> Instant.now().getEpochSecond()); } @Nested @@ -102,10 +111,10 @@ class TimeToWaitTest { @Test @DisplayName("Test that the job gets scheduled if it is not manual and if it is the first run with legacy schedule schema") void testFirstJobNonManual() throws IOException, JsonValidationException, ConfigNotFoundException { - Mockito.when(mJobPersistence.getLastReplicationJob(connectionId)) + when(mJobPersistence.getLastReplicationJob(connectionId)) .thenReturn(Optional.empty()); - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithLegacySchedule); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -119,7 +128,7 @@ void testFirstJobNonManual() throws IOException, JsonValidationException, Config @Test @DisplayName("Test that the job will wait for a long time if it is manual in the legacy schedule schema") void testManual() throws IOException, JsonValidationException, ConfigNotFoundException { - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithoutSchedule); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -133,7 +142,7 @@ void testManual() throws IOException, JsonValidationException, ConfigNotFoundExc @Test @DisplayName("Test that the job will wait for a long time if it is disabled") void testDisable() throws IOException, JsonValidationException, ConfigNotFoundException { - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithScheduleDisable); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -147,7 +156,7 @@ void testDisable() throws IOException, JsonValidationException, ConfigNotFoundEx @Test @DisplayName("Test that the connection will wait for a long time if it is deleted") void testDeleted() throws IOException, JsonValidationException, ConfigNotFoundException { - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithScheduleDeleted); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -163,13 +172,13 @@ void testDeleted() throws IOException, JsonValidationException, ConfigNotFoundEx void testWait() throws IOException, JsonValidationException, ConfigNotFoundException { configFetchActivity = new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, SYNC_JOB_MAX_ATTEMPTS, () -> 60L * 3); - Mockito.when(mJob.getStartedAtInSecond()) + when(mJob.getStartedAtInSecond()) .thenReturn(Optional.of(60L)); - Mockito.when(mJobPersistence.getLastReplicationJob(connectionId)) + when(mJobPersistence.getLastReplicationJob(connectionId)) .thenReturn(Optional.of(mJob)); - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithLegacySchedule); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -185,13 +194,13 @@ void testWait() throws IOException, JsonValidationException, ConfigNotFoundExcep void testNotWaitIfLate() throws IOException, JsonValidationException, ConfigNotFoundException { configFetchActivity = new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, SYNC_JOB_MAX_ATTEMPTS, () -> 60L * 10); - Mockito.when(mJob.getStartedAtInSecond()) + when(mJob.getStartedAtInSecond()) .thenReturn(Optional.of(60L)); - Mockito.when(mJobPersistence.getLastReplicationJob(connectionId)) + when(mJobPersistence.getLastReplicationJob(connectionId)) .thenReturn(Optional.of(mJob)); - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithLegacySchedule); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -207,7 +216,7 @@ void testNotWaitIfLate() throws IOException, JsonValidationException, ConfigNotF @Test @DisplayName("Test that the job will wait a long time if it is MANUAL scheduleType") void testManualScheduleType() throws IOException, JsonValidationException, ConfigNotFoundException { - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithManualScheduleType); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -221,10 +230,10 @@ void testManualScheduleType() throws IOException, JsonValidationException, Confi @Test @DisplayName("Test that the job will be immediately scheduled if it is a BASIC_SCHEDULE type on the first run") void testBasicScheduleTypeFirstRun() throws IOException, JsonValidationException, ConfigNotFoundException { - Mockito.when(mJobPersistence.getLastReplicationJob(connectionId)) + when(mJobPersistence.getLastReplicationJob(connectionId)) .thenReturn(Optional.empty()); - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithBasicScheduleType); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -240,13 +249,13 @@ void testBasicScheduleTypeFirstRun() throws IOException, JsonValidationException void testBasicScheduleSubsequentRun() throws IOException, JsonValidationException, ConfigNotFoundException { configFetchActivity = new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, SYNC_JOB_MAX_ATTEMPTS, () -> 60L * 3); - Mockito.when(mJob.getStartedAtInSecond()) + when(mJob.getStartedAtInSecond()) .thenReturn(Optional.of(60L)); - Mockito.when(mJobPersistence.getLastReplicationJob(connectionId)) + when(mJobPersistence.getLastReplicationJob(connectionId)) .thenReturn(Optional.of(mJob)); - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithBasicScheduleType); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -260,19 +269,22 @@ void testBasicScheduleSubsequentRun() throws IOException, JsonValidationExceptio @Test @DisplayName("Test that the job will wait to be scheduled if it is a CRON type") void testCronScheduleSubsequentRun() throws IOException, JsonValidationException, ConfigNotFoundException { - final Calendar mockRightNow = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + final Calendar mockRightNow = Calendar.getInstance(TimeZone.getTimeZone(UTC)); mockRightNow.set(Calendar.HOUR_OF_DAY, 0); mockRightNow.set(Calendar.MINUTE, 0); mockRightNow.set(Calendar.SECOND, 0); mockRightNow.set(Calendar.MILLISECOND, 0); + when(mWorkspaceHelper.getWorkspaceForConnectionId(any())).thenReturn(UUID.randomUUID()); + configFetchActivity = - new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, SYNC_JOB_MAX_ATTEMPTS, () -> mockRightNow.getTimeInMillis() / 1000L); + new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, mWorkspaceHelper, SYNC_JOB_MAX_ATTEMPTS, + () -> mockRightNow.getTimeInMillis() / 1000L); - Mockito.when(mJobPersistence.getLastReplicationJob(connectionId)) + when(mJobPersistence.getLastReplicationJob(connectionId)) .thenReturn(Optional.of(mJob)); - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithCronScheduleType); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -286,20 +298,23 @@ void testCronScheduleSubsequentRun() throws IOException, JsonValidationException @Test @DisplayName("Test that the job will only be scheduled once per minimum cron interval") void testCronScheduleMinimumInterval() throws IOException, JsonValidationException, ConfigNotFoundException { - final Calendar mockRightNow = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + final Calendar mockRightNow = Calendar.getInstance(TimeZone.getTimeZone(UTC)); mockRightNow.set(Calendar.HOUR_OF_DAY, 12); mockRightNow.set(Calendar.MINUTE, 0); mockRightNow.set(Calendar.SECOND, 0); mockRightNow.set(Calendar.MILLISECOND, 0); + when(mWorkspaceHelper.getWorkspaceForConnectionId(any())).thenReturn(UUID.randomUUID()); + configFetchActivity = - new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, SYNC_JOB_MAX_ATTEMPTS, () -> mockRightNow.getTimeInMillis() / 1000L); + new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, mWorkspaceHelper, SYNC_JOB_MAX_ATTEMPTS, + () -> mockRightNow.getTimeInMillis() / 1000L); - Mockito.when(mJob.getStartedAtInSecond()).thenReturn(Optional.of(mockRightNow.getTimeInMillis() / 1000L)); - Mockito.when(mJobPersistence.getLastReplicationJob(connectionId)) + when(mJob.getStartedAtInSecond()).thenReturn(Optional.of(mockRightNow.getTimeInMillis() / 1000L)); + when(mJobPersistence.getLastReplicationJob(connectionId)) .thenReturn(Optional.of(mJob)); - Mockito.when(mConfigRepository.getStandardSync(connectionId)) + when(mConfigRepository.getStandardSync(connectionId)) .thenReturn(standardSyncWithCronScheduleType); final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); @@ -310,6 +325,36 @@ void testCronScheduleMinimumInterval() throws IOException, JsonValidationExcepti .hasHours(24); } + @Test + @DisplayName("Test that for specific workspace ids, we add some noise in the cron scheduling") + void testCronSchedulingNoise() throws IOException, JsonValidationException, ConfigNotFoundException { + final Calendar mockRightNow = Calendar.getInstance(TimeZone.getTimeZone(UTC)); + mockRightNow.set(Calendar.HOUR_OF_DAY, 0); + mockRightNow.set(Calendar.MINUTE, 0); + mockRightNow.set(Calendar.SECOND, 0); + mockRightNow.set(Calendar.MILLISECOND, 0); + + when(mWorkspaceHelper.getWorkspaceForConnectionId(any())).thenReturn(UUID.fromString("226edbc1-4a9c-4401-95a9-90435d667d9d")); + + configFetchActivity = + new ConfigFetchActivityImpl(mConfigRepository, mJobPersistence, mWorkspaceHelper, SYNC_JOB_MAX_ATTEMPTS, + () -> mockRightNow.getTimeInMillis() / 1000L); + + when(mJob.getStartedAtInSecond()).thenReturn(Optional.of(mockRightNow.getTimeInMillis() / 1000L)); + when(mJobPersistence.getLastReplicationJob(connectionId)) + .thenReturn(Optional.of(mJob)); + + when(mConfigRepository.getStandardSync(connectionId)) + .thenReturn(standardSyncWithCronScheduleType); + + final ScheduleRetrieverInput input = new ScheduleRetrieverInput(connectionId); + + final ScheduleRetrieverOutput output = configFetchActivity.getTimeToWait(input); + + // Note: compareTo returns positive if the left side is greater than the right. + Assertions.assertThat(output.getTimeToWait().compareTo(Duration.ofHours(12)) > 0).isTrue(); + } + @Nested class TestGetMaxAttempt { From 221d4e3db1bab109297b64e8f6c47c172631e5cc Mon Sep 17 00:00:00 2001 From: Brian Lai <51336873+brianjlai@users.noreply.github.com> Date: Wed, 21 Dec 2022 11:56:47 -0500 Subject: [PATCH 7/7] remove the extra yaml manifest committed and regenerate models w/ latest schema updates (#20743) Co-authored-by: maxi297 --- .../low_code_component_schema.yaml | 918 ------------------ .../sources/declarative/models/__init__.py | 1 - .../models/declarative_component_schema.py | 6 +- 3 files changed, 3 insertions(+), 922 deletions(-) delete mode 100644 airbyte-cdk/python/airbyte_cdk/sources/declarative/low_code_component_schema.yaml diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/low_code_component_schema.yaml b/airbyte-cdk/python/airbyte_cdk/sources/declarative/low_code_component_schema.yaml deleted file mode 100644 index 02fdc9d3b24b..000000000000 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/low_code_component_schema.yaml +++ /dev/null @@ -1,918 +0,0 @@ -"$schema": http://json-schema.org/draft-07/schema# -"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/airbyte_cdk/sources/declarative/handwritten_manifest.yaml -title: LowCodeComponentManifest -type: object -description: Low Code connector components -version: 1.0.0 -required: - - check - - streams - - version -properties: - check: - "$ref": "#/definitions/CheckStream" - streams: - type: array - items: - "$ref": "#/definitions/DeclarativeStream" - version: - type: string - spec: - "$ref": "#/definitions/Spec" -definitions: - AddedFieldDefinition: - description: Defines the field to add on a record - type: object - required: - - type - - path - - value - properties: - type: - enum: [AddedFieldDefinition] - path: - type: array - items: - type: string - value: - type: string - AddFields: - description: Transformation which adds field to an output record. The path of the added field can be nested. - type: object - required: - - type - - fields - properties: - type: - enum: [AddFields] - fields: - type: array - items: - - "$ref": "#/definitions/AddedFieldDefinition" - $options: - type: object - additionalProperties: true - ApiKeyAuthenticator: - description: Authenticator for requests requiring an api token - type: object - required: - - type - - api_token - properties: - type: - enum: [ApiKeyAuthenticator] - api_token: - type: string - header: - type: string - BasicHttpAuthenticator: - description: Authenticator for requests authenticated with a username and optional password - type: object - required: - - type - - username - properties: - type: - enum: [BasicHttpAuthenticator] - username: - type: string - password: - type: string - BearerAuthenticator: - description: Authenticator for requests authenticated with a Bearer token - type: object - required: - - type - - api_token - properties: - type: - enum: [BearerAuthenticator] - api_token: - type: string - CartesianProductStreamSlicer: - description: Stream slicer that iterates over the cartesian product of input stream slicers - type: object - required: - - type - - stream_slicers - properties: - type: - enum: [CartesianProductStreamSlicer] - stream_slicers: - type: array - items: - anyOf: - - "$ref": "#/definitions/CustomStreamSlicer" - - "$ref": "#/definitions/DatetimeStreamSlicer" - - "$ref": "#/definitions/ListStreamSlicer" - - "$ref": "#/definitions/SingleSlice" - - "$ref": "#/definitions/SubstreamSlicer" - $options: - type: object - additionalProperties: true - CheckStream: - description: Checks the connections by trying to read records from one or many of the streams selected by the developer - type: object - required: - - type - - stream_names - properties: - type: - enum: [CheckStream] - stream_names: - type: array - items: - type: string - CompositeErrorHandler: - description: Error handler that sequentially iterates over a list of error handlers - type: object - required: - - error_handlers - properties: - type: - enum: [CompositeErrorHandler] - error_handlers: - type: array - items: - anyOf: - - "$ref": "#/definitions/CompositeErrorHandler" - - "$ref": "#/definitions/DefaultErrorHandler" - $options: - type: object - additionalProperties: true - ConstantBackoffStrategy: - description: Backoff strategy with a constant backoff interval - type: object - required: - - type - - backoff_time_in_seconds - properties: - type: - enum: [ConstantBackoffStrategy] - backoff_time_in_seconds: - anyOf: - - type: number - - type: string - $options: - type: object - additionalProperties: true - CursorPagination: - description: Pagination strategy that evaluates an interpolated string to define the next page token - type: object - required: - - type - - cursor_value - properties: - type: - enum: [CursorPagination] - cursor_value: - type: string - page_size: - type: integer - stop_condition: - type: string - decoder: - "$ref": "#/definitions/JsonDecoder" - $options: - type: object - additionalProperties: true - CustomAuthenticator: - description: Authenticator component whose behavior is derived from a custom code implementation of the connector - type: object - additionalProperties: true - required: - - type - - class_name - properties: - type: - enum: [CustomAuthenticator] - class_name: - type: string - additionalProperties: true - $options: - type: object - additionalProperties: true - CustomBackoffStrategy: - description: Backoff strategy component whose behavior is derived from a custom code implementation of the connector - type: object - additionalProperties: true - required: - - type - - class_name - properties: - type: - enum: [CustomBackoffStrategy] - class_name: - type: string - $options: - type: object - additionalProperties: true - CustomErrorHandler: - description: Error handler component whose behavior is derived from a custom code implementation of the connector - type: object - additionalProperties: true - required: - - type - - class_name - properties: - type: - enum: [CustomErrorHandler] - class_name: - type: string - $options: - type: object - additionalProperties: true - CustomPaginationStrategy: - description: Pagination strategy component whose behavior is derived from a custom code implementation of the connector - type: object - additionalProperties: true - required: - - type - - class_name - properties: - type: - enum: [CustomPaginationStrategy] - class_name: - type: string - $options: - type: object - additionalProperties: true - CustomRecordExtractor: - description: Record extractor component whose behavior is derived from a custom code implementation of the connector - type: object - additionalProperties: true - required: - - type - - class_name - properties: - type: - enum: [CustomRecordExtractor] - class_name: - type: string - $options: - type: object - additionalProperties: true - CustomStreamSlicer: - description: Stream slicer component whose behavior is derived from a custom code implementation of the connector - type: object - additionalProperties: true - required: - - type - - class_name - properties: - type: - enum: [CustomStreamSlicer] - class_name: - type: string - $options: - type: object - additionalProperties: true - DatetimeStreamSlicer: - description: Stream slicer that slices the stream over a datetime range - type: object - required: - - type - - cursor_field - - end_datetime - - datetime_format - - start_datetime - - step - properties: - type: - enum: [DatetimeStreamSlicer] - cursor_field: - type: string - datetime_format: - type: string - end_datetime: - anyOf: - - type: string - - "$ref": "#/definitions/MinMaxDatetime" - start_datetime: - anyOf: - - type: string - - "$ref": "#/definitions/MinMaxDatetime" - step: - type: string - end_time_option: - "$ref": "#/definitions/RequestOption" - lookback_window: - type: string - start_time_option: - "$ref": "#/definitions/RequestOption" - stream_state_field_end: - type: string - stream_state_field_start: - type: string - $options: - type: object - additionalProperties: true - DeclarativeOauth2Authenticator: - description: Authenticator for requests using Oauth authentication - type: object - required: - - type - - client_id - - client_secret - - refresh_token - - token_refresh_endpoint - properties: - type: - enum: [DeclarativeOauth2Authenticator] - client_id: - type: string - client_secret: - type: string - refresh_token: - type: string - token_refresh_endpoint: - type: string - access_token_name: - type: string - expires_in_name: - type: string - grant_type: - type: string - refresh_request_body: - type: object - additionalProperties: true - scopes: - type: array - items: - type: string - token_expiry_date: - type: string - DeclarativeStream: - description: A stream whose behavior is described by a set of declarative low code components - type: object - additionalProperties: true - required: - - type - - retriever - properties: - type: - enum: [DeclarativeStream] - retriever: - "$ref": "#/definitions/SimpleRetriever" - checkpoint_interval: - type: integer - name: - type: string - default: "" - primary_key: - anyOf: - - type: string - - type: array - items: - type: string - - type: array - items: - type: array - items: - type: string - default: "" - schema_loader: - anyOf: - - "$ref": "#/definitions/DefaultSchemaLoader" - - "$ref": "#/definitions/JsonFileSchemaLoader" - stream_cursor_field: - anyOf: - - type: string - - type: array - items: - - type: string - transformations: - type: array - items: - anyOf: - - "$ref": "#/definitions/AddFields" - - "$ref": "#/definitions/RemoveFields" - $options: - type: object - additional_properties: true - DefaultErrorHandler: - description: The default error handler. Default behavior includes only retrying server errors (HTTP 5XX) and too many requests (HTTP 429) with an exponential backoff - type: object - properties: - type: - enum: [DefaultErrorHandler] - backoff_strategies: - type: array - items: - anyOf: - - "$ref": "#/definitions/ConstantBackoffStrategy" - - "$ref": "#/definitions/CustomBackoffStrategy" - - "$ref": "#/definitions/ExponentialBackoffStrategy" - - "$ref": "#/definitions/WaitTimeFromHeaderBackoffStrategy" - - "$ref": "#/definitions/WaitUntilTimeFromHeaderBackoffStrategy" - max_retries: - type: integer - default: 5 - response_filters: - type: array - items: - "$ref": "#/definitions/HttpResponseFilter" - $options: - type: object - additional_properties: true - DefaultPaginator: - description: Default pagination implementation to request pages of results with a fixed size until the pagination strategy no longer returns a next_page_token - type: object - required: - - type - - pagination_strategy - - url_base - properties: - type: - enum: [DefaultPaginator] - pagination_strategy: - anyOf: - - "$ref": "#/definitions/CursorPagination" - - "$ref": "#/definitions/CustomPaginationStrategy" - - "$ref": "#/definitions/OffsetIncrement" - - "$ref": "#/definitions/PageIncrement" - url_base: - type: string - decoder: - "$ref": "#/definitions/JsonDecoder" - page_size_option: - "$ref": "#/definitions/RequestOption" - page_token_option: - "$ref": "#/definitions/RequestOption" - $options: - type: object - additionalProperties: true - DefaultSchemaLoader: - description: Loads a schema from the default location or returns an empty schema for streams that have not defined their schema file yet. - type: object - required: - - type - properties: - type: - enum: [DefaultSchemaLoader] - $options: - type: object - additionalProperties: true - DpathExtractor: - description: Record extractor that searches a decoded response over a path defined as an array of fields - type: object - required: - - type - - field_pointer - properties: - type: - enum: [DpathExtractor] - field_pointer: - type: array - items: - - type: string - decoder: - "$ref": "#/definitions/JsonDecoder" - $options: - type: object - additionalProperties: true - ExponentialBackoffStrategy: - description: Backoff strategy with an exponential backoff interval - type: object - required: - - type - properties: - type: - enum: [ExponentialBackoffStrategy] - factor: - anyOf: - - type: number - - type: string - default: 5 - $options: - type: object - additionalProperties: true - HttpRequester: - description: Default implementation of a requester - type: object - required: - - type - - name - - path - - url_base - properties: - type: - enum: [HttpRequester] - name: - type: string - path: - type: string - url_base: - type: string - authenticator: - anyOf: - - "$ref": "#/definitions/ApiKeyAuthenticator" - - "$ref": "#/definitions/BasicHttpAuthenticator" - - "$ref": "#/definitions/BearerAuthenticator" - - "$ref": "#/definitions/CustomAuthenticator" - - "$ref": "#/definitions/DeclarativeOauth2Authenticator" - - "$ref": "#/definitions/NoAuth" - error_handler: - anyOf: - - "$ref": "#/definitions/DefaultErrorHandler" - - "$ref": "#/definitions/CustomErrorHandler" - - "$ref": "#/definitions/CompositeErrorHandler" - http_method: - anyOf: - - type: string - - type: string - enum: - - GET - - POST - default: GET - request_options_provider: - "$ref": "#/definitions/InterpolatedRequestOptionsProvider" - $options: - type: object - additionalProperties: true - HttpResponseFilter: - description: A filter that is used to select on properties of the HTTP response received - type: object - required: - - action - properties: - type: - enum: [HttpResponseFilter] - action: - type: string - enum: - - SUCCESS - - FAIL - - RETRY - - IGNORE - error_message: - type: string - error_message_contains: - type: string - http_codes: - type: array - items: - type: integer - uniqueItems: true - predicate: - type: string - $options: - type: object - additionalProperties: true - InterpolatedRequestOptionsProvider: - description: Defines the request options to set on an outgoing HTTP request by evaluating `InterpolatedMapping`s - type: object - required: - - type - properties: - type: - enum: [InterpolatedRequestOptionsProvider] - request_body_data: - anyOf: - - type: string - - type: object - additionalProperties: - type: string - request_body_json: - anyOf: - - type: string - - type: object - additionalProperties: - type: string - request_headers: - anyOf: - - type: string - - type: object - additionalProperties: - type: string - request_parameters: - anyOf: - - type: string - - type: object - additionalProperties: - type: string - $options: - type: object - additionalProperties: true - JsonFileSchemaLoader: - description: Loads the schema from a json file - type: object - required: - - type - properties: - type: - enum: [JsonFileSchemaLoader] - file_path: - type: string - $options: - type: object - additionalProperties: true - JsonDecoder: - type: object - required: - - type - properties: - type: - enum: [JsonDecoder] - ListStreamSlicer: - description: Stream slicer that iterates over the values of a list - type: object - required: - - type - - cursor_field - - slice_values - properties: - type: - enum: [ListStreamSlicer] - cursor_field: - type: string - slice_values: - anyOf: - - type: string - - type: array - items: - type: string - request_option: - "$ref": "#/definitions/RequestOption" - $options: - type: object - additionalProperties: true - MinMaxDatetime: - description: Compares the provided date against optional minimum or maximum times. The max_datetime serves as the ceiling and will be returned when datetime exceeds it. The min_datetime serves as the floor - type: object - required: - - type - - datetime - properties: - type: - enum: [MinMaxDatetime] - datetime: - type: string - datetime_format: - type: string - default: "" - max_datetime: - type: string - min_datetime: - type: string - $options: - type: object - additionalProperties: true - NoAuth: - description: Authenticator for requests requiring no authentication - type: object - required: - - type - properties: - type: - enum: [NoAuth] - $options: - type: object - additionalProperties: true - NoPagination: - description: Pagination implementation that never returns a next page - type: object - required: - - type - properties: - type: - enum: [NoPagination] - OffsetIncrement: - description: Pagination strategy that returns the number of records reads so far and returns it as the next page token - type: object - required: - - type - - page_size - properties: - type: - enum: [OffsetIncrement] - page_size: - type: integer - $options: - type: object - additionalProperties: true - PageIncrement: - description: Pagination strategy that returns the number of pages reads so far and returns it as the next page token - type: object - required: - - type - - page_size - properties: - type: - enum: [PageIncrement] - page_size: - type: integer - start_from_page: - type: integer - default: 0 - $options: - type: object - additionalProperties: true - ParentStreamConfig: - description: Describes how to create a stream slice from a parent stream - type: object - required: - - type - - parent_key - - stream - - stream_slice_field - properties: - type: - enum: [ParentStreamConfig] - parent_key: - type: string - stream: - "$ref": "#/definitions/DeclarativeStream" - stream_slice_field: - type: string - request_option: - "$ref": "#/definitions/RequestOption" - $options: - type: object - additionalProperties: true - RecordFilter: - description: Filter applied on a list of Records - type: object - required: - - type - properties: - type: - enum: [RecordFilter] - backoff_time_in_seconds: - type: string - default: "" - $options: - type: object - additionalProperties: true - RecordSelector: - description: Responsible for translating an HTTP response into a list of records by extracting records from the response and optionally filtering records based on a heuristic. - type: object - required: - - type - - extractor - properties: - type: - enum: [RecordSelector] - extractor: - anyOf: - - "$ref": "#/definitions/CustomRecordExtractor" - - "$ref": "#/definitions/DpathExtractor" - record_filter: - "$ref": "#/definitions/RecordFilter" - $options: - type: object - additionalProperties: true - RemoveFields: - description: A transformation which removes fields from a record. The fields removed are designated using FieldPointers. During transformation, if a field or any of its parents does not exist in the record, no error is thrown. - type: object - required: - - type - - field_pointers - properties: - type: - enum: [RemoveFields] - field_pointers: - type: array - items: - items: - type: string - RequestOption: - description: Describes an option to set on a request - type: object - required: - - type - - inject_into - properties: - type: - enum: [RequestOption] - inject_into: - enum: - - request_parameter - - header - - path - - body_data - - body_json - field_name: - type: string - SimpleRetriever: - description: Retrieves records by synchronously sending requests to fetch records. The retriever acts as an orchestrator between the requester, the record selector, the paginator, and the stream slicer. - type: object - required: - - type - - record_selector - - requester - properties: - type: - enum: [SimpleRetriever] - record_selector: - "$ref": "#/definitions/RecordSelector" - requester: - "$ref": "#/definitions/HttpRequester" - name: - type: string - default: "" - paginator: - anyOf: - - "$ref": "#/definitions/DefaultPaginator" - - "$ref": "#/definitions/NoPagination" - primary_key: - anyOf: - - type: string - - type: array - items: - type: string - - type: array - items: - type: array - items: - type: string - default: "" - stream_slicer: - anyOf: - - "$ref": "#/definitions/CartesianProductStreamSlicer" - - "$ref": "#/definitions/CustomStreamSlicer" - - "$ref": "#/definitions/DatetimeStreamSlicer" - - "$ref": "#/definitions/ListStreamSlicer" - - "$ref": "#/definitions/SingleSlice" - - "$ref": "#/definitions/SubstreamSlicer" - $options: - type: object - additionalProperties: true - SingleSlice: - description: Stream slicer returning only a single stream slice - type: object - required: - - type - properties: - type: - enum: [SingleSlice] - $options: - type: object - additionalProperties: true - Spec: - description: A connection specification made up of information about the connector and how it can be configured - type: object - required: - - type - - connection_specification - - documentation_url - properties: - type: - enum: [Spec] - connection_specification: - type: object - additionalProperties: true - documentation_url: - type: string - SubstreamSlicer: - description: Stream slicer that iterates over the parent's stream slices and records and emits slices by interpolating the slice_definition mapping - type: object - required: - - type - - parent_stream_configs - properties: - type: - enum: [SubstreamSlicer] - parent_stream_configs: - type: array - items: - "$ref": "#/definitions/ParentStreamConfig" - $options: - type: object - additionalProperties: true - WaitTimeFromHeaderBackoffStrategy: - description: Extract wait time from http header - type: object - required: - - type - - header - properties: - type: - enum: [WaitTimeFromHeaderBackoffStrategy] - header: - type: string - regex: - type: string - $options: - type: object - additionalProperties: true - WaitUntilTimeFromHeaderBackoffStrategy: - description: Extract time at which we can retry the request from response header and wait for the difference between now and that time - type: object - required: - - type - - header - properties: - type: - enum: [WaitUntilTimeFromHeaderBackoffStrategy] - header: - type: string - min_wait: - anyOf: - - type: number - - type: string - regex: - type: string - $options: - type: object - additionalProperties: true diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/__init__.py index 986458e0779a..de07af17cd15 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/__init__.py @@ -1,3 +1,2 @@ # generated by generate-component-manifest-files from .declarative_component_schema import * -from .low_code_component_schema import * diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index aff43c54c4a5..e2153536819c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -126,8 +126,8 @@ class Config: _options: Optional[Dict[str, Any]] = Field(None, alias="$options") -class DeclarativeOauth2Authenticator(BaseModel): - type: Literal["DeclarativeOauth2Authenticator"] +class OAuthAuthenticator(BaseModel): + type: Literal["OAuthAuthenticator"] client_id: str client_secret: str refresh_token: str @@ -399,7 +399,7 @@ class HttpRequester(BaseModel): BasicHttpAuthenticator, BearerAuthenticator, CustomAuthenticator, - DeclarativeOauth2Authenticator, + OAuthAuthenticator, NoAuth, SessionTokenAuthenticator, ]