From 9fb95ecbf283eb10b934e9ac66938764abdd2189 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 1 Jun 2021 00:51:08 +0300 Subject: [PATCH 01/60] Added spec.json --- .../source_paypal_transaction/spec.json | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json new file mode 100644 index 000000000000..229ac90c7f4c --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json @@ -0,0 +1,29 @@ +{ + "documentationUrl": "https://developer.paypal.com/docs/api/transaction-search/v1/", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Paypal Transaction Search", + "type": "object", + "required": ["client_id", "secret", "start_date"], + "additionalProperties": false, + "properties": { + "client_id": { + "title": "Client ID", + "type": "string", + "description": "The Paypal Client ID for API credentials", + }, + "secret": { + "title": "Secret", + "type": "string", + "description": "The Secret for a given Client ID.", + "airbyte_secret": true + }, + "start_date": { + "type": "string", + "title": "Start Date", + "description": "Start Date for data extraction in Internet date and time format https://datatracker.ietf.org/doc/html/rfc3339#section-5.6", + "examples": ["2021-06-11T23:59:59-0700"] + } + } + } +} From 1fb7d36181d9c14785b273da6d5417e0f33df70b Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 1 Jun 2021 00:51:37 +0300 Subject: [PATCH 02/60] Initialization --- .../source-paypal-transaction/.dockerignore | 7 + .../source-paypal-transaction/Dockerfile | 15 ++ .../source-paypal-transaction/README.md | 129 ++++++++++ .../acceptance-test-config.yml | 30 +++ .../acceptance-test-docker.sh | 7 + .../source-paypal-transaction/build.gradle | 14 ++ .../integration_tests/__init__.py | 0 .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 36 +++ .../integration_tests/catalog.json | 39 +++ .../integration_tests/configured_catalog.json | 56 +++++ .../integration_tests/invalid_config.json | 3 + .../integration_tests/sample_config.json | 3 + .../integration_tests/sample_state.json | 5 + .../source-paypal-transaction/main.py | 33 +++ .../requirements.txt | 2 + .../source-paypal-transaction/setup.py | 48 ++++ .../source_paypal_transaction/__init__.py | 27 ++ .../source_paypal_transaction/schemas/TODO.md | 25 ++ .../schemas/customers.json | 16 ++ .../schemas/employees.json | 19 ++ .../source_paypal_transaction/source.py | 230 ++++++++++++++++++ .../unit_tests/unit_test.py | 27 ++ 23 files changed, 776 insertions(+) create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/.dockerignore create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/Dockerfile create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/README.md create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/build.gradle create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/main.py create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/requirements.txt create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/setup.py create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/__init__.py create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/customers.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/employees.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py diff --git a/airbyte-integrations/connectors/source-paypal-transaction/.dockerignore b/airbyte-integrations/connectors/source-paypal-transaction/.dockerignore new file mode 100644 index 000000000000..7d3fd691a105 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_paypal_transaction +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile b/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile new file mode 100644 index 000000000000..ccdfa2b4a372 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.7-slim + +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +WORKDIR /airbyte/integration_code +COPY source_paypal_transaction ./source_paypal_transaction +COPY main.py ./ +COPY setup.py ./ +RUN pip install . + +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-paypal-transaction diff --git a/airbyte-integrations/connectors/source-paypal-transaction/README.md b/airbyte-integrations/connectors/source-paypal-transaction/README.md new file mode 100644 index 000000000000..4b34bbf50cfe --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/README.md @@ -0,0 +1,129 @@ +# Paypal Transaction Source + +This is the repository for the Paypal Transaction source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/paypal-transaction). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-paypal-transaction:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/paypal-transaction) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_paypal_transaction/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source paypal-transaction test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-paypal-transaction:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-paypal-transaction:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-paypal-transaction:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paypal-transaction:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paypal-transaction:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-paypal-transaction:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](source-acceptance-tests.md) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-paypal-transaction:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-paypal-transaction:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml new file mode 100644 index 000000000000..e38b0cf2c95b --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/source-acceptance-tests.md) +# for more information about how to configure these tests +connector_image: airbyte/source-paypal-transaction:dev +tests: + spec: + - spec_path: "source_paypal_transaction/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "exception" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + validate_output_from_all_streams: yes +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.txt" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: # TODO if your connector does not implement incremental sync, remove this block + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-docker.sh new file mode 100644 index 000000000000..1425ff74f151 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-docker.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input diff --git a/airbyte-integrations/connectors/source-paypal-transaction/build.gradle b/airbyte-integrations/connectors/source-paypal-transaction/build.gradle new file mode 100644 index 000000000000..4a6226795c51 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/build.gradle @@ -0,0 +1,14 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_paypal_transaction' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/__init__.py b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py new file mode 100644 index 000000000000..eeb4a2d3e02e --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py @@ -0,0 +1,36 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """ This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/catalog.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/catalog.json new file mode 100644 index 000000000000..6799946a6851 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/catalog.json @@ -0,0 +1,39 @@ +{ + "streams": [ + { + "name": "TODO fix this file", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": "column1", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + }, + { + "name": "table1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..74de5e17e466 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json @@ -0,0 +1,56 @@ +// TODO: Construct a configured catalog that can be used for testing. Each stream's `json_schema` field should match the corresponding json schema file. +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "signup_date": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "employees", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "years_of_service": { + "type": ["null", "integer"] + }, + "start_date": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json new file mode 100644 index 000000000000..f3732995784f --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "todo-wrong-field": "this should be an incomplete config file, used in standard tests" +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json new file mode 100644 index 000000000000..ecc4913b84c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/main.py b/airbyte-integrations/connectors/source-paypal-transaction/main.py new file mode 100644 index 000000000000..881549ef9405 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/main.py @@ -0,0 +1,33 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_paypal_transaction import SourcePaypalTransaction + +if __name__ == "__main__": + source = SourcePaypalTransaction() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/requirements.txt b/airbyte-integrations/connectors/source-paypal-transaction/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-paypal-transaction/setup.py b/airbyte-integrations/connectors/source-paypal-transaction/setup.py new file mode 100644 index 000000000000..4a62e6df834d --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/setup.py @@ -0,0 +1,48 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "source-acceptance-test", +] + +setup( + name="source_paypal_transaction", + description="Source implementation for Paypal Transaction.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/__init__.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/__init__.py new file mode 100644 index 000000000000..403d505dcb87 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/__init__.py @@ -0,0 +1,27 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .source import SourcePaypalTransaction + +__all__ = ["SourcePaypalTransaction"] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/TODO.md b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/TODO.md new file mode 100644 index 000000000000..cf1efadb3c9c --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/TODO.md @@ -0,0 +1,25 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. + +The schema of a stream is the return value of `Stream.get_json_schema`. + +## Static schemas +By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. + +Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. + +## Dynamic schemas +If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). + +## Dynamically modifying static schemas +Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: +``` +def get_json_schema(self): + schema = super().get_json_schema() + schema['dynamically_determined_property'] = "property" + return schema +``` + +Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/customers.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/customers.json new file mode 100644 index 000000000000..9a4b13485836 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/customers.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "signup_date": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/employees.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/employees.json new file mode 100644 index 000000000000..2fa01a0fa1ff --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/employees.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "years_of_service": { + "type": ["null", "integer"] + }, + "start_date": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py new file mode 100644 index 000000000000..d42066a8c7b7 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -0,0 +1,230 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + +""" +TODO: Most comments in this class are instructive and should be deleted after the source is implemented. + +This file provides a stubbed example of how to use the Airbyte CDK to develop both a source connector which supports full refresh or and an +incremental syncs from an HTTP API. + +The various TODOs are both implementation hints and steps - fulfilling all the TODOs should be sufficient to implement one basic and one incremental +stream from a source. This pattern is the same one used by Airbyte internally to implement connectors. + +The approach here is not authoritative, and devs are free to use their own judgement. + +There are additional required TODOs in the files within the integration_tests folder and the spec.json file. +""" + + +# Basic full refresh stream +class PaypalTransactionStream(HttpStream, ABC): + """ + TODO remove this comment + + This class represents a stream output by the connector. + This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, + parsing responses etc.. + + Each stream should extend this class (or another abstract subclass of it) to specify behavior unique to that stream. + + Typically for REST APIs each stream corresponds to a resource in the API. For example if the API + contains the endpoints + - GET v1/customers + - GET v1/employees + + then you should have three classes: + `class PaypalTransactionStream(HttpStream, ABC)` which is the current class + `class Customers(PaypalTransactionStream)` contains behavior to pull data for customers using v1/customers + `class Employees(PaypalTransactionStream)` contains behavior to pull data for employees using v1/employees + + If some streams implement incremental sync, it is typical to create another class + `class IncrementalPaypalTransactionStream((PaypalTransactionStream), ABC)` then have concrete stream implementations extend it. An example + is provided below. + + See the reference docs for the full list of configurable options. + """ + + # TODO: Fill in the url base. Required. + url_base = "https://example-api.com/v1/" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. + + This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed + to most other methods in this class to help you form headers, request bodies, query params, etc.. + + For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a + 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. + The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. + + :param response: the most recent response from the API + :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. + If there are no more pages in the result, return None. + """ + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + """ + TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. + Usually contains common params e.g. pagination size etc. + """ + return {} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + TODO: Override this method to define how a response is parsed. + :return an iterable containing each record in the response + """ + yield {} + + +class Customers(PaypalTransactionStream): + """ + TODO: Change class name to match the table/data source this stream corresponds to. + """ + + # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. + primary_key = "customer_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + should return "customers". Required. + """ + return "customers" + + +# Basic incremental stream +class IncrementalPaypalTransactionStream(PaypalTransactionStream, ABC): + """ + TODO fill in details of this class to implement functionality related to incremental syncs for your connector. + if you do not need to implement incremental sync for any streams, remove this class. + """ + + # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. + state_checkpoint_interval = None + + @property + def cursor_field(self) -> str: + """ + TODO + Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is + usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. + + :return str: The name of the cursor field. + """ + return [] + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and + the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. + """ + return {} + + +class Employees(IncrementalPaypalTransactionStream): + """ + TODO: Change class name to match the table/data source this stream corresponds to. + """ + + # TODO: Fill in the cursor_field. Required. + cursor_field = "start_date" + + # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. + primary_key = "employee_id" + + def path(self, **kwargs) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/employees then this should + return "single". Required. + """ + return "employees" + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + """ + TODO: Optionally override this method to define this stream's slices. If slicing is not needed, delete this method. + + Slices control when state is saved. Specifically, state is saved after a slice has been fully read. + This is useful if the API offers reads by groups or filters, and can be paired with the state object to make reads efficient. See the "concepts" + section of the docs for more information. + + The function is called before reading any records in a stream. It returns an Iterable of dicts, each containing the + necessary data to craft a request for a slice. The stream state is usually referenced to determine what slices need to be created. + This means that data in a slice is usually closely related to a stream's cursor_field and stream_state. + + An HTTP request is made for each returned slice. The same slice can be accessed in the path, request_params and request_header functions to help + craft that specific request. + + For example, if https://example-api.com/v1/employees offers a date query params that returns data for that particular day, one way to implement + this would be to consult the stream state object for the last synced date, then return a slice containing each date from the last synced date + till now. The request_params function would then grab the date from the stream_slice and make it part of the request by injecting it into + the date query param. + """ + raise NotImplementedError("Implement stream slices or delete this method!") + + +# Source +class SourcePaypalTransaction(AbstractSource): + + url_base = "https://api-m.sandbox.paypal.com" + # url_base = "https://api-m.paypal.com" + + def check_connection(self, logger, config) -> Tuple[bool, any]: + """ + TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API + + See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 + for an example. + + :param config: the user-input config object conforming to the connector's spec.json + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + return True, None + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + TODO: Replace the streams below with your own streams. + + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + # TODO remove the authenticator if not required. + auth = TokenAuthenticator(token="api_key") # Oauth2Authenticator is also available if you need oauth support + return [Customers(authenticator=auth), Employees(authenticator=auth)] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py new file mode 100644 index 000000000000..b8a8150b507f --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py @@ -0,0 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +def test_example_method(): + assert True From 311f3d460f545fb188340e928e6d5d46d6a6fbfa Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 1 Jun 2021 02:54:22 +0300 Subject: [PATCH 03/60] added oauth2 autorization --- .../source_paypal_transaction/source.py | 133 +++++++++--------- 1 file changed, 64 insertions(+), 69 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index d42066a8c7b7..2274782edeeb 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -30,7 +30,7 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator, Oauth2Authenticator """ TODO: Most comments in this class are instructive and should be deleted after the source is implemented. @@ -75,8 +75,8 @@ class PaypalTransactionStream(HttpStream, ABC): See the reference docs for the full list of configurable options. """ - # TODO: Fill in the url base. Required. - url_base = "https://example-api.com/v1/" + url_base = "https://api-m.sandbox.paypal.com/v1/reporting/" + # url_base = "https://api-m.paypal.com/v1/reporting/" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """ @@ -112,100 +112,87 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp yield {} -class Customers(PaypalTransactionStream): +class Transactions(PaypalTransactionStream): """ - TODO: Change class name to match the table/data source this stream corresponds to. + Stream for Transactions /v1/reporting/transactions """ - # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. - primary_key = "customer_id" + primary_key = "transaction_id" def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this should return "customers". Required. """ - return "customers" - - -# Basic incremental stream -class IncrementalPaypalTransactionStream(PaypalTransactionStream, ABC): - """ - TODO fill in details of this class to implement functionality related to incremental syncs for your connector. - if you do not need to implement incremental sync for any streams, remove this class. - """ - - # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. - state_checkpoint_interval = None - @property - def cursor_field(self) -> str: - """ - TODO - Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is - usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. - - :return str: The name of the cursor field. - """ - return [] - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and - the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. - """ - return {} + return "transactions" -class Employees(IncrementalPaypalTransactionStream): +class Balances(PaypalTransactionStream): """ - TODO: Change class name to match the table/data source this stream corresponds to. + Stream for Balances /v1/reporting/balances """ - # TODO: Fill in the cursor_field. Required. - cursor_field = "start_date" - # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. - primary_key = "employee_id" + primary_key = "customer_id" - def path(self, **kwargs) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/employees then this should - return "single". Required. + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: """ - return "employees" - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + should return "customers". Required. """ - TODO: Optionally override this method to define this stream's slices. If slicing is not needed, delete this method. - Slices control when state is saved. Specifically, state is saved after a slice has been fully read. - This is useful if the API offers reads by groups or filters, and can be paired with the state object to make reads efficient. See the "concepts" - section of the docs for more information. + return "balances" - The function is called before reading any records in a stream. It returns an Iterable of dicts, each containing the - necessary data to craft a request for a slice. The stream state is usually referenced to determine what slices need to be created. - This means that data in a slice is usually closely related to a stream's cursor_field and stream_state. - An HTTP request is made for each returned slice. The same slice can be accessed in the path, request_params and request_header functions to help - craft that specific request. +class PayPalOauth2Authenticator(Oauth2Authenticator): + """ + curl -v POST https://api-m.sandbox.paypal.com/v1/oauth2/token \ + -H "Accept: application/json" \ + -H "Accept-Language: en_US" \ + -u "CLIENT_ID:SECRET" \ + -d "grant_type=client_credentials" - For example, if https://example-api.com/v1/employees offers a date query params that returns data for that particular day, one way to implement - this would be to consult the stream state object for the last synced date, then return a slice containing each date from the last synced date - till now. The request_params function would then grab the date from the stream_slice and make it part of the request by injecting it into - the date query param. - """ - raise NotImplementedError("Implement stream slices or delete this method!") + """ + def get_refresh_request_body(self) -> Mapping[str, Any]: + """ Override to define additional parameters """ + payload: MutableMapping[str, Any] = { + "grant_type": "client_credentials" + } + return payload + + def refresh_access_token(self) -> Tuple[str, int]: + """ + returns a tuple of (access_token, token_lifespan_in_seconds) + """ + try: + data = "grant_type=client_credentials" + headers = { + 'Accept': 'application/json', + 'Accept-Language': 'en_US' + } + auth = (self.client_id, self.client_secret) + response = requests.request( + method="POST", + url=self.token_refresh_endpoint, + data=data, + headers=headers, + auth=auth) + response.raise_for_status() + response_json = response.json() + print(response_json) + return response_json["access_token"], response_json["expires_in"] + except Exception as e: + raise Exception(f"Error while refreshing access token: {e}") from e # Source class SourcePaypalTransaction(AbstractSource): - url_base = "https://api-m.sandbox.paypal.com" - # url_base = "https://api-m.paypal.com" - def check_connection(self, logger, config) -> Tuple[bool, any]: """ TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API @@ -217,6 +204,13 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ + token = PayPalOauth2Authenticator( + token_refresh_endpoint='https://api-m.sandbox.paypal.com/v1/oauth2/token', + client_id=config["client_id"], + client_secret=config["secret"], + refresh_token='').get_access_token() + + print(f"token {token}") return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: @@ -227,4 +221,5 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ # TODO remove the authenticator if not required. auth = TokenAuthenticator(token="api_key") # Oauth2Authenticator is also available if you need oauth support - return [Customers(authenticator=auth), Employees(authenticator=auth)] + # return [Transactions(authenticator=auth), Balances(authenticator=auth)] + return [Transactions(authenticator=auth)] From 30d0c401a2adfe3daf5925a933b6cb95051ef91e Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 3 Jun 2021 12:16:52 +0300 Subject: [PATCH 04/60] added spec, check, discover + catalogs/configurared_catalogs --- .../integration_tests/configured_catalog.json | 55 +++++++++++++------ .../configured_catalog_transaction.json | 29 ++++++++++ .../schemas/balances.json | 38 +++++++++++++ .../schemas/transactions.json | 17 ++++++ .../source_paypal_transaction/source.py | 16 ++++-- .../source_paypal_transaction/spec.json | 2 +- 6 files changed, 133 insertions(+), 24 deletions(-) create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json index 74de5e17e466..19ebd0f37f06 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json @@ -1,22 +1,22 @@ -// TODO: Construct a configured catalog that can be used for testing. Each stream's `json_schema` field should match the corresponding json schema file. { "streams": [ { "stream": { - "name": "customers", + "name": "transactions", "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "id": { + "account_number": { "type": ["null", "string"] }, - "name": { + "last_refreshed_datetime": { "type": ["null", "string"] }, - "signup_date": { - "type": ["null", "string"], - "format": "date-time" + "page": { + "type": ["null", "integer"] + }, + "total_items": { + "type": ["null", "integer"] } } }, @@ -29,27 +29,46 @@ "stream": { "name": "employees", "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { + "account_id": { "type": ["null", "string"] }, - "years_of_service": { - "type": ["null", "integer"] + "as_of_time": { + "type": ["null", "string"], + "format": "date-time" }, - "start_date": { + "last_refresh_time": { "type": ["null", "string"], "format": "date-time" + }, + "balance": { + "type": ["object", "null"], + "properties": { + "currency": { + "type": ["null", "string"] + }, + "primary": { + "type": ["boolean"] + }, + "total_balance": { + "type": ["object", "null"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + } } } }, - "supported_sync_modes": ["full_refresh", "incremental"] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", + "sync_mode": "full_refresh", "destination_sync_mode": "append" } ] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json new file mode 100644 index 000000000000..1924ce3bd1a9 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json @@ -0,0 +1,29 @@ +{ + "streams": [ + { + "stream": { + "name": "transactions", + "json_schema": { + "type": "object", + "properties": { + "account_number": { + "type": ["null", "string"] + }, + "last_refreshed_datetime": { + "type": ["null", "string"] + }, + "page": { + "type": ["null", "integer"] + }, + "total_items": { + "type": ["null", "integer"] + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json new file mode 100644 index 000000000000..69c6b92d6e80 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json @@ -0,0 +1,38 @@ +{ + "type": "object", + "properties": { + "account_id": { + "type": ["null", "string"] + }, + "as_of_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "last_refresh_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "balance": { + "type": ["object", "null"], + "properties": { + "currency": { + "type": ["null", "string"] + }, + "primary": { + "type": ["boolean"] + }, + "total_balance": { + "type": ["object", "null"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json new file mode 100644 index 000000000000..981c82a6a552 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json @@ -0,0 +1,17 @@ +{ + "type": "object", + "properties": { + "account_number": { + "type": ["null", "string"] + }, + "last_refreshed_datetime": { + "type": ["null", "string"] + }, + "page": { + "type": ["null", "integer"] + }, + "total_items": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 2274782edeeb..26263f9c6001 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -117,7 +117,7 @@ class Transactions(PaypalTransactionStream): Stream for Transactions /v1/reporting/transactions """ - primary_key = "transaction_id" + primary_key = "last_refreshed_datetime" def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None @@ -136,7 +136,7 @@ class Balances(PaypalTransactionStream): """ # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. - primary_key = "customer_id" + primary_key = "as_of_time" def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None @@ -209,6 +209,8 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: client_id=config["client_id"], client_secret=config["secret"], refresh_token='').get_access_token() + if not token: + return False, 'Unable to fetch Paypal API token due to incorrect client_id or secret' print(f"token {token}") return True, None @@ -219,7 +221,11 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: :param config: A Mapping of the user input configuration as defined in the connector spec. """ - # TODO remove the authenticator if not required. - auth = TokenAuthenticator(token="api_key") # Oauth2Authenticator is also available if you need oauth support + token = PayPalOauth2Authenticator( + token_refresh_endpoint='https://api-m.sandbox.paypal.com/v1/oauth2/token', + client_id=config["client_id"], + client_secret=config["secret"], + refresh_token='').get_access_token() + auth = TokenAuthenticator(token=token) # Oauth2Authenticator is also available if you need oauth support # return [Transactions(authenticator=auth), Balances(authenticator=auth)] - return [Transactions(authenticator=auth)] + return [Transactions(authenticator=auth), Balances(authenticator=auth)] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json index 229ac90c7f4c..5cd95638bfac 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json @@ -10,7 +10,7 @@ "client_id": { "title": "Client ID", "type": "string", - "description": "The Paypal Client ID for API credentials", + "description": "The Paypal Client ID for API credentials" }, "secret": { "title": "Secret", From ecd06afb2b502d3e02b712c298cedcb05da60068 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Fri, 4 Jun 2021 12:27:07 +0300 Subject: [PATCH 05/60] updated request_params --- .../source_paypal_transaction/source.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 26263f9c6001..822d42fe156a 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -25,7 +25,7 @@ from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple - +import datetime import requests from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -102,14 +102,20 @@ def request_params( TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. Usually contains common params e.g. pagination size etc. """ - return {} + return { + 'start_date': '2021-05-15T00:00:00-0700', + 'end_date': '2021-06-15T00:00:00-0700', + 'fields': 'all', + 'page_size': '100' + } def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: """ TODO: Override this method to define how a response is parsed. :return an iterable containing each record in the response """ - yield {} + #yield {} + return [response.json()] class Transactions(PaypalTransactionStream): @@ -228,4 +234,6 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: refresh_token='').get_access_token() auth = TokenAuthenticator(token=token) # Oauth2Authenticator is also available if you need oauth support # return [Transactions(authenticator=auth), Balances(authenticator=auth)] - return [Transactions(authenticator=auth), Balances(authenticator=auth)] + #start_date = datetime.strptime(config["start_date"], "%Y-%m-%d") + #return [Transactions(authenticator=auth), Balances(authenticator=auth)] + return [Transactions(authenticator=auth)] From 51078e398c14f7ce55ddf542ab07b3a225224e9f Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Mon, 7 Jun 2021 20:00:44 +0300 Subject: [PATCH 06/60] added paging, slicing (1d) --- .../source_paypal_transaction/source.py | 94 ++++++++++++++----- 1 file changed, 73 insertions(+), 21 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 822d42fe156a..1735aa3f5876 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -1,7 +1,7 @@ # # MIT License # -# Copyright (c) 2020 Airbyte +# Copyright (c) 2021 Airbyte # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -21,16 +21,18 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # - +import logging +logging.basicConfig(level=logging.DEBUG) from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -import datetime +from datetime import datetime, timedelta + import requests from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator, Oauth2Authenticator +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator, Oauth2Authenticator, HttpAuthenticator """ TODO: Most comments in this class are instructive and should be deleted after the source is implemented. @@ -93,7 +95,13 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. If there are no more pages in the result, return None. """ - return None + decoded_response = response.json() + total_pages = decoded_response.get('total_pages') + page_number = decoded_response.get('page') + if page_number >= total_pages: + return None + else: + return {"page": page_number+1} def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None @@ -102,11 +110,22 @@ def request_params( TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. Usually contains common params e.g. pagination size etc. """ + page_number = 1 + if next_page_token: + page_number = next_page_token.get('page') + + print(f'stream_state {stream_state}') + print(f'stream_slice {stream_slice}') + + start_date = stream_slice['date'] + end_date_dt = datetime.fromisoformat(start_date) + timedelta(days=self.stream_size_in_days) + end_date = end_date_dt.isoformat() return { - 'start_date': '2021-05-15T00:00:00-0700', - 'end_date': '2021-06-15T00:00:00-0700', + 'start_date': start_date, + 'end_date': end_date, 'fields': 'all', - 'page_size': '100' + 'page_size': '1', + 'page': page_number } def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: @@ -114,26 +133,61 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp TODO: Override this method to define how a response is parsed. :return an iterable containing each record in the response """ - #yield {} - return [response.json()] + json_response = response.json() + records = json_response.get(self.data_field, []) if self.data_field is not None else json_response + yield from records class Transactions(PaypalTransactionStream): """ Stream for Transactions /v1/reporting/transactions """ + data_field = "transaction_details" + primary_key = "transaction_id" + cursor_field = "date" + stream_size_in_days = 1 - primary_key = "last_refreshed_datetime" + def __init__(self, start_date: datetime, **kwargs): + super().__init__(**kwargs) + self.start_date = start_date def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: + return "transactions" + + # def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: + # # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state + # # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. + # if current_stream_state is not None and 'date' in current_stream_state: + # current_parsed_date = datetime.strptime(current_stream_state['date'], '%Y-%m-%d') + # latest_record_date = datetime.strptime(latest_record['date'], '%Y-%m-%d') + # return {'date': max(current_parsed_date, latest_record_date).strftime('%Y-%m-%d')} + # else: + # return {'date': self.start_date.strftime('%Y-%m-%d')} + + def _chunk_date_range(self, start_date: datetime) -> List[Mapping[str, any]]: """ - Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this - should return "customers". Required. + Returns a list of each day between the start date and now. + The return value is a list of dicts {'date': date_string}. """ + dates = [] + while start_date < datetime.now().astimezone(): + dates.append({'date': start_date.isoformat()}) + start_date += timedelta(days=self.stream_size_in_days) - return "transactions" + print(f'dates {dates}') + return dates + + def stream_slices( + self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, any]]]: + + start_date = self.start_date + if stream_state and 'date' in stream_state: + start_date = datetime.fromisoformat(stream_state['date']) + + return self._chunk_date_range(start_date) class Balances(PaypalTransactionStream): @@ -141,17 +195,14 @@ class Balances(PaypalTransactionStream): Stream for Balances /v1/reporting/balances """ + data_field = "transaction_details" + # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. primary_key = "as_of_time" def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this - should return "customers". Required. - """ - return "balances" @@ -234,6 +285,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: refresh_token='').get_access_token() auth = TokenAuthenticator(token=token) # Oauth2Authenticator is also available if you need oauth support # return [Transactions(authenticator=auth), Balances(authenticator=auth)] - #start_date = datetime.strptime(config["start_date"], "%Y-%m-%d") + print(f'TOKEN: {token}') + start_date = datetime.strptime(config["start_date"], "%Y-%m-%d").astimezone() #return [Transactions(authenticator=auth), Balances(authenticator=auth)] - return [Transactions(authenticator=auth)] + return [Transactions(authenticator=auth, start_date=start_date)] From 496d605c1716564e6d903ec980798ada1796ef6c Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Mon, 7 Jun 2021 21:51:58 +0300 Subject: [PATCH 07/60] Use oath2 for paypal --- .../source_paypal_transaction/source.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 1735aa3f5876..dbbb0880e482 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -213,7 +213,6 @@ class PayPalOauth2Authenticator(Oauth2Authenticator): -H "Accept-Language: en_US" \ -u "CLIENT_ID:SECRET" \ -d "grant_type=client_credentials" - """ def get_refresh_request_body(self) -> Mapping[str, Any]: """ Override to define additional parameters """ @@ -241,13 +240,11 @@ def refresh_access_token(self) -> Tuple[str, int]: auth=auth) response.raise_for_status() response_json = response.json() - print(response_json) return response_json["access_token"], response_json["expires_in"] except Exception as e: raise Exception(f"Error while refreshing access token: {e}") from e -# Source class SourcePaypalTransaction(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: @@ -269,7 +266,6 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: if not token: return False, 'Unable to fetch Paypal API token due to incorrect client_id or secret' - print(f"token {token}") return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: @@ -278,14 +274,11 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: :param config: A Mapping of the user input configuration as defined in the connector spec. """ - token = PayPalOauth2Authenticator( + authenticator = PayPalOauth2Authenticator( token_refresh_endpoint='https://api-m.sandbox.paypal.com/v1/oauth2/token', client_id=config["client_id"], client_secret=config["secret"], - refresh_token='').get_access_token() - auth = TokenAuthenticator(token=token) # Oauth2Authenticator is also available if you need oauth support - # return [Transactions(authenticator=auth), Balances(authenticator=auth)] - print(f'TOKEN: {token}') + refresh_token='') start_date = datetime.strptime(config["start_date"], "%Y-%m-%d").astimezone() #return [Transactions(authenticator=auth), Balances(authenticator=auth)] - return [Transactions(authenticator=auth, start_date=start_date)] + return [Transactions(authenticator=authenticator, start_date=start_date)] From a4b40e92f2b4f2bf02b72ca0ab388fa1e7b5e69e Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 10 Jun 2021 12:06:08 +0300 Subject: [PATCH 08/60] incremental sync, acceptance test --- .../acceptance-test-config.yml | 2 +- .../configured_catalog_transaction.json | 4 +- .../integration_tests/invalid_config.json | 6 +- .../integration_tests/sample_config.json | 6 +- .../integration_tests/sample_state.json | 4 +- .../source_paypal_transaction/source.py | 122 ++++++++++-------- 6 files changed, 78 insertions(+), 66 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml index e38b0cf2c95b..08d0e07e3577 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml @@ -8,7 +8,7 @@ tests: - config_path: "secrets/config.json" status: "succeed" - config_path: "integration_tests/invalid_config.json" - status: "exception" + status: "failed" discovery: - config_path: "secrets/config.json" basic_read: diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json index 1924ce3bd1a9..23ec64e5a5cb 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json @@ -20,9 +20,9 @@ } } }, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "full_refresh", + "sync_mode": "incremental", "destination_sync_mode": "overwrite" } ] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json index f3732995784f..52839c5ac77e 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json @@ -1,3 +1,5 @@ { - "todo-wrong-field": "this should be an incomplete config file, used in standard tests" -} + "client_id": "AWAz___", + "secret": "ENC8__", + "start_date": "2021-06-01" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json index ecc4913b84c7..fff6d380f653 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json @@ -1,3 +1,5 @@ { - "fix-me": "TODO" -} + "client_id": "AWAz0aQCdk", + "secret": "ENC8PBtgHBH-", + "start_date": "2021-06-01" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json index 3587e579822d..dd461b09ceb6 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json @@ -1,5 +1,5 @@ { - "todo-stream-name": { - "todo-field-name": "value" + "transactions": { + "date": "2021-06-04T17:34:43+00:00" } } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index dbbb0880e482..45040b04014e 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -1,7 +1,7 @@ # # MIT License # -# Copyright (c) 2021 Airbyte +# Copyright (c) 2020 Airbyte # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -21,19 +21,19 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # -import logging -logging.basicConfig(level=logging.DEBUG) +import logging from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple from datetime import datetime, timedelta +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator, Oauth2Authenticator, HttpAuthenticator +from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator +logging.basicConfig(level=logging.DEBUG) """ TODO: Most comments in this class are instructive and should be deleted after the source is implemented. @@ -96,12 +96,12 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, If there are no more pages in the result, return None. """ decoded_response = response.json() - total_pages = decoded_response.get('total_pages') - page_number = decoded_response.get('page') + total_pages = decoded_response.get("total_pages") + page_number = decoded_response.get("page") if page_number >= total_pages: return None else: - return {"page": page_number+1} + return {"page": page_number + 1} def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None @@ -112,21 +112,17 @@ def request_params( """ page_number = 1 if next_page_token: - page_number = next_page_token.get('page') + page_number = next_page_token.get("page") - print(f'stream_state {stream_state}') - print(f'stream_slice {stream_slice}') - - start_date = stream_slice['date'] + start_date = stream_slice["date"] end_date_dt = datetime.fromisoformat(start_date) + timedelta(days=self.stream_size_in_days) + + date_time_now = datetime.now().astimezone() + if end_date_dt > date_time_now: + end_date_dt = date_time_now + end_date = end_date_dt.isoformat() - return { - 'start_date': start_date, - 'end_date': end_date, - 'fields': 'all', - 'page_size': '1', - 'page': page_number - } + return {"start_date": start_date, "end_date": end_date, "fields": "all", "page_size": "1", "page": page_number} def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: """ @@ -137,14 +133,27 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp records = json_response.get(self.data_field, []) if self.data_field is not None else json_response yield from records + @staticmethod + def get_field(record: Mapping[str, Any], field_path: List[str]): + + data = record + for attr in field_path: + if data: + data = data.get(attr) + else: + break + + return data + class Transactions(PaypalTransactionStream): """ Stream for Transactions /v1/reporting/transactions """ + data_field = "transaction_details" primary_key = "transaction_id" - cursor_field = "date" + cursor_field = ["transaction_info", "transaction_initiation_date"] stream_size_in_days = 1 def __init__(self, start_date: datetime, **kwargs): @@ -156,15 +165,24 @@ def path( ) -> str: return "transactions" - # def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: - # # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state - # # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. - # if current_stream_state is not None and 'date' in current_stream_state: - # current_parsed_date = datetime.strptime(current_stream_state['date'], '%Y-%m-%d') - # latest_record_date = datetime.strptime(latest_record['date'], '%Y-%m-%d') - # return {'date': max(current_parsed_date, latest_record_date).strftime('%Y-%m-%d')} - # else: - # return {'date': self.start_date.strftime('%Y-%m-%d')} + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: + # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state + # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. + latest_record_date_str = self.get_field(latest_record, self.cursor_field) + + if current_stream_state and "date" in current_stream_state and latest_record_date_str: + if len(latest_record_date_str) == 24: + # Add ':' to timezone part to match iso format, example: + # python iso format: 2021-06-04T00:00:00+03:00 + # format from record: 2021-06-04T00:00:00+0300 + latest_record_date_str = ":".join([latest_record_date_str[:22], latest_record_date_str[22:]]) + + latest_record_date = datetime.fromisoformat(latest_record_date_str) + current_parsed_date = datetime.fromisoformat(current_stream_state["date"]) + + return {"date": max(current_parsed_date, latest_record_date).isoformat()} + else: + return {"date": self.start_date.isoformat()} def _chunk_date_range(self, start_date: datetime) -> List[Mapping[str, any]]: """ @@ -172,11 +190,9 @@ def _chunk_date_range(self, start_date: datetime) -> List[Mapping[str, any]]: The return value is a list of dicts {'date': date_string}. """ dates = [] - while start_date < datetime.now().astimezone(): - dates.append({'date': start_date.isoformat()}) + while start_date < datetime.now().astimezone() - timedelta(days=2): + dates.append({"date": start_date.isoformat()}) start_date += timedelta(days=self.stream_size_in_days) - - print(f'dates {dates}') return dates def stream_slices( @@ -184,8 +200,8 @@ def stream_slices( ) -> Iterable[Optional[Mapping[str, any]]]: start_date = self.start_date - if stream_state and 'date' in stream_state: - start_date = datetime.fromisoformat(stream_state['date']) + if stream_state and "date" in stream_state: + start_date = datetime.fromisoformat(stream_state["date"]) return self._chunk_date_range(start_date) @@ -214,11 +230,10 @@ class PayPalOauth2Authenticator(Oauth2Authenticator): -u "CLIENT_ID:SECRET" \ -d "grant_type=client_credentials" """ + def get_refresh_request_body(self) -> Mapping[str, Any]: """ Override to define additional parameters """ - payload: MutableMapping[str, Any] = { - "grant_type": "client_credentials" - } + payload: MutableMapping[str, Any] = {"grant_type": "client_credentials"} return payload def refresh_access_token(self) -> Tuple[str, int]: @@ -227,17 +242,9 @@ def refresh_access_token(self) -> Tuple[str, int]: """ try: data = "grant_type=client_credentials" - headers = { - 'Accept': 'application/json', - 'Accept-Language': 'en_US' - } + headers = {"Accept": "application/json", "Accept-Language": "en_US"} auth = (self.client_id, self.client_secret) - response = requests.request( - method="POST", - url=self.token_refresh_endpoint, - data=data, - headers=headers, - auth=auth) + response = requests.request(method="POST", url=self.token_refresh_endpoint, data=data, headers=headers, auth=auth) response.raise_for_status() response_json = response.json() return response_json["access_token"], response_json["expires_in"] @@ -246,7 +253,6 @@ def refresh_access_token(self) -> Tuple[str, int]: class SourcePaypalTransaction(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: """ TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API @@ -259,26 +265,28 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ token = PayPalOauth2Authenticator( - token_refresh_endpoint='https://api-m.sandbox.paypal.com/v1/oauth2/token', + token_refresh_endpoint="https://api-m.sandbox.paypal.com/v1/oauth2/token", client_id=config["client_id"], client_secret=config["secret"], - refresh_token='').get_access_token() + refresh_token="", + ).get_access_token() if not token: - return False, 'Unable to fetch Paypal API token due to incorrect client_id or secret' + return False, "Unable to fetch Paypal API token due to incorrect client_id or secret" return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ + """58 TODO: Replace the streams below with your own streams. :param config: A Mapping of the user input configuration as defined in the connector spec. """ authenticator = PayPalOauth2Authenticator( - token_refresh_endpoint='https://api-m.sandbox.paypal.com/v1/oauth2/token', + token_refresh_endpoint="https://api-m.sandbox.paypal.com/v1/oauth2/token", client_id=config["client_id"], client_secret=config["secret"], - refresh_token='') + refresh_token="", + ) start_date = datetime.strptime(config["start_date"], "%Y-%m-%d").astimezone() - #return [Transactions(authenticator=auth), Balances(authenticator=auth)] + # return [Transactions(authenticator=auth), Balances(authenticator=auth)] return [Transactions(authenticator=authenticator, start_date=start_date)] From 24feba6db534f5f49b1a7b2d3b9fd920b90f07ce Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 10 Jun 2021 12:06:30 +0300 Subject: [PATCH 09/60] incremental sync, acceptance test --- .../source-paypal-transaction/integration_tests/state.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json new file mode 100644 index 000000000000..dd461b09ceb6 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json @@ -0,0 +1,5 @@ +{ + "transactions": { + "date": "2021-06-04T17:34:43+00:00" + } +} From 4b2bcabdf9fda1937a6c8ce7f419c7d6667d41c2 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 1 Jun 2021 00:51:08 +0300 Subject: [PATCH 10/60] Added spec.json --- .../source_paypal_transaction/spec.json | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json new file mode 100644 index 000000000000..229ac90c7f4c --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json @@ -0,0 +1,29 @@ +{ + "documentationUrl": "https://developer.paypal.com/docs/api/transaction-search/v1/", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Paypal Transaction Search", + "type": "object", + "required": ["client_id", "secret", "start_date"], + "additionalProperties": false, + "properties": { + "client_id": { + "title": "Client ID", + "type": "string", + "description": "The Paypal Client ID for API credentials", + }, + "secret": { + "title": "Secret", + "type": "string", + "description": "The Secret for a given Client ID.", + "airbyte_secret": true + }, + "start_date": { + "type": "string", + "title": "Start Date", + "description": "Start Date for data extraction in Internet date and time format https://datatracker.ietf.org/doc/html/rfc3339#section-5.6", + "examples": ["2021-06-11T23:59:59-0700"] + } + } + } +} From 3e8da8e08e5a0ee9947afb76241d302401a174ad Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 1 Jun 2021 00:51:37 +0300 Subject: [PATCH 11/60] Initialization --- .../source-paypal-transaction/.dockerignore | 7 + .../source-paypal-transaction/Dockerfile | 15 ++ .../source-paypal-transaction/README.md | 129 ++++++++++ .../acceptance-test-config.yml | 30 +++ .../acceptance-test-docker.sh | 7 + .../source-paypal-transaction/build.gradle | 14 ++ .../integration_tests/__init__.py | 0 .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 36 +++ .../integration_tests/catalog.json | 39 +++ .../integration_tests/configured_catalog.json | 56 +++++ .../integration_tests/invalid_config.json | 3 + .../integration_tests/sample_config.json | 3 + .../integration_tests/sample_state.json | 5 + .../source-paypal-transaction/main.py | 33 +++ .../requirements.txt | 2 + .../source-paypal-transaction/setup.py | 48 ++++ .../source_paypal_transaction/__init__.py | 27 ++ .../source_paypal_transaction/schemas/TODO.md | 25 ++ .../schemas/customers.json | 16 ++ .../schemas/employees.json | 19 ++ .../source_paypal_transaction/source.py | 230 ++++++++++++++++++ .../unit_tests/unit_test.py | 27 ++ 23 files changed, 776 insertions(+) create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/.dockerignore create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/Dockerfile create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/README.md create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/build.gradle create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/main.py create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/requirements.txt create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/setup.py create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/__init__.py create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/customers.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/employees.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py diff --git a/airbyte-integrations/connectors/source-paypal-transaction/.dockerignore b/airbyte-integrations/connectors/source-paypal-transaction/.dockerignore new file mode 100644 index 000000000000..7d3fd691a105 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_paypal_transaction +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile b/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile new file mode 100644 index 000000000000..ccdfa2b4a372 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.7-slim + +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +WORKDIR /airbyte/integration_code +COPY source_paypal_transaction ./source_paypal_transaction +COPY main.py ./ +COPY setup.py ./ +RUN pip install . + +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-paypal-transaction diff --git a/airbyte-integrations/connectors/source-paypal-transaction/README.md b/airbyte-integrations/connectors/source-paypal-transaction/README.md new file mode 100644 index 000000000000..4b34bbf50cfe --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/README.md @@ -0,0 +1,129 @@ +# Paypal Transaction Source + +This is the repository for the Paypal Transaction source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/paypal-transaction). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-paypal-transaction:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/paypal-transaction) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_paypal_transaction/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source paypal-transaction test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-paypal-transaction:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-paypal-transaction:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-paypal-transaction:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paypal-transaction:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paypal-transaction:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-paypal-transaction:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](source-acceptance-tests.md) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-paypal-transaction:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-paypal-transaction:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml new file mode 100644 index 000000000000..e38b0cf2c95b --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/source-acceptance-tests.md) +# for more information about how to configure these tests +connector_image: airbyte/source-paypal-transaction:dev +tests: + spec: + - spec_path: "source_paypal_transaction/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "exception" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + validate_output_from_all_streams: yes +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.txt" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: # TODO if your connector does not implement incremental sync, remove this block + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-docker.sh new file mode 100644 index 000000000000..1425ff74f151 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-docker.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input diff --git a/airbyte-integrations/connectors/source-paypal-transaction/build.gradle b/airbyte-integrations/connectors/source-paypal-transaction/build.gradle new file mode 100644 index 000000000000..4a6226795c51 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/build.gradle @@ -0,0 +1,14 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_paypal_transaction' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/__init__.py b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py new file mode 100644 index 000000000000..eeb4a2d3e02e --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py @@ -0,0 +1,36 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """ This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/catalog.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/catalog.json new file mode 100644 index 000000000000..6799946a6851 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/catalog.json @@ -0,0 +1,39 @@ +{ + "streams": [ + { + "name": "TODO fix this file", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": "column1", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + }, + { + "name": "table1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..74de5e17e466 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json @@ -0,0 +1,56 @@ +// TODO: Construct a configured catalog that can be used for testing. Each stream's `json_schema` field should match the corresponding json schema file. +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "signup_date": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "employees", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "years_of_service": { + "type": ["null", "integer"] + }, + "start_date": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json new file mode 100644 index 000000000000..f3732995784f --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "todo-wrong-field": "this should be an incomplete config file, used in standard tests" +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json new file mode 100644 index 000000000000..ecc4913b84c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/main.py b/airbyte-integrations/connectors/source-paypal-transaction/main.py new file mode 100644 index 000000000000..881549ef9405 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/main.py @@ -0,0 +1,33 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_paypal_transaction import SourcePaypalTransaction + +if __name__ == "__main__": + source = SourcePaypalTransaction() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/requirements.txt b/airbyte-integrations/connectors/source-paypal-transaction/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-paypal-transaction/setup.py b/airbyte-integrations/connectors/source-paypal-transaction/setup.py new file mode 100644 index 000000000000..4a62e6df834d --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/setup.py @@ -0,0 +1,48 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "source-acceptance-test", +] + +setup( + name="source_paypal_transaction", + description="Source implementation for Paypal Transaction.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/__init__.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/__init__.py new file mode 100644 index 000000000000..403d505dcb87 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/__init__.py @@ -0,0 +1,27 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .source import SourcePaypalTransaction + +__all__ = ["SourcePaypalTransaction"] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/TODO.md b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/TODO.md new file mode 100644 index 000000000000..cf1efadb3c9c --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/TODO.md @@ -0,0 +1,25 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. + +The schema of a stream is the return value of `Stream.get_json_schema`. + +## Static schemas +By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. + +Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. + +## Dynamic schemas +If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). + +## Dynamically modifying static schemas +Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: +``` +def get_json_schema(self): + schema = super().get_json_schema() + schema['dynamically_determined_property'] = "property" + return schema +``` + +Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/customers.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/customers.json new file mode 100644 index 000000000000..9a4b13485836 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/customers.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "signup_date": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/employees.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/employees.json new file mode 100644 index 000000000000..2fa01a0fa1ff --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/employees.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "years_of_service": { + "type": ["null", "integer"] + }, + "start_date": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py new file mode 100644 index 000000000000..d42066a8c7b7 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -0,0 +1,230 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + +""" +TODO: Most comments in this class are instructive and should be deleted after the source is implemented. + +This file provides a stubbed example of how to use the Airbyte CDK to develop both a source connector which supports full refresh or and an +incremental syncs from an HTTP API. + +The various TODOs are both implementation hints and steps - fulfilling all the TODOs should be sufficient to implement one basic and one incremental +stream from a source. This pattern is the same one used by Airbyte internally to implement connectors. + +The approach here is not authoritative, and devs are free to use their own judgement. + +There are additional required TODOs in the files within the integration_tests folder and the spec.json file. +""" + + +# Basic full refresh stream +class PaypalTransactionStream(HttpStream, ABC): + """ + TODO remove this comment + + This class represents a stream output by the connector. + This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, + parsing responses etc.. + + Each stream should extend this class (or another abstract subclass of it) to specify behavior unique to that stream. + + Typically for REST APIs each stream corresponds to a resource in the API. For example if the API + contains the endpoints + - GET v1/customers + - GET v1/employees + + then you should have three classes: + `class PaypalTransactionStream(HttpStream, ABC)` which is the current class + `class Customers(PaypalTransactionStream)` contains behavior to pull data for customers using v1/customers + `class Employees(PaypalTransactionStream)` contains behavior to pull data for employees using v1/employees + + If some streams implement incremental sync, it is typical to create another class + `class IncrementalPaypalTransactionStream((PaypalTransactionStream), ABC)` then have concrete stream implementations extend it. An example + is provided below. + + See the reference docs for the full list of configurable options. + """ + + # TODO: Fill in the url base. Required. + url_base = "https://example-api.com/v1/" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. + + This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed + to most other methods in this class to help you form headers, request bodies, query params, etc.. + + For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a + 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. + The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. + + :param response: the most recent response from the API + :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. + If there are no more pages in the result, return None. + """ + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + """ + TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. + Usually contains common params e.g. pagination size etc. + """ + return {} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + TODO: Override this method to define how a response is parsed. + :return an iterable containing each record in the response + """ + yield {} + + +class Customers(PaypalTransactionStream): + """ + TODO: Change class name to match the table/data source this stream corresponds to. + """ + + # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. + primary_key = "customer_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + should return "customers". Required. + """ + return "customers" + + +# Basic incremental stream +class IncrementalPaypalTransactionStream(PaypalTransactionStream, ABC): + """ + TODO fill in details of this class to implement functionality related to incremental syncs for your connector. + if you do not need to implement incremental sync for any streams, remove this class. + """ + + # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. + state_checkpoint_interval = None + + @property + def cursor_field(self) -> str: + """ + TODO + Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is + usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. + + :return str: The name of the cursor field. + """ + return [] + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and + the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. + """ + return {} + + +class Employees(IncrementalPaypalTransactionStream): + """ + TODO: Change class name to match the table/data source this stream corresponds to. + """ + + # TODO: Fill in the cursor_field. Required. + cursor_field = "start_date" + + # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. + primary_key = "employee_id" + + def path(self, **kwargs) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/employees then this should + return "single". Required. + """ + return "employees" + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + """ + TODO: Optionally override this method to define this stream's slices. If slicing is not needed, delete this method. + + Slices control when state is saved. Specifically, state is saved after a slice has been fully read. + This is useful if the API offers reads by groups or filters, and can be paired with the state object to make reads efficient. See the "concepts" + section of the docs for more information. + + The function is called before reading any records in a stream. It returns an Iterable of dicts, each containing the + necessary data to craft a request for a slice. The stream state is usually referenced to determine what slices need to be created. + This means that data in a slice is usually closely related to a stream's cursor_field and stream_state. + + An HTTP request is made for each returned slice. The same slice can be accessed in the path, request_params and request_header functions to help + craft that specific request. + + For example, if https://example-api.com/v1/employees offers a date query params that returns data for that particular day, one way to implement + this would be to consult the stream state object for the last synced date, then return a slice containing each date from the last synced date + till now. The request_params function would then grab the date from the stream_slice and make it part of the request by injecting it into + the date query param. + """ + raise NotImplementedError("Implement stream slices or delete this method!") + + +# Source +class SourcePaypalTransaction(AbstractSource): + + url_base = "https://api-m.sandbox.paypal.com" + # url_base = "https://api-m.paypal.com" + + def check_connection(self, logger, config) -> Tuple[bool, any]: + """ + TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API + + See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 + for an example. + + :param config: the user-input config object conforming to the connector's spec.json + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + return True, None + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + TODO: Replace the streams below with your own streams. + + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + # TODO remove the authenticator if not required. + auth = TokenAuthenticator(token="api_key") # Oauth2Authenticator is also available if you need oauth support + return [Customers(authenticator=auth), Employees(authenticator=auth)] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py new file mode 100644 index 000000000000..b8a8150b507f --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py @@ -0,0 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +def test_example_method(): + assert True From cc241c2377a175e8795c1261a993835bce759c34 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 1 Jun 2021 02:54:22 +0300 Subject: [PATCH 12/60] added oauth2 autorization --- .../source_paypal_transaction/source.py | 133 +++++++++--------- 1 file changed, 64 insertions(+), 69 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index d42066a8c7b7..2274782edeeb 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -30,7 +30,7 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator, Oauth2Authenticator """ TODO: Most comments in this class are instructive and should be deleted after the source is implemented. @@ -75,8 +75,8 @@ class PaypalTransactionStream(HttpStream, ABC): See the reference docs for the full list of configurable options. """ - # TODO: Fill in the url base. Required. - url_base = "https://example-api.com/v1/" + url_base = "https://api-m.sandbox.paypal.com/v1/reporting/" + # url_base = "https://api-m.paypal.com/v1/reporting/" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """ @@ -112,100 +112,87 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp yield {} -class Customers(PaypalTransactionStream): +class Transactions(PaypalTransactionStream): """ - TODO: Change class name to match the table/data source this stream corresponds to. + Stream for Transactions /v1/reporting/transactions """ - # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. - primary_key = "customer_id" + primary_key = "transaction_id" def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this should return "customers". Required. """ - return "customers" - - -# Basic incremental stream -class IncrementalPaypalTransactionStream(PaypalTransactionStream, ABC): - """ - TODO fill in details of this class to implement functionality related to incremental syncs for your connector. - if you do not need to implement incremental sync for any streams, remove this class. - """ - - # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. - state_checkpoint_interval = None - @property - def cursor_field(self) -> str: - """ - TODO - Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is - usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. - - :return str: The name of the cursor field. - """ - return [] - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and - the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. - """ - return {} + return "transactions" -class Employees(IncrementalPaypalTransactionStream): +class Balances(PaypalTransactionStream): """ - TODO: Change class name to match the table/data source this stream corresponds to. + Stream for Balances /v1/reporting/balances """ - # TODO: Fill in the cursor_field. Required. - cursor_field = "start_date" - # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. - primary_key = "employee_id" + primary_key = "customer_id" - def path(self, **kwargs) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/employees then this should - return "single". Required. + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: """ - return "employees" - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + should return "customers". Required. """ - TODO: Optionally override this method to define this stream's slices. If slicing is not needed, delete this method. - Slices control when state is saved. Specifically, state is saved after a slice has been fully read. - This is useful if the API offers reads by groups or filters, and can be paired with the state object to make reads efficient. See the "concepts" - section of the docs for more information. + return "balances" - The function is called before reading any records in a stream. It returns an Iterable of dicts, each containing the - necessary data to craft a request for a slice. The stream state is usually referenced to determine what slices need to be created. - This means that data in a slice is usually closely related to a stream's cursor_field and stream_state. - An HTTP request is made for each returned slice. The same slice can be accessed in the path, request_params and request_header functions to help - craft that specific request. +class PayPalOauth2Authenticator(Oauth2Authenticator): + """ + curl -v POST https://api-m.sandbox.paypal.com/v1/oauth2/token \ + -H "Accept: application/json" \ + -H "Accept-Language: en_US" \ + -u "CLIENT_ID:SECRET" \ + -d "grant_type=client_credentials" - For example, if https://example-api.com/v1/employees offers a date query params that returns data for that particular day, one way to implement - this would be to consult the stream state object for the last synced date, then return a slice containing each date from the last synced date - till now. The request_params function would then grab the date from the stream_slice and make it part of the request by injecting it into - the date query param. - """ - raise NotImplementedError("Implement stream slices or delete this method!") + """ + def get_refresh_request_body(self) -> Mapping[str, Any]: + """ Override to define additional parameters """ + payload: MutableMapping[str, Any] = { + "grant_type": "client_credentials" + } + return payload + + def refresh_access_token(self) -> Tuple[str, int]: + """ + returns a tuple of (access_token, token_lifespan_in_seconds) + """ + try: + data = "grant_type=client_credentials" + headers = { + 'Accept': 'application/json', + 'Accept-Language': 'en_US' + } + auth = (self.client_id, self.client_secret) + response = requests.request( + method="POST", + url=self.token_refresh_endpoint, + data=data, + headers=headers, + auth=auth) + response.raise_for_status() + response_json = response.json() + print(response_json) + return response_json["access_token"], response_json["expires_in"] + except Exception as e: + raise Exception(f"Error while refreshing access token: {e}") from e # Source class SourcePaypalTransaction(AbstractSource): - url_base = "https://api-m.sandbox.paypal.com" - # url_base = "https://api-m.paypal.com" - def check_connection(self, logger, config) -> Tuple[bool, any]: """ TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API @@ -217,6 +204,13 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ + token = PayPalOauth2Authenticator( + token_refresh_endpoint='https://api-m.sandbox.paypal.com/v1/oauth2/token', + client_id=config["client_id"], + client_secret=config["secret"], + refresh_token='').get_access_token() + + print(f"token {token}") return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: @@ -227,4 +221,5 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ # TODO remove the authenticator if not required. auth = TokenAuthenticator(token="api_key") # Oauth2Authenticator is also available if you need oauth support - return [Customers(authenticator=auth), Employees(authenticator=auth)] + # return [Transactions(authenticator=auth), Balances(authenticator=auth)] + return [Transactions(authenticator=auth)] From 54649a86ebf529c30db2448c818e40c25ccc42a9 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 3 Jun 2021 12:16:52 +0300 Subject: [PATCH 13/60] added spec, check, discover + catalogs/configurared_catalogs --- .../integration_tests/configured_catalog.json | 55 +++++++++++++------ .../configured_catalog_transaction.json | 29 ++++++++++ .../schemas/balances.json | 38 +++++++++++++ .../schemas/transactions.json | 17 ++++++ .../source_paypal_transaction/source.py | 16 ++++-- .../source_paypal_transaction/spec.json | 2 +- 6 files changed, 133 insertions(+), 24 deletions(-) create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json index 74de5e17e466..19ebd0f37f06 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json @@ -1,22 +1,22 @@ -// TODO: Construct a configured catalog that can be used for testing. Each stream's `json_schema` field should match the corresponding json schema file. { "streams": [ { "stream": { - "name": "customers", + "name": "transactions", "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "id": { + "account_number": { "type": ["null", "string"] }, - "name": { + "last_refreshed_datetime": { "type": ["null", "string"] }, - "signup_date": { - "type": ["null", "string"], - "format": "date-time" + "page": { + "type": ["null", "integer"] + }, + "total_items": { + "type": ["null", "integer"] } } }, @@ -29,27 +29,46 @@ "stream": { "name": "employees", "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { + "account_id": { "type": ["null", "string"] }, - "years_of_service": { - "type": ["null", "integer"] + "as_of_time": { + "type": ["null", "string"], + "format": "date-time" }, - "start_date": { + "last_refresh_time": { "type": ["null", "string"], "format": "date-time" + }, + "balance": { + "type": ["object", "null"], + "properties": { + "currency": { + "type": ["null", "string"] + }, + "primary": { + "type": ["boolean"] + }, + "total_balance": { + "type": ["object", "null"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + } } } }, - "supported_sync_modes": ["full_refresh", "incremental"] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", + "sync_mode": "full_refresh", "destination_sync_mode": "append" } ] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json new file mode 100644 index 000000000000..1924ce3bd1a9 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json @@ -0,0 +1,29 @@ +{ + "streams": [ + { + "stream": { + "name": "transactions", + "json_schema": { + "type": "object", + "properties": { + "account_number": { + "type": ["null", "string"] + }, + "last_refreshed_datetime": { + "type": ["null", "string"] + }, + "page": { + "type": ["null", "integer"] + }, + "total_items": { + "type": ["null", "integer"] + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json new file mode 100644 index 000000000000..69c6b92d6e80 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json @@ -0,0 +1,38 @@ +{ + "type": "object", + "properties": { + "account_id": { + "type": ["null", "string"] + }, + "as_of_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "last_refresh_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "balance": { + "type": ["object", "null"], + "properties": { + "currency": { + "type": ["null", "string"] + }, + "primary": { + "type": ["boolean"] + }, + "total_balance": { + "type": ["object", "null"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json new file mode 100644 index 000000000000..981c82a6a552 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json @@ -0,0 +1,17 @@ +{ + "type": "object", + "properties": { + "account_number": { + "type": ["null", "string"] + }, + "last_refreshed_datetime": { + "type": ["null", "string"] + }, + "page": { + "type": ["null", "integer"] + }, + "total_items": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 2274782edeeb..26263f9c6001 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -117,7 +117,7 @@ class Transactions(PaypalTransactionStream): Stream for Transactions /v1/reporting/transactions """ - primary_key = "transaction_id" + primary_key = "last_refreshed_datetime" def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None @@ -136,7 +136,7 @@ class Balances(PaypalTransactionStream): """ # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. - primary_key = "customer_id" + primary_key = "as_of_time" def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None @@ -209,6 +209,8 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: client_id=config["client_id"], client_secret=config["secret"], refresh_token='').get_access_token() + if not token: + return False, 'Unable to fetch Paypal API token due to incorrect client_id or secret' print(f"token {token}") return True, None @@ -219,7 +221,11 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: :param config: A Mapping of the user input configuration as defined in the connector spec. """ - # TODO remove the authenticator if not required. - auth = TokenAuthenticator(token="api_key") # Oauth2Authenticator is also available if you need oauth support + token = PayPalOauth2Authenticator( + token_refresh_endpoint='https://api-m.sandbox.paypal.com/v1/oauth2/token', + client_id=config["client_id"], + client_secret=config["secret"], + refresh_token='').get_access_token() + auth = TokenAuthenticator(token=token) # Oauth2Authenticator is also available if you need oauth support # return [Transactions(authenticator=auth), Balances(authenticator=auth)] - return [Transactions(authenticator=auth)] + return [Transactions(authenticator=auth), Balances(authenticator=auth)] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json index 229ac90c7f4c..5cd95638bfac 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json @@ -10,7 +10,7 @@ "client_id": { "title": "Client ID", "type": "string", - "description": "The Paypal Client ID for API credentials", + "description": "The Paypal Client ID for API credentials" }, "secret": { "title": "Secret", From 87109366fee3a1867fdc8b83d1f37115403c4354 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Fri, 4 Jun 2021 12:27:07 +0300 Subject: [PATCH 14/60] updated request_params --- .../source_paypal_transaction/source.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 26263f9c6001..822d42fe156a 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -25,7 +25,7 @@ from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple - +import datetime import requests from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -102,14 +102,20 @@ def request_params( TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. Usually contains common params e.g. pagination size etc. """ - return {} + return { + 'start_date': '2021-05-15T00:00:00-0700', + 'end_date': '2021-06-15T00:00:00-0700', + 'fields': 'all', + 'page_size': '100' + } def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: """ TODO: Override this method to define how a response is parsed. :return an iterable containing each record in the response """ - yield {} + #yield {} + return [response.json()] class Transactions(PaypalTransactionStream): @@ -228,4 +234,6 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: refresh_token='').get_access_token() auth = TokenAuthenticator(token=token) # Oauth2Authenticator is also available if you need oauth support # return [Transactions(authenticator=auth), Balances(authenticator=auth)] - return [Transactions(authenticator=auth), Balances(authenticator=auth)] + #start_date = datetime.strptime(config["start_date"], "%Y-%m-%d") + #return [Transactions(authenticator=auth), Balances(authenticator=auth)] + return [Transactions(authenticator=auth)] From b1df4697ee1665c0dab8de03216f4839dd255653 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Mon, 7 Jun 2021 20:00:44 +0300 Subject: [PATCH 15/60] added paging, slicing (1d) --- .../source_paypal_transaction/source.py | 94 ++++++++++++++----- 1 file changed, 73 insertions(+), 21 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 822d42fe156a..1735aa3f5876 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -1,7 +1,7 @@ # # MIT License # -# Copyright (c) 2020 Airbyte +# Copyright (c) 2021 Airbyte # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -21,16 +21,18 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # - +import logging +logging.basicConfig(level=logging.DEBUG) from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -import datetime +from datetime import datetime, timedelta + import requests from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator, Oauth2Authenticator +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator, Oauth2Authenticator, HttpAuthenticator """ TODO: Most comments in this class are instructive and should be deleted after the source is implemented. @@ -93,7 +95,13 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. If there are no more pages in the result, return None. """ - return None + decoded_response = response.json() + total_pages = decoded_response.get('total_pages') + page_number = decoded_response.get('page') + if page_number >= total_pages: + return None + else: + return {"page": page_number+1} def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None @@ -102,11 +110,22 @@ def request_params( TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. Usually contains common params e.g. pagination size etc. """ + page_number = 1 + if next_page_token: + page_number = next_page_token.get('page') + + print(f'stream_state {stream_state}') + print(f'stream_slice {stream_slice}') + + start_date = stream_slice['date'] + end_date_dt = datetime.fromisoformat(start_date) + timedelta(days=self.stream_size_in_days) + end_date = end_date_dt.isoformat() return { - 'start_date': '2021-05-15T00:00:00-0700', - 'end_date': '2021-06-15T00:00:00-0700', + 'start_date': start_date, + 'end_date': end_date, 'fields': 'all', - 'page_size': '100' + 'page_size': '1', + 'page': page_number } def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: @@ -114,26 +133,61 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp TODO: Override this method to define how a response is parsed. :return an iterable containing each record in the response """ - #yield {} - return [response.json()] + json_response = response.json() + records = json_response.get(self.data_field, []) if self.data_field is not None else json_response + yield from records class Transactions(PaypalTransactionStream): """ Stream for Transactions /v1/reporting/transactions """ + data_field = "transaction_details" + primary_key = "transaction_id" + cursor_field = "date" + stream_size_in_days = 1 - primary_key = "last_refreshed_datetime" + def __init__(self, start_date: datetime, **kwargs): + super().__init__(**kwargs) + self.start_date = start_date def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: + return "transactions" + + # def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: + # # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state + # # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. + # if current_stream_state is not None and 'date' in current_stream_state: + # current_parsed_date = datetime.strptime(current_stream_state['date'], '%Y-%m-%d') + # latest_record_date = datetime.strptime(latest_record['date'], '%Y-%m-%d') + # return {'date': max(current_parsed_date, latest_record_date).strftime('%Y-%m-%d')} + # else: + # return {'date': self.start_date.strftime('%Y-%m-%d')} + + def _chunk_date_range(self, start_date: datetime) -> List[Mapping[str, any]]: """ - Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this - should return "customers". Required. + Returns a list of each day between the start date and now. + The return value is a list of dicts {'date': date_string}. """ + dates = [] + while start_date < datetime.now().astimezone(): + dates.append({'date': start_date.isoformat()}) + start_date += timedelta(days=self.stream_size_in_days) - return "transactions" + print(f'dates {dates}') + return dates + + def stream_slices( + self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, any]]]: + + start_date = self.start_date + if stream_state and 'date' in stream_state: + start_date = datetime.fromisoformat(stream_state['date']) + + return self._chunk_date_range(start_date) class Balances(PaypalTransactionStream): @@ -141,17 +195,14 @@ class Balances(PaypalTransactionStream): Stream for Balances /v1/reporting/balances """ + data_field = "transaction_details" + # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. primary_key = "as_of_time" def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this - should return "customers". Required. - """ - return "balances" @@ -234,6 +285,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: refresh_token='').get_access_token() auth = TokenAuthenticator(token=token) # Oauth2Authenticator is also available if you need oauth support # return [Transactions(authenticator=auth), Balances(authenticator=auth)] - #start_date = datetime.strptime(config["start_date"], "%Y-%m-%d") + print(f'TOKEN: {token}') + start_date = datetime.strptime(config["start_date"], "%Y-%m-%d").astimezone() #return [Transactions(authenticator=auth), Balances(authenticator=auth)] - return [Transactions(authenticator=auth)] + return [Transactions(authenticator=auth, start_date=start_date)] From c36bc78b10add52b5a246bc43b017a4546012e31 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Mon, 7 Jun 2021 21:51:58 +0300 Subject: [PATCH 16/60] Use oath2 for paypal --- .../source_paypal_transaction/source.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 1735aa3f5876..dbbb0880e482 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -213,7 +213,6 @@ class PayPalOauth2Authenticator(Oauth2Authenticator): -H "Accept-Language: en_US" \ -u "CLIENT_ID:SECRET" \ -d "grant_type=client_credentials" - """ def get_refresh_request_body(self) -> Mapping[str, Any]: """ Override to define additional parameters """ @@ -241,13 +240,11 @@ def refresh_access_token(self) -> Tuple[str, int]: auth=auth) response.raise_for_status() response_json = response.json() - print(response_json) return response_json["access_token"], response_json["expires_in"] except Exception as e: raise Exception(f"Error while refreshing access token: {e}") from e -# Source class SourcePaypalTransaction(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: @@ -269,7 +266,6 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: if not token: return False, 'Unable to fetch Paypal API token due to incorrect client_id or secret' - print(f"token {token}") return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: @@ -278,14 +274,11 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: :param config: A Mapping of the user input configuration as defined in the connector spec. """ - token = PayPalOauth2Authenticator( + authenticator = PayPalOauth2Authenticator( token_refresh_endpoint='https://api-m.sandbox.paypal.com/v1/oauth2/token', client_id=config["client_id"], client_secret=config["secret"], - refresh_token='').get_access_token() - auth = TokenAuthenticator(token=token) # Oauth2Authenticator is also available if you need oauth support - # return [Transactions(authenticator=auth), Balances(authenticator=auth)] - print(f'TOKEN: {token}') + refresh_token='') start_date = datetime.strptime(config["start_date"], "%Y-%m-%d").astimezone() #return [Transactions(authenticator=auth), Balances(authenticator=auth)] - return [Transactions(authenticator=auth, start_date=start_date)] + return [Transactions(authenticator=authenticator, start_date=start_date)] From 8019d8373bd5b675c0af6dbc079925ddfdb73ef1 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 10 Jun 2021 12:06:08 +0300 Subject: [PATCH 17/60] incremental sync, acceptance test --- .../acceptance-test-config.yml | 2 +- .../configured_catalog_transaction.json | 4 +- .../integration_tests/invalid_config.json | 6 +- .../integration_tests/sample_config.json | 6 +- .../integration_tests/sample_state.json | 4 +- .../source_paypal_transaction/source.py | 122 ++++++++++-------- 6 files changed, 78 insertions(+), 66 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml index e38b0cf2c95b..08d0e07e3577 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml @@ -8,7 +8,7 @@ tests: - config_path: "secrets/config.json" status: "succeed" - config_path: "integration_tests/invalid_config.json" - status: "exception" + status: "failed" discovery: - config_path: "secrets/config.json" basic_read: diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json index 1924ce3bd1a9..23ec64e5a5cb 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json @@ -20,9 +20,9 @@ } } }, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "full_refresh", + "sync_mode": "incremental", "destination_sync_mode": "overwrite" } ] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json index f3732995784f..52839c5ac77e 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json @@ -1,3 +1,5 @@ { - "todo-wrong-field": "this should be an incomplete config file, used in standard tests" -} + "client_id": "AWAz___", + "secret": "ENC8__", + "start_date": "2021-06-01" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json index ecc4913b84c7..fff6d380f653 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json @@ -1,3 +1,5 @@ { - "fix-me": "TODO" -} + "client_id": "AWAz0aQCdk", + "secret": "ENC8PBtgHBH-", + "start_date": "2021-06-01" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json index 3587e579822d..dd461b09ceb6 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json @@ -1,5 +1,5 @@ { - "todo-stream-name": { - "todo-field-name": "value" + "transactions": { + "date": "2021-06-04T17:34:43+00:00" } } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index dbbb0880e482..45040b04014e 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -1,7 +1,7 @@ # # MIT License # -# Copyright (c) 2021 Airbyte +# Copyright (c) 2020 Airbyte # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -21,19 +21,19 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # -import logging -logging.basicConfig(level=logging.DEBUG) +import logging from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple from datetime import datetime, timedelta +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator, Oauth2Authenticator, HttpAuthenticator +from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator +logging.basicConfig(level=logging.DEBUG) """ TODO: Most comments in this class are instructive and should be deleted after the source is implemented. @@ -96,12 +96,12 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, If there are no more pages in the result, return None. """ decoded_response = response.json() - total_pages = decoded_response.get('total_pages') - page_number = decoded_response.get('page') + total_pages = decoded_response.get("total_pages") + page_number = decoded_response.get("page") if page_number >= total_pages: return None else: - return {"page": page_number+1} + return {"page": page_number + 1} def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None @@ -112,21 +112,17 @@ def request_params( """ page_number = 1 if next_page_token: - page_number = next_page_token.get('page') + page_number = next_page_token.get("page") - print(f'stream_state {stream_state}') - print(f'stream_slice {stream_slice}') - - start_date = stream_slice['date'] + start_date = stream_slice["date"] end_date_dt = datetime.fromisoformat(start_date) + timedelta(days=self.stream_size_in_days) + + date_time_now = datetime.now().astimezone() + if end_date_dt > date_time_now: + end_date_dt = date_time_now + end_date = end_date_dt.isoformat() - return { - 'start_date': start_date, - 'end_date': end_date, - 'fields': 'all', - 'page_size': '1', - 'page': page_number - } + return {"start_date": start_date, "end_date": end_date, "fields": "all", "page_size": "1", "page": page_number} def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: """ @@ -137,14 +133,27 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp records = json_response.get(self.data_field, []) if self.data_field is not None else json_response yield from records + @staticmethod + def get_field(record: Mapping[str, Any], field_path: List[str]): + + data = record + for attr in field_path: + if data: + data = data.get(attr) + else: + break + + return data + class Transactions(PaypalTransactionStream): """ Stream for Transactions /v1/reporting/transactions """ + data_field = "transaction_details" primary_key = "transaction_id" - cursor_field = "date" + cursor_field = ["transaction_info", "transaction_initiation_date"] stream_size_in_days = 1 def __init__(self, start_date: datetime, **kwargs): @@ -156,15 +165,24 @@ def path( ) -> str: return "transactions" - # def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: - # # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state - # # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. - # if current_stream_state is not None and 'date' in current_stream_state: - # current_parsed_date = datetime.strptime(current_stream_state['date'], '%Y-%m-%d') - # latest_record_date = datetime.strptime(latest_record['date'], '%Y-%m-%d') - # return {'date': max(current_parsed_date, latest_record_date).strftime('%Y-%m-%d')} - # else: - # return {'date': self.start_date.strftime('%Y-%m-%d')} + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: + # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state + # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. + latest_record_date_str = self.get_field(latest_record, self.cursor_field) + + if current_stream_state and "date" in current_stream_state and latest_record_date_str: + if len(latest_record_date_str) == 24: + # Add ':' to timezone part to match iso format, example: + # python iso format: 2021-06-04T00:00:00+03:00 + # format from record: 2021-06-04T00:00:00+0300 + latest_record_date_str = ":".join([latest_record_date_str[:22], latest_record_date_str[22:]]) + + latest_record_date = datetime.fromisoformat(latest_record_date_str) + current_parsed_date = datetime.fromisoformat(current_stream_state["date"]) + + return {"date": max(current_parsed_date, latest_record_date).isoformat()} + else: + return {"date": self.start_date.isoformat()} def _chunk_date_range(self, start_date: datetime) -> List[Mapping[str, any]]: """ @@ -172,11 +190,9 @@ def _chunk_date_range(self, start_date: datetime) -> List[Mapping[str, any]]: The return value is a list of dicts {'date': date_string}. """ dates = [] - while start_date < datetime.now().astimezone(): - dates.append({'date': start_date.isoformat()}) + while start_date < datetime.now().astimezone() - timedelta(days=2): + dates.append({"date": start_date.isoformat()}) start_date += timedelta(days=self.stream_size_in_days) - - print(f'dates {dates}') return dates def stream_slices( @@ -184,8 +200,8 @@ def stream_slices( ) -> Iterable[Optional[Mapping[str, any]]]: start_date = self.start_date - if stream_state and 'date' in stream_state: - start_date = datetime.fromisoformat(stream_state['date']) + if stream_state and "date" in stream_state: + start_date = datetime.fromisoformat(stream_state["date"]) return self._chunk_date_range(start_date) @@ -214,11 +230,10 @@ class PayPalOauth2Authenticator(Oauth2Authenticator): -u "CLIENT_ID:SECRET" \ -d "grant_type=client_credentials" """ + def get_refresh_request_body(self) -> Mapping[str, Any]: """ Override to define additional parameters """ - payload: MutableMapping[str, Any] = { - "grant_type": "client_credentials" - } + payload: MutableMapping[str, Any] = {"grant_type": "client_credentials"} return payload def refresh_access_token(self) -> Tuple[str, int]: @@ -227,17 +242,9 @@ def refresh_access_token(self) -> Tuple[str, int]: """ try: data = "grant_type=client_credentials" - headers = { - 'Accept': 'application/json', - 'Accept-Language': 'en_US' - } + headers = {"Accept": "application/json", "Accept-Language": "en_US"} auth = (self.client_id, self.client_secret) - response = requests.request( - method="POST", - url=self.token_refresh_endpoint, - data=data, - headers=headers, - auth=auth) + response = requests.request(method="POST", url=self.token_refresh_endpoint, data=data, headers=headers, auth=auth) response.raise_for_status() response_json = response.json() return response_json["access_token"], response_json["expires_in"] @@ -246,7 +253,6 @@ def refresh_access_token(self) -> Tuple[str, int]: class SourcePaypalTransaction(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: """ TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API @@ -259,26 +265,28 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ token = PayPalOauth2Authenticator( - token_refresh_endpoint='https://api-m.sandbox.paypal.com/v1/oauth2/token', + token_refresh_endpoint="https://api-m.sandbox.paypal.com/v1/oauth2/token", client_id=config["client_id"], client_secret=config["secret"], - refresh_token='').get_access_token() + refresh_token="", + ).get_access_token() if not token: - return False, 'Unable to fetch Paypal API token due to incorrect client_id or secret' + return False, "Unable to fetch Paypal API token due to incorrect client_id or secret" return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ + """58 TODO: Replace the streams below with your own streams. :param config: A Mapping of the user input configuration as defined in the connector spec. """ authenticator = PayPalOauth2Authenticator( - token_refresh_endpoint='https://api-m.sandbox.paypal.com/v1/oauth2/token', + token_refresh_endpoint="https://api-m.sandbox.paypal.com/v1/oauth2/token", client_id=config["client_id"], client_secret=config["secret"], - refresh_token='') + refresh_token="", + ) start_date = datetime.strptime(config["start_date"], "%Y-%m-%d").astimezone() - #return [Transactions(authenticator=auth), Balances(authenticator=auth)] + # return [Transactions(authenticator=auth), Balances(authenticator=auth)] return [Transactions(authenticator=authenticator, start_date=start_date)] From ce98c15f175659b3ae6d3d13b59719b932c95c7e Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 10 Jun 2021 12:06:30 +0300 Subject: [PATCH 18/60] incremental sync, acceptance test --- .../source-paypal-transaction/integration_tests/state.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json new file mode 100644 index 000000000000..dd461b09ceb6 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json @@ -0,0 +1,5 @@ +{ + "transactions": { + "date": "2021-06-04T17:34:43+00:00" + } +} From fbeb3a3e86df77a462e0c0d24ab749d774c903b9 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 1 Jun 2021 00:51:08 +0300 Subject: [PATCH 19/60] Added spec.json --- .../source_paypal_transaction/spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json index 5cd95638bfac..229ac90c7f4c 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json @@ -10,7 +10,7 @@ "client_id": { "title": "Client ID", "type": "string", - "description": "The Paypal Client ID for API credentials" + "description": "The Paypal Client ID for API credentials", }, "secret": { "title": "Secret", From aec3b9d79a258e158311ccb1e32501cb20c184d5 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 1 Jun 2021 00:51:37 +0300 Subject: [PATCH 20/60] Initialization --- .../acceptance-test-config.yml | 2 +- .../integration_tests/configured_catalog.json | 55 ++--- .../integration_tests/invalid_config.json | 6 +- .../integration_tests/sample_config.json | 6 +- .../integration_tests/sample_state.json | 4 +- .../source_paypal_transaction/source.py | 212 +++++++----------- 6 files changed, 100 insertions(+), 185 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml index 08d0e07e3577..e38b0cf2c95b 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml @@ -8,7 +8,7 @@ tests: - config_path: "secrets/config.json" status: "succeed" - config_path: "integration_tests/invalid_config.json" - status: "failed" + status: "exception" discovery: - config_path: "secrets/config.json" basic_read: diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json index 19ebd0f37f06..74de5e17e466 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json @@ -1,22 +1,22 @@ +// TODO: Construct a configured catalog that can be used for testing. Each stream's `json_schema` field should match the corresponding json schema file. { "streams": [ { "stream": { - "name": "transactions", + "name": "customers", "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "account_number": { + "id": { "type": ["null", "string"] }, - "last_refreshed_datetime": { + "name": { "type": ["null", "string"] }, - "page": { - "type": ["null", "integer"] - }, - "total_items": { - "type": ["null", "integer"] + "signup_date": { + "type": ["null", "string"], + "format": "date-time" } } }, @@ -29,46 +29,27 @@ "stream": { "name": "employees", "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "account_id": { + "id": { "type": ["null", "string"] }, - "as_of_time": { - "type": ["null", "string"], - "format": "date-time" + "name": { + "type": ["null", "string"] + }, + "years_of_service": { + "type": ["null", "integer"] }, - "last_refresh_time": { + "start_date": { "type": ["null", "string"], "format": "date-time" - }, - "balance": { - "type": ["object", "null"], - "properties": { - "currency": { - "type": ["null", "string"] - }, - "primary": { - "type": ["boolean"] - }, - "total_balance": { - "type": ["object", "null"], - "properties": { - "currency_code": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - } } } }, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "full_refresh", + "sync_mode": "incremental", "destination_sync_mode": "append" } ] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json index 52839c5ac77e..f3732995784f 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json @@ -1,5 +1,3 @@ { - "client_id": "AWAz___", - "secret": "ENC8__", - "start_date": "2021-06-01" -} \ No newline at end of file + "todo-wrong-field": "this should be an incomplete config file, used in standard tests" +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json index fff6d380f653..ecc4913b84c7 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json @@ -1,5 +1,3 @@ { - "client_id": "AWAz0aQCdk", - "secret": "ENC8PBtgHBH-", - "start_date": "2021-06-01" -} \ No newline at end of file + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json index dd461b09ceb6..3587e579822d 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json @@ -1,5 +1,5 @@ { - "transactions": { - "date": "2021-06-04T17:34:43+00:00" + "todo-stream-name": { + "todo-field-name": "value" } } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 45040b04014e..d42066a8c7b7 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -22,18 +22,16 @@ # SOFTWARE. # -import logging + from abc import ABC -from datetime import datetime, timedelta from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -logging.basicConfig(level=logging.DEBUG) """ TODO: Most comments in this class are instructive and should be deleted after the source is implemented. @@ -77,8 +75,8 @@ class PaypalTransactionStream(HttpStream, ABC): See the reference docs for the full list of configurable options. """ - url_base = "https://api-m.sandbox.paypal.com/v1/reporting/" - # url_base = "https://api-m.paypal.com/v1/reporting/" + # TODO: Fill in the url base. Required. + url_base = "https://example-api.com/v1/" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """ @@ -95,13 +93,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. If there are no more pages in the result, return None. """ - decoded_response = response.json() - total_pages = decoded_response.get("total_pages") - page_number = decoded_response.get("page") - if page_number >= total_pages: - return None - else: - return {"page": page_number + 1} + return None def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None @@ -110,149 +102,110 @@ def request_params( TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. Usually contains common params e.g. pagination size etc. """ - page_number = 1 - if next_page_token: - page_number = next_page_token.get("page") - - start_date = stream_slice["date"] - end_date_dt = datetime.fromisoformat(start_date) + timedelta(days=self.stream_size_in_days) - - date_time_now = datetime.now().astimezone() - if end_date_dt > date_time_now: - end_date_dt = date_time_now - - end_date = end_date_dt.isoformat() - return {"start_date": start_date, "end_date": end_date, "fields": "all", "page_size": "1", "page": page_number} + return {} def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: """ TODO: Override this method to define how a response is parsed. :return an iterable containing each record in the response """ - json_response = response.json() - records = json_response.get(self.data_field, []) if self.data_field is not None else json_response - yield from records - - @staticmethod - def get_field(record: Mapping[str, Any], field_path: List[str]): + yield {} - data = record - for attr in field_path: - if data: - data = data.get(attr) - else: - break - return data - - -class Transactions(PaypalTransactionStream): +class Customers(PaypalTransactionStream): """ - Stream for Transactions /v1/reporting/transactions + TODO: Change class name to match the table/data source this stream corresponds to. """ - data_field = "transaction_details" - primary_key = "transaction_id" - cursor_field = ["transaction_info", "transaction_initiation_date"] - stream_size_in_days = 1 - - def __init__(self, start_date: datetime, **kwargs): - super().__init__(**kwargs) - self.start_date = start_date + # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. + primary_key = "customer_id" def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - return "transactions" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: - # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state - # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. - latest_record_date_str = self.get_field(latest_record, self.cursor_field) + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + should return "customers". Required. + """ + return "customers" - if current_stream_state and "date" in current_stream_state and latest_record_date_str: - if len(latest_record_date_str) == 24: - # Add ':' to timezone part to match iso format, example: - # python iso format: 2021-06-04T00:00:00+03:00 - # format from record: 2021-06-04T00:00:00+0300 - latest_record_date_str = ":".join([latest_record_date_str[:22], latest_record_date_str[22:]]) - latest_record_date = datetime.fromisoformat(latest_record_date_str) - current_parsed_date = datetime.fromisoformat(current_stream_state["date"]) +# Basic incremental stream +class IncrementalPaypalTransactionStream(PaypalTransactionStream, ABC): + """ + TODO fill in details of this class to implement functionality related to incremental syncs for your connector. + if you do not need to implement incremental sync for any streams, remove this class. + """ - return {"date": max(current_parsed_date, latest_record_date).isoformat()} - else: - return {"date": self.start_date.isoformat()} + # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. + state_checkpoint_interval = None - def _chunk_date_range(self, start_date: datetime) -> List[Mapping[str, any]]: - """ - Returns a list of each day between the start date and now. - The return value is a list of dicts {'date': date_string}. + @property + def cursor_field(self) -> str: """ - dates = [] - while start_date < datetime.now().astimezone() - timedelta(days=2): - dates.append({"date": start_date.isoformat()}) - start_date += timedelta(days=self.stream_size_in_days) - return dates + TODO + Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is + usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. - def stream_slices( - self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, any]]]: - - start_date = self.start_date - if stream_state and "date" in stream_state: - start_date = datetime.fromisoformat(stream_state["date"]) + :return str: The name of the cursor field. + """ + return [] - return self._chunk_date_range(start_date) + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and + the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. + """ + return {} -class Balances(PaypalTransactionStream): +class Employees(IncrementalPaypalTransactionStream): """ - Stream for Balances /v1/reporting/balances + TODO: Change class name to match the table/data source this stream corresponds to. """ - data_field = "transaction_details" + # TODO: Fill in the cursor_field. Required. + cursor_field = "start_date" # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. - primary_key = "as_of_time" + primary_key = "employee_id" - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "balances" + def path(self, **kwargs) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/employees then this should + return "single". Required. + """ + return "employees" + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + """ + TODO: Optionally override this method to define this stream's slices. If slicing is not needed, delete this method. + Slices control when state is saved. Specifically, state is saved after a slice has been fully read. + This is useful if the API offers reads by groups or filters, and can be paired with the state object to make reads efficient. See the "concepts" + section of the docs for more information. -class PayPalOauth2Authenticator(Oauth2Authenticator): - """ - curl -v POST https://api-m.sandbox.paypal.com/v1/oauth2/token \ - -H "Accept: application/json" \ - -H "Accept-Language: en_US" \ - -u "CLIENT_ID:SECRET" \ - -d "grant_type=client_credentials" - """ + The function is called before reading any records in a stream. It returns an Iterable of dicts, each containing the + necessary data to craft a request for a slice. The stream state is usually referenced to determine what slices need to be created. + This means that data in a slice is usually closely related to a stream's cursor_field and stream_state. - def get_refresh_request_body(self) -> Mapping[str, Any]: - """ Override to define additional parameters """ - payload: MutableMapping[str, Any] = {"grant_type": "client_credentials"} - return payload + An HTTP request is made for each returned slice. The same slice can be accessed in the path, request_params and request_header functions to help + craft that specific request. - def refresh_access_token(self) -> Tuple[str, int]: - """ - returns a tuple of (access_token, token_lifespan_in_seconds) + For example, if https://example-api.com/v1/employees offers a date query params that returns data for that particular day, one way to implement + this would be to consult the stream state object for the last synced date, then return a slice containing each date from the last synced date + till now. The request_params function would then grab the date from the stream_slice and make it part of the request by injecting it into + the date query param. """ - try: - data = "grant_type=client_credentials" - headers = {"Accept": "application/json", "Accept-Language": "en_US"} - auth = (self.client_id, self.client_secret) - response = requests.request(method="POST", url=self.token_refresh_endpoint, data=data, headers=headers, auth=auth) - response.raise_for_status() - response_json = response.json() - return response_json["access_token"], response_json["expires_in"] - except Exception as e: - raise Exception(f"Error while refreshing access token: {e}") from e + raise NotImplementedError("Implement stream slices or delete this method!") +# Source class SourcePaypalTransaction(AbstractSource): + + url_base = "https://api-m.sandbox.paypal.com" + # url_base = "https://api-m.paypal.com" + def check_connection(self, logger, config) -> Tuple[bool, any]: """ TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API @@ -264,29 +217,14 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ - token = PayPalOauth2Authenticator( - token_refresh_endpoint="https://api-m.sandbox.paypal.com/v1/oauth2/token", - client_id=config["client_id"], - client_secret=config["secret"], - refresh_token="", - ).get_access_token() - if not token: - return False, "Unable to fetch Paypal API token due to incorrect client_id or secret" - return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """58 + """ TODO: Replace the streams below with your own streams. :param config: A Mapping of the user input configuration as defined in the connector spec. """ - authenticator = PayPalOauth2Authenticator( - token_refresh_endpoint="https://api-m.sandbox.paypal.com/v1/oauth2/token", - client_id=config["client_id"], - client_secret=config["secret"], - refresh_token="", - ) - start_date = datetime.strptime(config["start_date"], "%Y-%m-%d").astimezone() - # return [Transactions(authenticator=auth), Balances(authenticator=auth)] - return [Transactions(authenticator=authenticator, start_date=start_date)] + # TODO remove the authenticator if not required. + auth = TokenAuthenticator(token="api_key") # Oauth2Authenticator is also available if you need oauth support + return [Customers(authenticator=auth), Employees(authenticator=auth)] From 14a1927b0f6af3924aacc409c2df2171fae84aca Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 1 Jun 2021 02:54:22 +0300 Subject: [PATCH 21/60] added oauth2 autorization --- .../source_paypal_transaction/source.py | 133 +++++++++--------- 1 file changed, 64 insertions(+), 69 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index d42066a8c7b7..2274782edeeb 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -30,7 +30,7 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator, Oauth2Authenticator """ TODO: Most comments in this class are instructive and should be deleted after the source is implemented. @@ -75,8 +75,8 @@ class PaypalTransactionStream(HttpStream, ABC): See the reference docs for the full list of configurable options. """ - # TODO: Fill in the url base. Required. - url_base = "https://example-api.com/v1/" + url_base = "https://api-m.sandbox.paypal.com/v1/reporting/" + # url_base = "https://api-m.paypal.com/v1/reporting/" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """ @@ -112,100 +112,87 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp yield {} -class Customers(PaypalTransactionStream): +class Transactions(PaypalTransactionStream): """ - TODO: Change class name to match the table/data source this stream corresponds to. + Stream for Transactions /v1/reporting/transactions """ - # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. - primary_key = "customer_id" + primary_key = "transaction_id" def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this should return "customers". Required. """ - return "customers" - - -# Basic incremental stream -class IncrementalPaypalTransactionStream(PaypalTransactionStream, ABC): - """ - TODO fill in details of this class to implement functionality related to incremental syncs for your connector. - if you do not need to implement incremental sync for any streams, remove this class. - """ - - # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. - state_checkpoint_interval = None - @property - def cursor_field(self) -> str: - """ - TODO - Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is - usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. - - :return str: The name of the cursor field. - """ - return [] - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and - the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. - """ - return {} + return "transactions" -class Employees(IncrementalPaypalTransactionStream): +class Balances(PaypalTransactionStream): """ - TODO: Change class name to match the table/data source this stream corresponds to. + Stream for Balances /v1/reporting/balances """ - # TODO: Fill in the cursor_field. Required. - cursor_field = "start_date" - # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. - primary_key = "employee_id" + primary_key = "customer_id" - def path(self, **kwargs) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/employees then this should - return "single". Required. + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: """ - return "employees" - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + should return "customers". Required. """ - TODO: Optionally override this method to define this stream's slices. If slicing is not needed, delete this method. - Slices control when state is saved. Specifically, state is saved after a slice has been fully read. - This is useful if the API offers reads by groups or filters, and can be paired with the state object to make reads efficient. See the "concepts" - section of the docs for more information. + return "balances" - The function is called before reading any records in a stream. It returns an Iterable of dicts, each containing the - necessary data to craft a request for a slice. The stream state is usually referenced to determine what slices need to be created. - This means that data in a slice is usually closely related to a stream's cursor_field and stream_state. - An HTTP request is made for each returned slice. The same slice can be accessed in the path, request_params and request_header functions to help - craft that specific request. +class PayPalOauth2Authenticator(Oauth2Authenticator): + """ + curl -v POST https://api-m.sandbox.paypal.com/v1/oauth2/token \ + -H "Accept: application/json" \ + -H "Accept-Language: en_US" \ + -u "CLIENT_ID:SECRET" \ + -d "grant_type=client_credentials" - For example, if https://example-api.com/v1/employees offers a date query params that returns data for that particular day, one way to implement - this would be to consult the stream state object for the last synced date, then return a slice containing each date from the last synced date - till now. The request_params function would then grab the date from the stream_slice and make it part of the request by injecting it into - the date query param. - """ - raise NotImplementedError("Implement stream slices or delete this method!") + """ + def get_refresh_request_body(self) -> Mapping[str, Any]: + """ Override to define additional parameters """ + payload: MutableMapping[str, Any] = { + "grant_type": "client_credentials" + } + return payload + + def refresh_access_token(self) -> Tuple[str, int]: + """ + returns a tuple of (access_token, token_lifespan_in_seconds) + """ + try: + data = "grant_type=client_credentials" + headers = { + 'Accept': 'application/json', + 'Accept-Language': 'en_US' + } + auth = (self.client_id, self.client_secret) + response = requests.request( + method="POST", + url=self.token_refresh_endpoint, + data=data, + headers=headers, + auth=auth) + response.raise_for_status() + response_json = response.json() + print(response_json) + return response_json["access_token"], response_json["expires_in"] + except Exception as e: + raise Exception(f"Error while refreshing access token: {e}") from e # Source class SourcePaypalTransaction(AbstractSource): - url_base = "https://api-m.sandbox.paypal.com" - # url_base = "https://api-m.paypal.com" - def check_connection(self, logger, config) -> Tuple[bool, any]: """ TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API @@ -217,6 +204,13 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ + token = PayPalOauth2Authenticator( + token_refresh_endpoint='https://api-m.sandbox.paypal.com/v1/oauth2/token', + client_id=config["client_id"], + client_secret=config["secret"], + refresh_token='').get_access_token() + + print(f"token {token}") return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: @@ -227,4 +221,5 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ # TODO remove the authenticator if not required. auth = TokenAuthenticator(token="api_key") # Oauth2Authenticator is also available if you need oauth support - return [Customers(authenticator=auth), Employees(authenticator=auth)] + # return [Transactions(authenticator=auth), Balances(authenticator=auth)] + return [Transactions(authenticator=auth)] From a71ca3773637295eb6e3f17005647dcd8ab5cf25 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 3 Jun 2021 12:16:52 +0300 Subject: [PATCH 22/60] added spec, check, discover + catalogs/configurared_catalogs --- .../integration_tests/configured_catalog.json | 55 +++++++++++++------ .../configured_catalog_transaction.json | 4 +- .../source_paypal_transaction/source.py | 16 ++++-- .../source_paypal_transaction/spec.json | 2 +- 4 files changed, 51 insertions(+), 26 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json index 74de5e17e466..19ebd0f37f06 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json @@ -1,22 +1,22 @@ -// TODO: Construct a configured catalog that can be used for testing. Each stream's `json_schema` field should match the corresponding json schema file. { "streams": [ { "stream": { - "name": "customers", + "name": "transactions", "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "id": { + "account_number": { "type": ["null", "string"] }, - "name": { + "last_refreshed_datetime": { "type": ["null", "string"] }, - "signup_date": { - "type": ["null", "string"], - "format": "date-time" + "page": { + "type": ["null", "integer"] + }, + "total_items": { + "type": ["null", "integer"] } } }, @@ -29,27 +29,46 @@ "stream": { "name": "employees", "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { + "account_id": { "type": ["null", "string"] }, - "years_of_service": { - "type": ["null", "integer"] + "as_of_time": { + "type": ["null", "string"], + "format": "date-time" }, - "start_date": { + "last_refresh_time": { "type": ["null", "string"], "format": "date-time" + }, + "balance": { + "type": ["object", "null"], + "properties": { + "currency": { + "type": ["null", "string"] + }, + "primary": { + "type": ["boolean"] + }, + "total_balance": { + "type": ["object", "null"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + } } } }, - "supported_sync_modes": ["full_refresh", "incremental"] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", + "sync_mode": "full_refresh", "destination_sync_mode": "append" } ] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json index 23ec64e5a5cb..1924ce3bd1a9 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json @@ -20,9 +20,9 @@ } } }, - "supported_sync_modes": ["full_refresh", "incremental"] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", + "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" } ] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 2274782edeeb..26263f9c6001 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -117,7 +117,7 @@ class Transactions(PaypalTransactionStream): Stream for Transactions /v1/reporting/transactions """ - primary_key = "transaction_id" + primary_key = "last_refreshed_datetime" def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None @@ -136,7 +136,7 @@ class Balances(PaypalTransactionStream): """ # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. - primary_key = "customer_id" + primary_key = "as_of_time" def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None @@ -209,6 +209,8 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: client_id=config["client_id"], client_secret=config["secret"], refresh_token='').get_access_token() + if not token: + return False, 'Unable to fetch Paypal API token due to incorrect client_id or secret' print(f"token {token}") return True, None @@ -219,7 +221,11 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: :param config: A Mapping of the user input configuration as defined in the connector spec. """ - # TODO remove the authenticator if not required. - auth = TokenAuthenticator(token="api_key") # Oauth2Authenticator is also available if you need oauth support + token = PayPalOauth2Authenticator( + token_refresh_endpoint='https://api-m.sandbox.paypal.com/v1/oauth2/token', + client_id=config["client_id"], + client_secret=config["secret"], + refresh_token='').get_access_token() + auth = TokenAuthenticator(token=token) # Oauth2Authenticator is also available if you need oauth support # return [Transactions(authenticator=auth), Balances(authenticator=auth)] - return [Transactions(authenticator=auth)] + return [Transactions(authenticator=auth), Balances(authenticator=auth)] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json index 229ac90c7f4c..5cd95638bfac 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json @@ -10,7 +10,7 @@ "client_id": { "title": "Client ID", "type": "string", - "description": "The Paypal Client ID for API credentials", + "description": "The Paypal Client ID for API credentials" }, "secret": { "title": "Secret", From cf3d74ddc8eaac5c9dcdb6c7cd6e5efc0f5407c4 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Fri, 4 Jun 2021 12:27:07 +0300 Subject: [PATCH 23/60] updated request_params --- .../source_paypal_transaction/source.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 26263f9c6001..822d42fe156a 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -25,7 +25,7 @@ from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple - +import datetime import requests from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -102,14 +102,20 @@ def request_params( TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. Usually contains common params e.g. pagination size etc. """ - return {} + return { + 'start_date': '2021-05-15T00:00:00-0700', + 'end_date': '2021-06-15T00:00:00-0700', + 'fields': 'all', + 'page_size': '100' + } def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: """ TODO: Override this method to define how a response is parsed. :return an iterable containing each record in the response """ - yield {} + #yield {} + return [response.json()] class Transactions(PaypalTransactionStream): @@ -228,4 +234,6 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: refresh_token='').get_access_token() auth = TokenAuthenticator(token=token) # Oauth2Authenticator is also available if you need oauth support # return [Transactions(authenticator=auth), Balances(authenticator=auth)] - return [Transactions(authenticator=auth), Balances(authenticator=auth)] + #start_date = datetime.strptime(config["start_date"], "%Y-%m-%d") + #return [Transactions(authenticator=auth), Balances(authenticator=auth)] + return [Transactions(authenticator=auth)] From 647e538ac4cb363f4f21b60d37aa3663b755dd92 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Mon, 7 Jun 2021 20:00:44 +0300 Subject: [PATCH 24/60] added paging, slicing (1d) --- .../source_paypal_transaction/source.py | 94 ++++++++++++++----- 1 file changed, 73 insertions(+), 21 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 822d42fe156a..1735aa3f5876 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -1,7 +1,7 @@ # # MIT License # -# Copyright (c) 2020 Airbyte +# Copyright (c) 2021 Airbyte # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -21,16 +21,18 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # - +import logging +logging.basicConfig(level=logging.DEBUG) from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -import datetime +from datetime import datetime, timedelta + import requests from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator, Oauth2Authenticator +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator, Oauth2Authenticator, HttpAuthenticator """ TODO: Most comments in this class are instructive and should be deleted after the source is implemented. @@ -93,7 +95,13 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. If there are no more pages in the result, return None. """ - return None + decoded_response = response.json() + total_pages = decoded_response.get('total_pages') + page_number = decoded_response.get('page') + if page_number >= total_pages: + return None + else: + return {"page": page_number+1} def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None @@ -102,11 +110,22 @@ def request_params( TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. Usually contains common params e.g. pagination size etc. """ + page_number = 1 + if next_page_token: + page_number = next_page_token.get('page') + + print(f'stream_state {stream_state}') + print(f'stream_slice {stream_slice}') + + start_date = stream_slice['date'] + end_date_dt = datetime.fromisoformat(start_date) + timedelta(days=self.stream_size_in_days) + end_date = end_date_dt.isoformat() return { - 'start_date': '2021-05-15T00:00:00-0700', - 'end_date': '2021-06-15T00:00:00-0700', + 'start_date': start_date, + 'end_date': end_date, 'fields': 'all', - 'page_size': '100' + 'page_size': '1', + 'page': page_number } def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: @@ -114,26 +133,61 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp TODO: Override this method to define how a response is parsed. :return an iterable containing each record in the response """ - #yield {} - return [response.json()] + json_response = response.json() + records = json_response.get(self.data_field, []) if self.data_field is not None else json_response + yield from records class Transactions(PaypalTransactionStream): """ Stream for Transactions /v1/reporting/transactions """ + data_field = "transaction_details" + primary_key = "transaction_id" + cursor_field = "date" + stream_size_in_days = 1 - primary_key = "last_refreshed_datetime" + def __init__(self, start_date: datetime, **kwargs): + super().__init__(**kwargs) + self.start_date = start_date def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: + return "transactions" + + # def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: + # # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state + # # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. + # if current_stream_state is not None and 'date' in current_stream_state: + # current_parsed_date = datetime.strptime(current_stream_state['date'], '%Y-%m-%d') + # latest_record_date = datetime.strptime(latest_record['date'], '%Y-%m-%d') + # return {'date': max(current_parsed_date, latest_record_date).strftime('%Y-%m-%d')} + # else: + # return {'date': self.start_date.strftime('%Y-%m-%d')} + + def _chunk_date_range(self, start_date: datetime) -> List[Mapping[str, any]]: """ - Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this - should return "customers". Required. + Returns a list of each day between the start date and now. + The return value is a list of dicts {'date': date_string}. """ + dates = [] + while start_date < datetime.now().astimezone(): + dates.append({'date': start_date.isoformat()}) + start_date += timedelta(days=self.stream_size_in_days) - return "transactions" + print(f'dates {dates}') + return dates + + def stream_slices( + self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, any]]]: + + start_date = self.start_date + if stream_state and 'date' in stream_state: + start_date = datetime.fromisoformat(stream_state['date']) + + return self._chunk_date_range(start_date) class Balances(PaypalTransactionStream): @@ -141,17 +195,14 @@ class Balances(PaypalTransactionStream): Stream for Balances /v1/reporting/balances """ + data_field = "transaction_details" + # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. primary_key = "as_of_time" def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this - should return "customers". Required. - """ - return "balances" @@ -234,6 +285,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: refresh_token='').get_access_token() auth = TokenAuthenticator(token=token) # Oauth2Authenticator is also available if you need oauth support # return [Transactions(authenticator=auth), Balances(authenticator=auth)] - #start_date = datetime.strptime(config["start_date"], "%Y-%m-%d") + print(f'TOKEN: {token}') + start_date = datetime.strptime(config["start_date"], "%Y-%m-%d").astimezone() #return [Transactions(authenticator=auth), Balances(authenticator=auth)] - return [Transactions(authenticator=auth)] + return [Transactions(authenticator=auth, start_date=start_date)] From 3aabc39ab1115916b3e044e08f774682c1926855 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Mon, 7 Jun 2021 21:51:58 +0300 Subject: [PATCH 25/60] Use oath2 for paypal --- .../source_paypal_transaction/source.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 1735aa3f5876..dbbb0880e482 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -213,7 +213,6 @@ class PayPalOauth2Authenticator(Oauth2Authenticator): -H "Accept-Language: en_US" \ -u "CLIENT_ID:SECRET" \ -d "grant_type=client_credentials" - """ def get_refresh_request_body(self) -> Mapping[str, Any]: """ Override to define additional parameters """ @@ -241,13 +240,11 @@ def refresh_access_token(self) -> Tuple[str, int]: auth=auth) response.raise_for_status() response_json = response.json() - print(response_json) return response_json["access_token"], response_json["expires_in"] except Exception as e: raise Exception(f"Error while refreshing access token: {e}") from e -# Source class SourcePaypalTransaction(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: @@ -269,7 +266,6 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: if not token: return False, 'Unable to fetch Paypal API token due to incorrect client_id or secret' - print(f"token {token}") return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: @@ -278,14 +274,11 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: :param config: A Mapping of the user input configuration as defined in the connector spec. """ - token = PayPalOauth2Authenticator( + authenticator = PayPalOauth2Authenticator( token_refresh_endpoint='https://api-m.sandbox.paypal.com/v1/oauth2/token', client_id=config["client_id"], client_secret=config["secret"], - refresh_token='').get_access_token() - auth = TokenAuthenticator(token=token) # Oauth2Authenticator is also available if you need oauth support - # return [Transactions(authenticator=auth), Balances(authenticator=auth)] - print(f'TOKEN: {token}') + refresh_token='') start_date = datetime.strptime(config["start_date"], "%Y-%m-%d").astimezone() #return [Transactions(authenticator=auth), Balances(authenticator=auth)] - return [Transactions(authenticator=auth, start_date=start_date)] + return [Transactions(authenticator=authenticator, start_date=start_date)] From f7a6d82ef176d3f86e0adc2ef25944c5da5a3f43 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 10 Jun 2021 12:06:08 +0300 Subject: [PATCH 26/60] incremental sync, acceptance test --- .../acceptance-test-config.yml | 2 +- .../configured_catalog_transaction.json | 4 +- .../integration_tests/invalid_config.json | 6 +- .../integration_tests/sample_config.json | 6 +- .../integration_tests/sample_state.json | 4 +- .../source_paypal_transaction/source.py | 122 ++++++++++-------- 6 files changed, 78 insertions(+), 66 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml index e38b0cf2c95b..08d0e07e3577 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml @@ -8,7 +8,7 @@ tests: - config_path: "secrets/config.json" status: "succeed" - config_path: "integration_tests/invalid_config.json" - status: "exception" + status: "failed" discovery: - config_path: "secrets/config.json" basic_read: diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json index 1924ce3bd1a9..23ec64e5a5cb 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json @@ -20,9 +20,9 @@ } } }, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "full_refresh", + "sync_mode": "incremental", "destination_sync_mode": "overwrite" } ] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json index f3732995784f..52839c5ac77e 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json @@ -1,3 +1,5 @@ { - "todo-wrong-field": "this should be an incomplete config file, used in standard tests" -} + "client_id": "AWAz___", + "secret": "ENC8__", + "start_date": "2021-06-01" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json index ecc4913b84c7..fff6d380f653 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json @@ -1,3 +1,5 @@ { - "fix-me": "TODO" -} + "client_id": "AWAz0aQCdk", + "secret": "ENC8PBtgHBH-", + "start_date": "2021-06-01" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json index 3587e579822d..dd461b09ceb6 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json @@ -1,5 +1,5 @@ { - "todo-stream-name": { - "todo-field-name": "value" + "transactions": { + "date": "2021-06-04T17:34:43+00:00" } } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index dbbb0880e482..45040b04014e 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -1,7 +1,7 @@ # # MIT License # -# Copyright (c) 2021 Airbyte +# Copyright (c) 2020 Airbyte # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -21,19 +21,19 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # -import logging -logging.basicConfig(level=logging.DEBUG) +import logging from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple from datetime import datetime, timedelta +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator, Oauth2Authenticator, HttpAuthenticator +from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator +logging.basicConfig(level=logging.DEBUG) """ TODO: Most comments in this class are instructive and should be deleted after the source is implemented. @@ -96,12 +96,12 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, If there are no more pages in the result, return None. """ decoded_response = response.json() - total_pages = decoded_response.get('total_pages') - page_number = decoded_response.get('page') + total_pages = decoded_response.get("total_pages") + page_number = decoded_response.get("page") if page_number >= total_pages: return None else: - return {"page": page_number+1} + return {"page": page_number + 1} def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None @@ -112,21 +112,17 @@ def request_params( """ page_number = 1 if next_page_token: - page_number = next_page_token.get('page') + page_number = next_page_token.get("page") - print(f'stream_state {stream_state}') - print(f'stream_slice {stream_slice}') - - start_date = stream_slice['date'] + start_date = stream_slice["date"] end_date_dt = datetime.fromisoformat(start_date) + timedelta(days=self.stream_size_in_days) + + date_time_now = datetime.now().astimezone() + if end_date_dt > date_time_now: + end_date_dt = date_time_now + end_date = end_date_dt.isoformat() - return { - 'start_date': start_date, - 'end_date': end_date, - 'fields': 'all', - 'page_size': '1', - 'page': page_number - } + return {"start_date": start_date, "end_date": end_date, "fields": "all", "page_size": "1", "page": page_number} def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: """ @@ -137,14 +133,27 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp records = json_response.get(self.data_field, []) if self.data_field is not None else json_response yield from records + @staticmethod + def get_field(record: Mapping[str, Any], field_path: List[str]): + + data = record + for attr in field_path: + if data: + data = data.get(attr) + else: + break + + return data + class Transactions(PaypalTransactionStream): """ Stream for Transactions /v1/reporting/transactions """ + data_field = "transaction_details" primary_key = "transaction_id" - cursor_field = "date" + cursor_field = ["transaction_info", "transaction_initiation_date"] stream_size_in_days = 1 def __init__(self, start_date: datetime, **kwargs): @@ -156,15 +165,24 @@ def path( ) -> str: return "transactions" - # def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: - # # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state - # # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. - # if current_stream_state is not None and 'date' in current_stream_state: - # current_parsed_date = datetime.strptime(current_stream_state['date'], '%Y-%m-%d') - # latest_record_date = datetime.strptime(latest_record['date'], '%Y-%m-%d') - # return {'date': max(current_parsed_date, latest_record_date).strftime('%Y-%m-%d')} - # else: - # return {'date': self.start_date.strftime('%Y-%m-%d')} + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: + # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state + # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. + latest_record_date_str = self.get_field(latest_record, self.cursor_field) + + if current_stream_state and "date" in current_stream_state and latest_record_date_str: + if len(latest_record_date_str) == 24: + # Add ':' to timezone part to match iso format, example: + # python iso format: 2021-06-04T00:00:00+03:00 + # format from record: 2021-06-04T00:00:00+0300 + latest_record_date_str = ":".join([latest_record_date_str[:22], latest_record_date_str[22:]]) + + latest_record_date = datetime.fromisoformat(latest_record_date_str) + current_parsed_date = datetime.fromisoformat(current_stream_state["date"]) + + return {"date": max(current_parsed_date, latest_record_date).isoformat()} + else: + return {"date": self.start_date.isoformat()} def _chunk_date_range(self, start_date: datetime) -> List[Mapping[str, any]]: """ @@ -172,11 +190,9 @@ def _chunk_date_range(self, start_date: datetime) -> List[Mapping[str, any]]: The return value is a list of dicts {'date': date_string}. """ dates = [] - while start_date < datetime.now().astimezone(): - dates.append({'date': start_date.isoformat()}) + while start_date < datetime.now().astimezone() - timedelta(days=2): + dates.append({"date": start_date.isoformat()}) start_date += timedelta(days=self.stream_size_in_days) - - print(f'dates {dates}') return dates def stream_slices( @@ -184,8 +200,8 @@ def stream_slices( ) -> Iterable[Optional[Mapping[str, any]]]: start_date = self.start_date - if stream_state and 'date' in stream_state: - start_date = datetime.fromisoformat(stream_state['date']) + if stream_state and "date" in stream_state: + start_date = datetime.fromisoformat(stream_state["date"]) return self._chunk_date_range(start_date) @@ -214,11 +230,10 @@ class PayPalOauth2Authenticator(Oauth2Authenticator): -u "CLIENT_ID:SECRET" \ -d "grant_type=client_credentials" """ + def get_refresh_request_body(self) -> Mapping[str, Any]: """ Override to define additional parameters """ - payload: MutableMapping[str, Any] = { - "grant_type": "client_credentials" - } + payload: MutableMapping[str, Any] = {"grant_type": "client_credentials"} return payload def refresh_access_token(self) -> Tuple[str, int]: @@ -227,17 +242,9 @@ def refresh_access_token(self) -> Tuple[str, int]: """ try: data = "grant_type=client_credentials" - headers = { - 'Accept': 'application/json', - 'Accept-Language': 'en_US' - } + headers = {"Accept": "application/json", "Accept-Language": "en_US"} auth = (self.client_id, self.client_secret) - response = requests.request( - method="POST", - url=self.token_refresh_endpoint, - data=data, - headers=headers, - auth=auth) + response = requests.request(method="POST", url=self.token_refresh_endpoint, data=data, headers=headers, auth=auth) response.raise_for_status() response_json = response.json() return response_json["access_token"], response_json["expires_in"] @@ -246,7 +253,6 @@ def refresh_access_token(self) -> Tuple[str, int]: class SourcePaypalTransaction(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: """ TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API @@ -259,26 +265,28 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ token = PayPalOauth2Authenticator( - token_refresh_endpoint='https://api-m.sandbox.paypal.com/v1/oauth2/token', + token_refresh_endpoint="https://api-m.sandbox.paypal.com/v1/oauth2/token", client_id=config["client_id"], client_secret=config["secret"], - refresh_token='').get_access_token() + refresh_token="", + ).get_access_token() if not token: - return False, 'Unable to fetch Paypal API token due to incorrect client_id or secret' + return False, "Unable to fetch Paypal API token due to incorrect client_id or secret" return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ + """58 TODO: Replace the streams below with your own streams. :param config: A Mapping of the user input configuration as defined in the connector spec. """ authenticator = PayPalOauth2Authenticator( - token_refresh_endpoint='https://api-m.sandbox.paypal.com/v1/oauth2/token', + token_refresh_endpoint="https://api-m.sandbox.paypal.com/v1/oauth2/token", client_id=config["client_id"], client_secret=config["secret"], - refresh_token='') + refresh_token="", + ) start_date = datetime.strptime(config["start_date"], "%Y-%m-%d").astimezone() - #return [Transactions(authenticator=auth), Balances(authenticator=auth)] + # return [Transactions(authenticator=auth), Balances(authenticator=auth)] return [Transactions(authenticator=authenticator, start_date=start_date)] From f21cf64b8c9150410551f10d1f96637b82bb2b8b Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Mon, 14 Jun 2021 00:45:42 +0300 Subject: [PATCH 27/60] updated slices and api limits, added validation for input dates --- .../acceptance-test-config.yml | 9 +- .../integration_tests/abnormal_state.json | 4 +- .../integration_tests/configured_catalog.json | 50 +-------- .../integration_tests/invalid_config.json | 3 +- .../source_paypal_transaction/source.py | 106 ++++++++++++++---- .../source_paypal_transaction/spec.json | 12 +- 6 files changed, 107 insertions(+), 77 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml index 08d0e07e3577..60a2b0dfeffc 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml @@ -21,10 +21,11 @@ tests: # extra_fields: no # exact_order: no # extra_records: yes - incremental: # TODO if your connector does not implement incremental sync, remove this block - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - state_path: "integration_tests/abnormal_state.json" +# incremental: # TODO if your connector does not implement incremental sync, remove this block +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# state_path: "integration_tests/state.json" +# # state_path: "integration_tests/abnormal_state.json" full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json index 52b0f2c2118f..10d0414ad11e 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json @@ -1,5 +1,5 @@ { - "todo-stream-name": { - "todo-field-name": "todo-abnormal-value" + "transactions": { + "date": "2021-05-04T17:34:43+00:00" } } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json index 19ebd0f37f06..23ec64e5a5cb 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json @@ -20,56 +20,10 @@ } } }, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "full_refresh", + "sync_mode": "incremental", "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "employees", - "json_schema": { - "type": "object", - "properties": { - "account_id": { - "type": ["null", "string"] - }, - "as_of_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "last_refresh_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "balance": { - "type": ["object", "null"], - "properties": { - "currency": { - "type": ["null", "string"] - }, - "primary": { - "type": ["boolean"] - }, - "total_balance": { - "type": ["object", "null"], - "properties": { - "currency_code": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json index 52839c5ac77e..462481557435 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json @@ -1,5 +1,6 @@ { "client_id": "AWAz___", "secret": "ENC8__", - "start_date": "2021-06-01" + "start_date": "2021-06-01T05:00:00+03:00", + "end_date": "2021-06-04T17:00:00+03:00" } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 45040b04014e..0a0235efe436 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -78,6 +78,8 @@ class PaypalTransactionStream(HttpStream, ABC): """ url_base = "https://api-m.sandbox.paypal.com/v1/reporting/" + page_size = "500" # API limit + # url_base = "https://api-m.paypal.com/v1/reporting/" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -114,15 +116,13 @@ def request_params( if next_page_token: page_number = next_page_token.get("page") - start_date = stream_slice["date"] - end_date_dt = datetime.fromisoformat(start_date) + timedelta(days=self.stream_size_in_days) - - date_time_now = datetime.now().astimezone() - if end_date_dt > date_time_now: - end_date_dt = date_time_now - - end_date = end_date_dt.isoformat() - return {"start_date": start_date, "end_date": end_date, "fields": "all", "page_size": "1", "page": page_number} + return { + "start_date": stream_slice["start_date"], + "end_date": stream_slice["end_date"], + "fields": "all", + "page_size": self.page_size, + "page": page_number, + } def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: """ @@ -154,11 +154,46 @@ class Transactions(PaypalTransactionStream): data_field = "transaction_details" primary_key = "transaction_id" cursor_field = ["transaction_info", "transaction_initiation_date"] - stream_size_in_days = 1 - def __init__(self, start_date: datetime, **kwargs): + stream_slice_period: Mapping[str, int] = { + "days": 1 + } + # Date limits are needed to prevent API error: Data for the given start date is not available + start_date_limits: Mapping[str, Mapping] = { + "min_date": {"days": 3 * 364}, # 3 years + "max_date": {"hours": 12} + } + + def __init__(self, start_date: datetime, end_date: datetime, **kwargs): super().__init__(**kwargs) + + self._validate_input_dates(start_date=start_date, end_date=end_date) + self.start_date = start_date + self.end_date = end_date + + def _validate_input_dates(self, start_date, end_date): + + # Validate input dates + if start_date > end_date: + raise Exception(f"start_date {start_date} is greater than end_date {end_date}") + + current_date = datetime.now().replace(microsecond=0).astimezone() + current_date_delta = current_date - start_date + + # Check for minimal possible start_date + if current_date_delta > timedelta(**self.start_date_limits.get("min_date")): + raise Exception( + f"Start_date {start_date.isoformat()} is too old. " + f"Min date limit is {self.start_date_limits.get('min_date')} before now:{current_date.isoformat()}." + ) + + # Check for maximum possible start_date + if current_date_delta < timedelta(**self.start_date_limits.get("max_date")): + raise Exception( + f"Start_date {start_date.isoformat()} is too close to now. " + f"Max date limit is {self.start_date_limits.get('max_date')} before now:{current_date.isoformat()}." + ) def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None @@ -166,6 +201,7 @@ def path( return "transactions" def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: + # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. latest_record_date_str = self.get_field(latest_record, self.cursor_field) @@ -186,13 +222,21 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late def _chunk_date_range(self, start_date: datetime) -> List[Mapping[str, any]]: """ - Returns a list of each day between the start date and now. - The return value is a list of dicts {'date': date_string}. + Returns a list of each day (by default) between the start date and end date. + The return value is a list of dicts {'start_date': date_string, 'end_date': date_string}. """ dates = [] - while start_date < datetime.now().astimezone() - timedelta(days=2): - dates.append({"date": start_date.isoformat()}) - start_date += timedelta(days=self.stream_size_in_days) + + # start date should not be less than 12 hrs before current time, otherwise API throws an error: + # 'message': 'Data for the given start date is not available.' + start_date_limit_max = self.end_date - timedelta(**self.start_date_limits.get("max_date")) - timedelta(**self.stream_slice_period) + while start_date < start_date_limit_max: + end_date = start_date + timedelta(**self.stream_slice_period) + dates.append({"start_date": start_date.isoformat(), "end_date": end_date.isoformat()}) + start_date = end_date + + dates.append({"start_date": start_date.isoformat(), "end_date": self.end_date.isoformat()}) + return dates def stream_slices( @@ -253,6 +297,7 @@ def refresh_access_token(self) -> Tuple[str, int]: class SourcePaypalTransaction(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: """ TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API @@ -264,15 +309,30 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ - token = PayPalOauth2Authenticator( + start_date = datetime.fromisoformat(config["start_date"]) + end_date_str = config["end_date"] + if end_date_str: + end_date = datetime.fromisoformat(end_date_str) + else: + end_date = datetime.now().replace(microsecond=0).astimezone() + + authenticator = PayPalOauth2Authenticator( token_refresh_endpoint="https://api-m.sandbox.paypal.com/v1/oauth2/token", client_id=config["client_id"], client_secret=config["secret"], refresh_token="", - ).get_access_token() + ) + # Try to get API TOKEN + token = authenticator.get_access_token() if not token: return False, "Unable to fetch Paypal API token due to incorrect client_id or secret" + # Try to initiate a stream and validate input date params + try: + Transactions(authenticator=authenticator, start_date=start_date, end_date=end_date) + except Exception as e: + return False, e + return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: @@ -287,6 +347,12 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: client_secret=config["secret"], refresh_token="", ) - start_date = datetime.strptime(config["start_date"], "%Y-%m-%d").astimezone() + + start_date = datetime.fromisoformat(config["start_date"]) + end_date_str = config["end_date"] + if end_date_str: + end_date = datetime.fromisoformat(end_date_str) + else: + end_date = datetime.now().replace(microsecond=0).astimezone() # return [Transactions(authenticator=auth), Balances(authenticator=auth)] - return [Transactions(authenticator=authenticator, start_date=start_date)] + return [Transactions(authenticator=authenticator, start_date=start_date, end_date=end_date)] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json index 5cd95638bfac..c4cbf1900e5c 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json @@ -21,8 +21,16 @@ "start_date": { "type": "string", "title": "Start Date", - "description": "Start Date for data extraction in Internet date and time format https://datatracker.ietf.org/doc/html/rfc3339#section-5.6", - "examples": ["2021-06-11T23:59:59-0700"] + "description": "Start Date for data extraction in ISO format. Date must be in range from 3 years till 12 hrs before present time", + "examples": ["2021-06-11T23:59:59-00:00"], + "pattern": ["^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])[T,t]([0-1][0-9]|2[0-3]):[0-5][0-9]:([0-5][0-9]|60)([.][0-9]+)?([Zz]|[+-][0-9]{2}:[0-9]{2})$"] + }, + "end_date": { + "type": "string", + "title": "End Date", + "description": "End Date for data extraction in ISO format. if not specified, then current date/time is used by default", + "examples": ["2021-06-11T23:59:59-00:00"], + "pattern": ["^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])[T,t]([0-1][0-9]|2[0-3]):[0-5][0-9]:([0-5][0-9]|60)([.][0-9]+)?([Zz]|[+-][0-9]{2}:[0-9]{2})$"] } } } From 99f8b2df07176c5368f1ef4faba00f867d25993e Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 15 Jun 2021 02:17:49 +0300 Subject: [PATCH 28/60] added tests, fixed cursor related information in schemas and configured catalogs, removed old comments, re-arranged Base PaypalTransactionStream class --- .../acceptance-test-config.yml | 13 +- .../integration_tests/abnormal_state.json | 5 +- .../integration_tests/configured_catalog.json | 328 +++++++++++++++++- .../configured_catalog_balances.json | 77 ++++ .../configured_catalog_transaction.json | 29 -- .../configured_catalog_transactions.json | 267 ++++++++++++++ .../integration_tests/state.json | 3 + .../schemas/balances.json | 57 ++- .../schemas/customers.json | 16 - .../schemas/employees.json | 19 - .../schemas/transactions.json | 251 +++++++++++++- .../source_paypal_transaction/source.py | 313 +++++++++-------- 12 files changed, 1126 insertions(+), 252 deletions(-) create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_balances.json delete mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json delete mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/customers.json delete mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/employees.json diff --git a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml index 60a2b0dfeffc..cceb92e15404 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml @@ -21,11 +21,14 @@ tests: # extra_fields: no # exact_order: no # extra_records: yes -# incremental: # TODO if your connector does not implement incremental sync, remove this block -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# state_path: "integration_tests/state.json" -# # state_path: "integration_tests/abnormal_state.json" full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_transactions.json" + # state_path: "integration_tests/state.json" + future_state_path: "integration_tests/abnormal_state.json" + cursor_paths: + transactions: ["date"] + diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json index 10d0414ad11e..eefd4d0fd893 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json @@ -1,5 +1,8 @@ { "transactions": { - "date": "2021-05-04T17:34:43+00:00" + "date": "2021-06-05T02:00:00+00:00" + }, + "balances": { + "date": "2021-06-05T02:00:00+00:00" } } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json index 23ec64e5a5cb..d05c8474fc37 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json @@ -4,26 +4,338 @@ "stream": { "name": "transactions", "json_schema": { + "$schema": "http://json-schema.org/schema#", "type": "object", "properties": { - "account_number": { - "type": ["null", "string"] + "transaction_info": { + "type": "object", + "properties": { + "paypal_account_id": { + "type": "string" + }, + "transaction_id": { + "type": "string" + }, + "transaction_event_code": { + "type": "string" + }, + "transaction_initiation_date": { + "type": "string" + }, + "transaction_updated_date": { + "type": "string" + }, + "transaction_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "fee_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "insurance_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "shipping_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "shipping_discount_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "transaction_status": { + "type": "string" + }, + "transaction_subject": { + "type": "string" + }, + "transaction_note": { + "type": "string" + }, + "invoice_id": { + "type": "string" + }, + "custom_field": { + "type": "string" + }, + "protection_eligibility": { + "type": "string" + } + } }, - "last_refreshed_datetime": { - "type": ["null", "string"] + "payer_info": { + "type": "object", + "properties": { + "account_id": { + "type": "string" + }, + "email_address": { + "type": "string" + }, + "address_status": { + "type": "string" + }, + "payer_status": { + "type": "string" + }, + "payer_name": { + "type": "object", + "properties": { + "given_name": { + "type": "string" + }, + "surname": { + "type": "string" + }, + "alternate_full_name": { + "type": "string" + } + } + }, + "country_code": { + "type": "string" + } + } }, - "page": { - "type": ["null", "integer"] + "shipping_info": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "address": { + "type": "object", + "properties": { + "line1": { + "type": "string" + }, + "line2": { + "type": "string" + }, + "city": { + "type": "string" + }, + "country_code": { + "type": "string" + }, + "postal_code": { + "type": "string" + } + } + } + } }, - "total_items": { - "type": ["null", "integer"] + "cart_info": { + "type": "object", + "properties": { + "item_details": { + "type": "array", + "items": { + "type": "object", + "properties": { + "item_code": { + "type": "string" + }, + "item_name": { + "type": "string" + }, + "item_description": { + "type": "string" + }, + "item_quantity": { + "type": "string" + }, + "item_unit_price": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "item_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "tax_amounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "tax_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + } + } + } + }, + "total_item_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "invoice_number": { + "type": "string" + } + } + } + } + } + }, + "store_info": { + "type": "object" + }, + "auction_info": { + "type": "object" + }, + "incentive_info": { + "type": "object" } } }, + "source_defined_cursor": true, + "default_cursor_field": [ + "transaction_info", + "transaction_initiation_date" + ], "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "balances", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "account_id": { + "type": "string" + }, + "as_of_time": { + "type": "string" + }, + "balances": { + "type": "array", + "items": { + "type": "object", + "properties": { + "available_balance": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "currency": { + "type": "string" + }, + "primary": { + "type": "boolean" + }, + "total_balance": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "withheld_balance": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + } + } + } + }, + "last_refresh_time": { + "type": "string" + } + } + }, + "default_cursor_field": ["as_of_time"], + "supported_sync_modes": [ + "full_refresh", + "incremental" + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_balances.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_balances.json new file mode 100644 index 000000000000..ccc9a20b32ca --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_balances.json @@ -0,0 +1,77 @@ +{ + "streams": [ + { + "stream": { + "name": "balances", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "account_id": { + "type": "string" + }, + "as_of_time": { + "type": "string" + }, + "balances": { + "type": "array", + "items": { + "type": "object", + "properties": { + "available_balance": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "currency": { + "type": "string" + }, + "primary": { + "type": "boolean" + }, + "total_balance": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "withheld_balance": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + } + } + } + }, + "last_refresh_time": { + "type": "string" + } + } + }, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json deleted file mode 100644 index 23ec64e5a5cb..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "transactions", - "json_schema": { - "type": "object", - "properties": { - "account_number": { - "type": ["null", "string"] - }, - "last_refreshed_datetime": { - "type": ["null", "string"] - }, - "page": { - "type": ["null", "integer"] - }, - "total_items": { - "type": ["null", "integer"] - } - } - }, - "supported_sync_modes": ["full_refresh", "incremental"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json new file mode 100644 index 000000000000..e0d1ee789eab --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json @@ -0,0 +1,267 @@ +{ + "streams": [ + { + "stream": { + "name": "transactions", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "transaction_info": { + "type": "object", + "properties": { + "paypal_account_id": { + "type": "string" + }, + "transaction_id": { + "type": "string" + }, + "transaction_event_code": { + "type": "string" + }, + "transaction_initiation_date": { + "type": "string" + }, + "transaction_updated_date": { + "type": "string" + }, + "transaction_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "fee_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "insurance_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "shipping_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "shipping_discount_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "transaction_status": { + "type": "string" + }, + "transaction_subject": { + "type": "string" + }, + "transaction_note": { + "type": "string" + }, + "invoice_id": { + "type": "string" + }, + "custom_field": { + "type": "string" + }, + "protection_eligibility": { + "type": "string" + } + } + }, + "payer_info": { + "type": "object", + "properties": { + "account_id": { + "type": "string" + }, + "email_address": { + "type": "string" + }, + "address_status": { + "type": "string" + }, + "payer_status": { + "type": "string" + }, + "payer_name": { + "type": "object", + "properties": { + "given_name": { + "type": "string" + }, + "surname": { + "type": "string" + }, + "alternate_full_name": { + "type": "string" + } + } + }, + "country_code": { + "type": "string" + } + } + }, + "shipping_info": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "address": { + "type": "object", + "properties": { + "line1": { + "type": "string" + }, + "line2": { + "type": "string" + }, + "city": { + "type": "string" + }, + "country_code": { + "type": "string" + }, + "postal_code": { + "type": "string" + } + } + } + } + }, + "cart_info": { + "type": "object", + "properties": { + "item_details": { + "type": "array", + "items": { + "type": "object", + "properties": { + "item_code": { + "type": "string" + }, + "item_name": { + "type": "string" + }, + "item_description": { + "type": "string" + }, + "item_quantity": { + "type": "string" + }, + "item_unit_price": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "item_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "tax_amounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "tax_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + } + } + } + }, + "total_item_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "invoice_number": { + "type": "string" + } + } + } + } + } + }, + "store_info": { + "type": "object" + }, + "auction_info": { + "type": "object" + }, + "incentive_info": { + "type": "object" + } + } + }, + "source_defined_cursor": true, + "default_cursor_field": [ + "transaction_info", + "transaction_initiation_date" + ], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json index dd461b09ceb6..55e16864a243 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json @@ -1,5 +1,8 @@ { "transactions": { "date": "2021-06-04T17:34:43+00:00" + }, + "balances": { + "date": "2021-06-04T17:34:43+00:00" } } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json index 69c6b92d6e80..e75c1085da15 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json @@ -1,38 +1,59 @@ { + "$schema": "http://json-schema.org/schema#", "type": "object", "properties": { - "account_id": { - "type": ["null", "string"] - }, - "as_of_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "last_refresh_time": { - "type": ["null", "string"], - "format": "date-time" - }, "balance": { - "type": ["object", "null"], + "type": "object", "properties": { "currency": { - "type": ["null", "string"] + "type": "string" }, "primary": { - "type": ["boolean"] + "type": "boolean" }, "total_balance": { - "type": ["object", "null"], + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "available_balance": { + "type": "object", "properties": { "currency_code": { - "type": ["null", "string"] + "type": "string" }, "value": { - "type": ["null", "string"] + "type": "string" + } + } + }, + "withheld_balance": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" } } } } + }, + "account_id": { + "type": "string" + }, + "as_of_time": { + "type": "datetime" + }, + "last_refresh_time": { + "type": "datetime" } } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/customers.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/customers.json deleted file mode 100644 index 9a4b13485836..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/customers.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "signup_date": { - "type": ["null", "string"], - "format": "date-time" - } - } -} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/employees.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/employees.json deleted file mode 100644 index 2fa01a0fa1ff..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/employees.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "years_of_service": { - "type": ["null", "integer"] - }, - "start_date": { - "type": ["null", "string"], - "format": "date-time" - } - } -} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json index 981c82a6a552..f5c7cf8818a3 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json @@ -1,17 +1,250 @@ { + "$schema": "http://json-schema.org/schema#", "type": "object", "properties": { - "account_number": { - "type": ["null", "string"] + "transaction_info": { + "type": "object", + "properties": { + "paypal_account_id": { + "type": "string" + }, + "transaction_id": { + "type": "string" + }, + "transaction_event_code": { + "type": "string" + }, + "transaction_initiation_date": { + "type": "datetime" + }, + "transaction_updated_date": { + "type": "datetime" + }, + "transaction_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "fee_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "insurance_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "shipping_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "shipping_discount_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "transaction_status": { + "type": "string" + }, + "transaction_subject": { + "type": "string" + }, + "transaction_note": { + "type": "string" + }, + "invoice_id": { + "type": "string" + }, + "custom_field": { + "type": "string" + }, + "protection_eligibility": { + "type": "string" + } + } }, - "last_refreshed_datetime": { - "type": ["null", "string"] + "payer_info": { + "type": "object", + "properties": { + "account_id": { + "type": "string" + }, + "email_address": { + "type": "string" + }, + "address_status": { + "type": "string" + }, + "payer_status": { + "type": "string" + }, + "payer_name": { + "type": "object", + "properties": { + "given_name": { + "type": "string" + }, + "surname": { + "type": "string" + }, + "alternate_full_name": { + "type": "string" + } + } + }, + "country_code": { + "type": "string" + } + } }, - "page": { - "type": ["null", "integer"] + "shipping_info": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "address": { + "type": "object", + "properties": { + "line1": { + "type": "string" + }, + "line2": { + "type": "string" + }, + "city": { + "type": "string" + }, + "country_code": { + "type": "string" + }, + "postal_code": { + "type": "string" + } + } + } + } }, - "total_items": { - "type": ["null", "integer"] + "cart_info": { + "type": "object", + "properties": { + "item_details": { + "type": "array", + "items": { + "type": "object", + "properties": { + "item_code": { + "type": "string" + }, + "item_name": { + "type": "string" + }, + "item_description": { + "type": "string" + }, + "item_quantity": { + "type": "string" + }, + "item_unit_price": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "item_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "tax_amounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "tax_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + } + } + } + }, + "total_item_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "invoice_number": { + "type": "string" + } + } + } + } + } + }, + "store_info": { + "type": "object" + }, + "auction_info": { + "type": "object" + }, + "incentive_info": { + "type": "object" } } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 0a0235efe436..3a6621646707 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -25,7 +25,7 @@ import logging from abc import ABC from datetime import datetime, timedelta -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Callable import requests from airbyte_cdk.sources import AbstractSource @@ -33,137 +33,28 @@ from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator -logging.basicConfig(level=logging.DEBUG) -""" -TODO: Most comments in this class are instructive and should be deleted after the source is implemented. - -This file provides a stubbed example of how to use the Airbyte CDK to develop both a source connector which supports full refresh or and an -incremental syncs from an HTTP API. - -The various TODOs are both implementation hints and steps - fulfilling all the TODOs should be sufficient to implement one basic and one incremental -stream from a source. This pattern is the same one used by Airbyte internally to implement connectors. +from dateutil.parser import isoparse -The approach here is not authoritative, and devs are free to use their own judgement. - -There are additional required TODOs in the files within the integration_tests folder and the spec.json file. -""" +# TODO remove +logging.basicConfig(level=logging.DEBUG) -# Basic full refresh stream class PaypalTransactionStream(HttpStream, ABC): - """ - TODO remove this comment - This class represents a stream output by the connector. - This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, - parsing responses etc.. - - Each stream should extend this class (or another abstract subclass of it) to specify behavior unique to that stream. - - Typically for REST APIs each stream corresponds to a resource in the API. For example if the API - contains the endpoints - - GET v1/customers - - GET v1/employees - - then you should have three classes: - `class PaypalTransactionStream(HttpStream, ABC)` which is the current class - `class Customers(PaypalTransactionStream)` contains behavior to pull data for customers using v1/customers - `class Employees(PaypalTransactionStream)` contains behavior to pull data for employees using v1/employees - - If some streams implement incremental sync, it is typical to create another class - `class IncrementalPaypalTransactionStream((PaypalTransactionStream), ABC)` then have concrete stream implementations extend it. An example - is provided below. - - See the reference docs for the full list of configurable options. - """ + sandbox = True - url_base = "https://api-m.sandbox.paypal.com/v1/reporting/" page_size = "500" # API limit - # url_base = "https://api-m.paypal.com/v1/reporting/" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. - - This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed - to most other methods in this class to help you form headers, request bodies, query params, etc.. - - For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a - 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. - The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. - - :param response: the most recent response from the API - :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. - If there are no more pages in the result, return None. - """ - decoded_response = response.json() - total_pages = decoded_response.get("total_pages") - page_number = decoded_response.get("page") - if page_number >= total_pages: - return None - else: - return {"page": page_number + 1} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - """ - TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. - Usually contains common params e.g. pagination size etc. - """ - page_number = 1 - if next_page_token: - page_number = next_page_token.get("page") - - return { - "start_date": stream_slice["start_date"], - "end_date": stream_slice["end_date"], - "fields": "all", - "page_size": self.page_size, - "page": page_number, - } - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - TODO: Override this method to define how a response is parsed. - :return an iterable containing each record in the response - """ - json_response = response.json() - records = json_response.get(self.data_field, []) if self.data_field is not None else json_response - yield from records - - @staticmethod - def get_field(record: Mapping[str, Any], field_path: List[str]): - - data = record - for attr in field_path: - if data: - data = data.get(attr) - else: - break - - return data - - -class Transactions(PaypalTransactionStream): - """ - Stream for Transactions /v1/reporting/transactions - """ - - data_field = "transaction_details" - primary_key = "transaction_id" - cursor_field = ["transaction_info", "transaction_initiation_date"] - - stream_slice_period: Mapping[str, int] = { - "days": 1 - } # Date limits are needed to prevent API error: Data for the given start date is not available start_date_limits: Mapping[str, Mapping] = { - "min_date": {"days": 3 * 364}, # 3 years + "min_date": {"days": 3 * 364}, # API limit - 3 years "max_date": {"hours": 12} } + stream_slice_period: Mapping[str, int] = { + "days": 1 + } + def __init__(self, start_date: datetime, end_date: datetime, **kwargs): super().__init__(**kwargs) @@ -195,30 +86,82 @@ def _validate_input_dates(self, start_date, end_date): f"Max date limit is {self.start_date_limits.get('max_date')} before now:{current_date.isoformat()}." ) - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "transactions" + @property + def url_base(self) -> str: - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: + if self.sandbox: + url_base = "https://api-m.sandbox.paypal.com/v1/reporting/" + else: + url_base = "https://api-m.paypal.com/v1/reporting/" - # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state - # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. - latest_record_date_str = self.get_field(latest_record, self.cursor_field) + return url_base - if current_stream_state and "date" in current_stream_state and latest_record_date_str: - if len(latest_record_date_str) == 24: - # Add ':' to timezone part to match iso format, example: - # python iso format: 2021-06-04T00:00:00+03:00 - # format from record: 2021-06-04T00:00:00+0300 - latest_record_date_str = ":".join([latest_record_date_str[:22], latest_record_date_str[22:]]) + def request_headers( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Mapping[str, Any]: - latest_record_date = datetime.fromisoformat(latest_record_date_str) - current_parsed_date = datetime.fromisoformat(current_stream_state["date"]) + return {'Content-Type': 'application/json'} - return {"date": max(current_parsed_date, latest_record_date).isoformat()} + def parse_response__(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + TODO: Override this method to define how a response is parsed. + :return an iterable containing each record in the response + """ + json_response = response.json() + if self.data_field is not None: + yield from json_response.get(self.data_field, []) else: - return {"date": self.start_date.isoformat()} + yield json_response + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + TODO: Override this method to define how a response is parsed. + :return an iterable containing each record in the response + """ + json_response = response.json() + if self.data_field is not None: + data = json_response.get(self.data_field, []) + else: + data = [json_response] + + for record in data: + # In order to support direct datetime string comparison (which is performed in incremental acceptance tests) + # convert any date format to python iso format string for date based cursors + self.update_field(record, self.cursor_field, lambda x: isoparse(x).isoformat()) + yield record + + @staticmethod + def update_field(record: Mapping[str, Any], field_path: List[str], update: Callable[[Any], None]): + + data = record + if not isinstance(field_path, List): + field_path = [field_path] + + for attr in field_path[:-1]: + if data and isinstance(data, dict): + data = data.get(attr) + else: + break + + last_field = field_path[-1] + data[last_field] = update(data[last_field]) + + return data + + @staticmethod + def get_field(record: Mapping[str, Any], field_path: List[str]): + + data = record + if not isinstance(field_path, List): + field_path = [field_path] + + for attr in field_path: + if data and isinstance(data, dict): + data = data.get(attr) + else: + break + + return data def _chunk_date_range(self, start_date: datetime) -> List[Mapping[str, any]]: """ @@ -245,26 +188,99 @@ def stream_slices( start_date = self.start_date if stream_state and "date" in stream_state: - start_date = datetime.fromisoformat(stream_state["date"]) + start_date = isoparse(stream_state["date"]) return self._chunk_date_range(start_date) + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: -class Balances(PaypalTransactionStream): + # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state + # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. + latest_record_date_str: str = self.get_field(latest_record, self.cursor_field) + + if current_stream_state and "date" in current_stream_state and latest_record_date_str: + # isoparse supports different formats, like: + # python iso format: 2021-06-04T00:00:00+03:00 + # format from transactions record: 2021-06-04T00:00:00+0300 + # format from balances record: 2021-06-02T00:00:00Z + latest_record_date = isoparse(latest_record_date_str) + current_parsed_date = isoparse(current_stream_state["date"]) + + return {"date": max(current_parsed_date, latest_record_date).isoformat()} + else: + return {"date": self.start_date.isoformat()} + + +class Transactions(PaypalTransactionStream): """ - Stream for Balances /v1/reporting/balances + Stream for Transactions /v1/reporting/transactions """ data_field = "transaction_details" + primary_key = ["transaction_info", "transaction_id"] + cursor_field = ["transaction_info", "transaction_initiation_date"] + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + + return "transactions" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + + decoded_response = response.json() + total_pages = decoded_response.get("total_pages") + page_number = decoded_response.get("page") + if page_number >= total_pages: + return None + else: + return {"page": page_number + 1} + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + + page_number = 1 + if next_page_token: + page_number = next_page_token.get("page") + + return { + "start_date": stream_slice["start_date"], + "end_date": stream_slice["end_date"], + "fields": "all", + "page_size": self.page_size, + "page": page_number, + } + + +class Balances(PaypalTransactionStream): + """ + Stream for Balances /v1/reporting/balances + """ - # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. primary_key = "as_of_time" + cursor_field = "as_of_time" + data_field = None def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: + return "balances" + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + + return { + "as_of_time": stream_slice["start_date"], + # "currency_code": "USD" + } + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + + return None + class PayPalOauth2Authenticator(Oauth2Authenticator): """ @@ -309,10 +325,10 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ - start_date = datetime.fromisoformat(config["start_date"]) + start_date = isoparse(config["start_date"]) end_date_str = config["end_date"] if end_date_str: - end_date = datetime.fromisoformat(end_date_str) + end_date = isoparse(end_date_str) else: end_date = datetime.now().replace(microsecond=0).astimezone() @@ -348,11 +364,14 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: refresh_token="", ) - start_date = datetime.fromisoformat(config["start_date"]) + start_date = isoparse(config["start_date"]) end_date_str = config["end_date"] if end_date_str: - end_date = datetime.fromisoformat(end_date_str) + end_date = isoparse(end_date_str) else: end_date = datetime.now().replace(microsecond=0).astimezone() - # return [Transactions(authenticator=auth), Balances(authenticator=auth)] - return [Transactions(authenticator=authenticator, start_date=start_date, end_date=end_date)] + + return [ + Transactions(authenticator=authenticator, start_date=start_date, end_date=end_date), + Balances(authenticator=authenticator, start_date=start_date, end_date=end_date) + ] From a74542ae6abeb87f261bb7d455b1ec8520569daf Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 17 Jun 2021 13:51:37 +0300 Subject: [PATCH 29/60] added input param 'env' to support production and sandbox envs --- .../source_paypal_transaction/source.py | 58 ++++++------------- .../source_paypal_transaction/spec.json | 5 ++ 2 files changed, 24 insertions(+), 39 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 3a6621646707..cd450286bbd9 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -22,22 +22,17 @@ # SOFTWARE. # -import logging from abc import ABC from datetime import datetime, timedelta -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Callable +from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator - from dateutil.parser import isoparse -# TODO remove -logging.basicConfig(level=logging.DEBUG) - class PaypalTransactionStream(HttpStream, ABC): @@ -46,22 +41,18 @@ class PaypalTransactionStream(HttpStream, ABC): page_size = "500" # API limit # Date limits are needed to prevent API error: Data for the given start date is not available - start_date_limits: Mapping[str, Mapping] = { - "min_date": {"days": 3 * 364}, # API limit - 3 years - "max_date": {"hours": 12} - } + start_date_limits: Mapping[str, Mapping] = {"min_date": {"days": 3 * 364}, "max_date": {"hours": 12}} # API limit - 3 years - stream_slice_period: Mapping[str, int] = { - "days": 1 - } + stream_slice_period: Mapping[str, int] = {"days": 1} - def __init__(self, start_date: datetime, end_date: datetime, **kwargs): + def __init__(self, start_date: datetime, end_date: datetime, env: str = "Production", **kwargs): super().__init__(**kwargs) self._validate_input_dates(start_date=start_date, end_date=end_date) self.start_date = start_date self.end_date = end_date + self.env = env def _validate_input_dates(self, start_date, end_date): @@ -89,10 +80,10 @@ def _validate_input_dates(self, start_date, end_date): @property def url_base(self) -> str: - if self.sandbox: - url_base = "https://api-m.sandbox.paypal.com/v1/reporting/" - else: + if self.env == "Production": url_base = "https://api-m.paypal.com/v1/reporting/" + else: + url_base = "https://api-m.sandbox.paypal.com/v1/reporting/" return url_base @@ -100,24 +91,10 @@ def request_headers( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> Mapping[str, Any]: - return {'Content-Type': 'application/json'} - - def parse_response__(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - TODO: Override this method to define how a response is parsed. - :return an iterable containing each record in the response - """ - json_response = response.json() - if self.data_field is not None: - yield from json_response.get(self.data_field, []) - else: - yield json_response + return {"Content-Type": "application/json"} def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - TODO: Override this method to define how a response is parsed. - :return an iterable containing each record in the response - """ + json_response = response.json() if self.data_field is not None: data = json_response.get(self.data_field, []) @@ -274,7 +251,6 @@ def request_params( return { "as_of_time": stream_slice["start_date"], - # "currency_code": "USD" } def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -313,7 +289,6 @@ def refresh_access_token(self) -> Tuple[str, int]: class SourcePaypalTransaction(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: """ TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API @@ -332,8 +307,13 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: else: end_date = datetime.now().replace(microsecond=0).astimezone() + if config["environment"] == "Production": + url_auth = "https://api-m.paypal.com/v1/oauth2/token" + else: + url_auth = "https://api-m.sandbox.paypal.com/v1/oauth2/token" + authenticator = PayPalOauth2Authenticator( - token_refresh_endpoint="https://api-m.sandbox.paypal.com/v1/oauth2/token", + token_refresh_endpoint=url_auth, client_id=config["client_id"], client_secret=config["secret"], refresh_token="", @@ -345,7 +325,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: # Try to initiate a stream and validate input date params try: - Transactions(authenticator=authenticator, start_date=start_date, end_date=end_date) + Transactions(authenticator=authenticator, start_date=start_date, end_date=end_date, env=config["environment"]) except Exception as e: return False, e @@ -372,6 +352,6 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: end_date = datetime.now().replace(microsecond=0).astimezone() return [ - Transactions(authenticator=authenticator, start_date=start_date, end_date=end_date), - Balances(authenticator=authenticator, start_date=start_date, end_date=end_date) + Transactions(authenticator=authenticator, start_date=start_date, end_date=end_date, env=config["environment"]), + Balances(authenticator=authenticator, start_date=start_date, end_date=end_date, env=config["environment"]), ] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json index c4cbf1900e5c..b16cce345c17 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json @@ -31,6 +31,11 @@ "description": "End Date for data extraction in ISO format. if not specified, then current date/time is used by default", "examples": ["2021-06-11T23:59:59-00:00"], "pattern": ["^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])[T,t]([0-1][0-9]|2[0-3]):[0-5][0-9]:([0-5][0-9]|60)([.][0-9]+)?([Zz]|[+-][0-9]{2}:[0-9]{2})$"] + }, + "environment": { + "type": "string", + "description": "Production or Sandbox environment to extract data from", + "enum": ["Production", "Sandbox"] } } } From e404aa876ea6b8c44e43ac40b822b79da3427f55 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Sat, 19 Jun 2021 01:51:00 +0300 Subject: [PATCH 30/60] added support for sandbox option, updated pattern for optional end date option --- .../configured_catalog_transactions.json | 2 +- .../integration_tests/invalid_config.json | 4 +- .../integration_tests/sample_config.json | 7 +- .../integration_tests/sample_state.json | 3 + .../source_paypal_transaction/source.py | 98 +++++++++++-------- .../source_paypal_transaction/spec.json | 15 +-- 6 files changed, 75 insertions(+), 54 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json index e0d1ee789eab..41a224a1588b 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json @@ -261,7 +261,7 @@ "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json index 462481557435..e3960dc221ab 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json @@ -1,6 +1,6 @@ { "client_id": "AWAz___", "secret": "ENC8__", - "start_date": "2021-06-01T05:00:00+03:00", - "end_date": "2021-06-04T17:00:00+03:00" + "start_date": "2000-06-01T05:00:00+03:00", + "is_sandbox": false } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json index fff6d380f653..a9035251e13f 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json @@ -1,5 +1,6 @@ { - "client_id": "AWAz0aQCdk", - "secret": "ENC8PBtgHBH-", - "start_date": "2021-06-01" + "client_id": "PAYPAL_CLIENT_ID", + "secret": "PAYPAL_SECRET", + "start_date": "2021-06-01T00:00:00+00:00", + "is_sandbox": false } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json index dd461b09ceb6..55e16864a243 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json @@ -1,5 +1,8 @@ { "transactions": { "date": "2021-06-04T17:34:43+00:00" + }, + "balances": { + "date": "2021-06-04T17:34:43+00:00" } } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index cd450286bbd9..366d91d7f7ec 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -34,25 +34,45 @@ from dateutil.parser import isoparse -class PaypalTransactionStream(HttpStream, ABC): +def get_endpoint(is_sandbox: bool = False) -> str: + if is_sandbox: + endpoint = "https://api-m.sandbox.paypal.com" + else: + endpoint = "https://api-m.paypal.com" + return endpoint + + +def get_end_date(config: Mapping[str, Any]) -> datetime: + end_date = None + now = datetime.now().replace(microsecond=0).astimezone() + if "end_date" in config and config["end_date"]: + end_date = isoparse(config["end_date"]) + + # If date is in future then set it to now: + if not end_date or end_date > now: + end_date = now + + return end_date - sandbox = True + +class PaypalTransactionStream(HttpStream, ABC): page_size = "500" # API limit # Date limits are needed to prevent API error: Data for the given start date is not available - start_date_limits: Mapping[str, Mapping] = {"min_date": {"days": 3 * 364}, "max_date": {"hours": 12}} # API limit - 3 years + # API limit: 3 years < start_date < 12 hrs before now + start_date_limits: Mapping[str, Mapping] = {"min_date": {"days": 3 * 364}, "max_date": {"hours": 12}} stream_slice_period: Mapping[str, int] = {"days": 1} - def __init__(self, start_date: datetime, end_date: datetime, env: str = "Production", **kwargs): + def __init__(self, start_date: datetime, end_date: datetime, is_sandbox: bool = False, **kwargs): super().__init__(**kwargs) self._validate_input_dates(start_date=start_date, end_date=end_date) self.start_date = start_date self.end_date = end_date - self.env = env + self.is_sandbox = is_sandbox def _validate_input_dates(self, start_date, end_date): @@ -80,12 +100,7 @@ def _validate_input_dates(self, start_date, end_date): @property def url_base(self) -> str: - if self.env == "Production": - url_base = "https://api-m.paypal.com/v1/reporting/" - else: - url_base = "https://api-m.sandbox.paypal.com/v1/reporting/" - - return url_base + return f"{get_endpoint(self.is_sandbox)}/v1/reporting/" def request_headers( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None @@ -257,6 +272,20 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, return None + def stream_slices( + self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, any]]]: + + slices = super().stream_slices(sync_mode, cursor_field, stream_state) + + # Add one more slice to extract balance at the time of 'end_date' + last_slice = slices[-1] + if last_slice["start_date"] != last_slice["end_date"]: + start_date = end_date = last_slice["end_date"] + slices.append({"start_date": start_date, "end_date": end_date}) + + return slices + class PayPalOauth2Authenticator(Oauth2Authenticator): """ @@ -267,6 +296,15 @@ class PayPalOauth2Authenticator(Oauth2Authenticator): -d "grant_type=client_credentials" """ + def __init__(self, config): + + super().__init__( + token_refresh_endpoint=f"{get_endpoint(config['is_sandbox'])}/v1/oauth2/token", + client_id=config["client_id"], + client_secret=config["secret"], + refresh_token="", + ) + def get_refresh_request_body(self) -> Mapping[str, Any]: """ Override to define additional parameters """ payload: MutableMapping[str, Any] = {"grant_type": "client_credentials"} @@ -301,23 +339,9 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ start_date = isoparse(config["start_date"]) - end_date_str = config["end_date"] - if end_date_str: - end_date = isoparse(end_date_str) - else: - end_date = datetime.now().replace(microsecond=0).astimezone() + end_date = get_end_date(config) - if config["environment"] == "Production": - url_auth = "https://api-m.paypal.com/v1/oauth2/token" - else: - url_auth = "https://api-m.sandbox.paypal.com/v1/oauth2/token" - - authenticator = PayPalOauth2Authenticator( - token_refresh_endpoint=url_auth, - client_id=config["client_id"], - client_secret=config["secret"], - refresh_token="", - ) + authenticator = PayPalOauth2Authenticator(config) # Try to get API TOKEN token = authenticator.get_access_token() if not token: @@ -325,7 +349,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: # Try to initiate a stream and validate input date params try: - Transactions(authenticator=authenticator, start_date=start_date, end_date=end_date, env=config["environment"]) + Transactions(authenticator=authenticator, start_date=start_date, end_date=end_date, is_sandbox=config["is_sandbox"]) except Exception as e: return False, e @@ -337,21 +361,13 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: :param config: A Mapping of the user input configuration as defined in the connector spec. """ - authenticator = PayPalOauth2Authenticator( - token_refresh_endpoint="https://api-m.sandbox.paypal.com/v1/oauth2/token", - client_id=config["client_id"], - client_secret=config["secret"], - refresh_token="", - ) + + authenticator = PayPalOauth2Authenticator(config) start_date = isoparse(config["start_date"]) - end_date_str = config["end_date"] - if end_date_str: - end_date = isoparse(end_date_str) - else: - end_date = datetime.now().replace(microsecond=0).astimezone() + end_date = get_end_date(config) return [ - Transactions(authenticator=authenticator, start_date=start_date, end_date=end_date, env=config["environment"]), - Balances(authenticator=authenticator, start_date=start_date, end_date=end_date, env=config["environment"]), + Transactions(authenticator=authenticator, start_date=start_date, end_date=end_date, is_sandbox=config["is_sandbox"]), + Balances(authenticator=authenticator, start_date=start_date, end_date=end_date, is_sandbox=config["is_sandbox"]), ] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json index b16cce345c17..e1a9bb5cef08 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Paypal Transaction Search", "type": "object", - "required": ["client_id", "secret", "start_date"], + "required": ["client_id", "secret", "start_date", "is_sandbox"], "additionalProperties": false, "properties": { "client_id": { @@ -23,19 +23,20 @@ "title": "Start Date", "description": "Start Date for data extraction in ISO format. Date must be in range from 3 years till 12 hrs before present time", "examples": ["2021-06-11T23:59:59-00:00"], - "pattern": ["^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])[T,t]([0-1][0-9]|2[0-3]):[0-5][0-9]:([0-5][0-9]|60)([.][0-9]+)?([Zz]|[+-][0-9]{2}:[0-9]{2})$"] + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}[+-][0-9]{2}:[0-9]{2}$" }, "end_date": { "type": "string", "title": "End Date", "description": "End Date for data extraction in ISO format. if not specified, then current date/time is used by default", "examples": ["2021-06-11T23:59:59-00:00"], - "pattern": ["^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])[T,t]([0-1][0-9]|2[0-3]):[0-5][0-9]:([0-5][0-9]|60)([.][0-9]+)?([Zz]|[+-][0-9]{2}:[0-9]{2})$"] + "pattern": "^$|[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}[+-][0-9]{2}:[0-9]{2}$" }, - "environment": { - "type": "string", - "description": "Production or Sandbox environment to extract data from", - "enum": ["Production", "Sandbox"] + "is_sandbox": { + "title": "Is Sandbox", + "description": "Whether or not to Sandbox or Production environment to extract data from", + "type": "boolean", + "default": false } } } From cc7c6fc81e498d14c7f58c32991270500fc4d181 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Sat, 19 Jun 2021 02:58:50 +0300 Subject: [PATCH 31/60] added github secrets --- .github/workflows/publish-command.yml | 1 + .github/workflows/test-command.yml | 1 + tools/bin/ci_credentials.sh | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 866ffc26c66a..c866539bffab 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -85,6 +85,7 @@ jobs: MICROSOFT_TEAMS_TEST_CREDS: ${{ secrets.MICROSOFT_TEAMS_TEST_CREDS }} MIXPANEL_INTEGRATION_TEST_CREDS: ${{ secrets.MIXPANEL_INTEGRATION_TEST_CREDS }} MSSQL_RDS_TEST_CREDS: ${{ secrets.MSSQL_RDS_TEST_CREDS }} + PAYPAL_TRANSACTION_CREDS: ${{ secrets.SOURCE_PAYPAL_TRANSACTION_CREDS }} POSTHOG_TEST_CREDS: ${{ secrets.POSTHOG_TEST_CREDS }} RECHARGE_INTEGRATION_TEST_CREDS: ${{ secrets.RECHARGE_INTEGRATION_TEST_CREDS }} QUICKBOOKS_TEST_CREDS: ${{ secrets.QUICKBOOKS_TEST_CREDS }} diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index a9fdfbeaaab3..124c7dccf9ce 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -85,6 +85,7 @@ jobs: MICROSOFT_TEAMS_TEST_CREDS: ${{ secrets.MICROSOFT_TEAMS_TEST_CREDS }} MIXPANEL_INTEGRATION_TEST_CREDS: ${{ secrets.MIXPANEL_INTEGRATION_TEST_CREDS }} MSSQL_RDS_TEST_CREDS: ${{ secrets.MSSQL_RDS_TEST_CREDS }} + PAYPAL_TRANSACTION_CREDS: ${{ secrets.SOURCE_PAYPAL_TRANSACTION_CREDS }} POSTHOG_TEST_CREDS: ${{ secrets.POSTHOG_TEST_CREDS }} RECHARGE_INTEGRATION_TEST_CREDS: ${{ secrets.RECHARGE_INTEGRATION_TEST_CREDS }} QUICKBOOKS_TEST_CREDS: ${{ secrets.QUICKBOOKS_TEST_CREDS }} diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index e33d14f48e4c..fcf8dc836f63 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -65,6 +65,7 @@ write_standard_creds source-marketo-singer "$SOURCE_MARKETO_SINGER_INTEGRATION_T write_standard_creds source-microsoft-teams "$MICROSOFT_TEAMS_TEST_CREDS" write_standard_creds source-mixpanel-singer "$MIXPANEL_INTEGRATION_TEST_CREDS" write_standard_creds source-mssql "$MSSQL_RDS_TEST_CREDS" +write_standard_creds source-paypal-transaction "$PAYPAL_TRANSACTION_CREDS" write_standard_creds source-posthog "$POSTHOG_TEST_CREDS" write_standard_creds source-quickbooks-singer "$QUICKBOOKS_TEST_CREDS" write_standard_creds source-okta "$SOURCE_OKTA_TEST_CREDS" From 3416eba702379145e85d810e4c57c8d34966024d Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Sun, 20 Jun 2021 21:53:43 +0300 Subject: [PATCH 32/60] added support for sandbox option, updated pattern for optional end date option --- .../acceptance-test-config.yml | 7 - .../integration_tests/abnormal_state.json | 4 +- .../integration_tests/state.json | 4 +- .../source_paypal_transaction/source.py | 199 ++++++++++-------- 4 files changed, 121 insertions(+), 93 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml index cceb92e15404..1e86e6269464 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml @@ -15,19 +15,12 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" validate_output_from_all_streams: yes -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.txt" -# extra_fields: no -# exact_order: no -# extra_records: yes full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog_transactions.json" - # state_path: "integration_tests/state.json" future_state_path: "integration_tests/abnormal_state.json" cursor_paths: transactions: ["date"] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json index eefd4d0fd893..f466dc277261 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json @@ -1,8 +1,8 @@ { "transactions": { - "date": "2021-06-05T02:00:00+00:00" + "date": "2021-06-18T16:00:00+00:00" }, "balances": { - "date": "2021-06-05T02:00:00+00:00" + "date": "2021-06-18T16:00:00+00:00" } } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json index 55e16864a243..dc08b6e0fe0e 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json @@ -1,8 +1,8 @@ { "transactions": { - "date": "2021-06-04T17:34:43+00:00" + "date": "2021-06-18T16:24:13+03:00" }, "balances": { - "date": "2021-06-04T17:34:43+00:00" + "date": "2021-06-18T16:24:13+03:00" } } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 366d91d7f7ec..4ea0e31731f4 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -24,6 +24,7 @@ from abc import ABC from datetime import datetime, timedelta +from pprint import pprint from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests @@ -60,41 +61,68 @@ class PaypalTransactionStream(HttpStream, ABC): page_size = "500" # API limit # Date limits are needed to prevent API error: Data for the given start date is not available - # API limit: 3 years < start_date < 12 hrs before now - start_date_limits: Mapping[str, Mapping] = {"min_date": {"days": 3 * 364}, "max_date": {"hours": 12}} + # API limit: (now() - start_date_min) < start_date < (now() - start_date_max) + start_date_min: Mapping[str, int] = {"days": 3 * 364} # API limit - 3 years + start_date_max: Mapping[str, int] = {"hours": 0} + + stream_slice_period: Mapping[str, int] = {"days": 1} # max period is 31 days (API limit) + + def __init__( + self, + start_date: Optional[datetime] = None, + end_date: Optional[datetime] = None, + is_sandbox: bool = False, + config: Optional[Mapping[str, Any]] = None, + **kwargs, + ): + + # Initiate data from config + if config: + start_date = isoparse(config["start_date"]) + end_date_str = config.get("end_date") + end_date = isoparse(end_date_str) if end_date_str else None + is_sandbox = config["is_sandbox"] - stream_slice_period: Mapping[str, int] = {"days": 1} + self.is_sandbox = is_sandbox + self.start_date = start_date + self._end_date = end_date + self.is_sandbox = is_sandbox - def __init__(self, start_date: datetime, end_date: datetime, is_sandbox: bool = False, **kwargs): - super().__init__(**kwargs) + self._validate_input_dates() - self._validate_input_dates(start_date=start_date, end_date=end_date) + super().__init__(**kwargs) - self.start_date = start_date - self.end_date = end_date - self.is_sandbox = is_sandbox + @property + def end_date(self): + """Return initiated end_date or now()""" + now = datetime.now().replace(microsecond=0).astimezone() + if not self._end_date or self._end_date > now: + # If no end_date or end_date is in future then return now: + return now + else: + return self._end_date - def _validate_input_dates(self, start_date, end_date): + def _validate_input_dates(self): # Validate input dates - if start_date > end_date: - raise Exception(f"start_date {start_date} is greater than end_date {end_date}") + if self.start_date > self.end_date: + raise Exception(f"start_date {self.start_date.isoformat()} is greater than end_date {self.end_date.isoformat()}") current_date = datetime.now().replace(microsecond=0).astimezone() - current_date_delta = current_date - start_date + current_date_delta = current_date - self.start_date # Check for minimal possible start_date - if current_date_delta > timedelta(**self.start_date_limits.get("min_date")): + if current_date_delta > timedelta(**self.start_date_min): raise Exception( - f"Start_date {start_date.isoformat()} is too old. " - f"Min date limit is {self.start_date_limits.get('min_date')} before now:{current_date.isoformat()}." + f"Start_date {self.start_date.isoformat()} is too old. " + f"Min date limit is {self.start_date_min} before now:{current_date.isoformat()}." ) # Check for maximum possible start_date - if current_date_delta < timedelta(**self.start_date_limits.get("max_date")): + if current_date_delta < timedelta(**self.start_date_max): raise Exception( - f"Start_date {start_date.isoformat()} is too close to now. " - f"Max date limit is {self.start_date_limits.get('max_date')} before now:{current_date.isoformat()}." + f"Start_date {self.start_date.isoformat()} is too close to now. " + f"Max date limit is {self.start_date_max} before now:{current_date.isoformat()}." ) @property @@ -119,7 +147,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp for record in data: # In order to support direct datetime string comparison (which is performed in incremental acceptance tests) # convert any date format to python iso format string for date based cursors - self.update_field(record, self.cursor_field, lambda x: isoparse(x).isoformat()) + self.update_field(record, self.cursor_field, lambda date: isoparse(date).isoformat()) yield record @staticmethod @@ -155,35 +183,6 @@ def get_field(record: Mapping[str, Any], field_path: List[str]): return data - def _chunk_date_range(self, start_date: datetime) -> List[Mapping[str, any]]: - """ - Returns a list of each day (by default) between the start date and end date. - The return value is a list of dicts {'start_date': date_string, 'end_date': date_string}. - """ - dates = [] - - # start date should not be less than 12 hrs before current time, otherwise API throws an error: - # 'message': 'Data for the given start date is not available.' - start_date_limit_max = self.end_date - timedelta(**self.start_date_limits.get("max_date")) - timedelta(**self.stream_slice_period) - while start_date < start_date_limit_max: - end_date = start_date + timedelta(**self.stream_slice_period) - dates.append({"start_date": start_date.isoformat(), "end_date": end_date.isoformat()}) - start_date = end_date - - dates.append({"start_date": start_date.isoformat(), "end_date": self.end_date.isoformat()}) - - return dates - - def stream_slices( - self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, any]]]: - - start_date = self.start_date - if stream_state and "date" in stream_state: - start_date = isoparse(stream_state["date"]) - - return self._chunk_date_range(start_date) - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state @@ -212,6 +211,8 @@ class Transactions(PaypalTransactionStream): primary_key = ["transaction_info", "transaction_id"] cursor_field = ["transaction_info", "transaction_initiation_date"] + start_date_max = {"hours": 36} # this limit is found experimentally + def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: @@ -223,10 +224,10 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, decoded_response = response.json() total_pages = decoded_response.get("total_pages") page_number = decoded_response.get("page") - if page_number >= total_pages: - return None - else: + if page_number < total_pages: return {"page": page_number + 1} + else: + return None def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None @@ -244,6 +245,42 @@ def request_params( "page": page_number, } + def stream_slices( + self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, any]]]: + """ + Returns a list of slices for each day (by default) between the start date and end date. + The return value is a list of dicts {'start_date': date_string, 'end_date': date_string}. + """ + if stream_state and stream_state.get("date"): + start_date = isoparse(stream_state.get("date")) + else: + start_date = self.start_date + + # choose first slice period (for the most recent time) + first_slice_period = max(timedelta(**self.start_date_max), timedelta(**self.stream_slice_period)) + end_date = self.end_date + + start_date_slice = end_date - first_slice_period + end_date_slice = end_date + + # Do not run any requests if start_date is less than 36 hrs before current time, otherwise API throws an error: + # 'message': 'Data for the given start date is not available.' + if start_date > start_date_slice or \ + start_date > end_date: + return [] + + dates = [] + while start_date < start_date_slice: + dates.append({"start_date": start_date_slice.isoformat(), "end_date": end_date_slice.isoformat()}) + end_date_slice = start_date_slice + start_date_slice = start_date_slice - timedelta(**self.stream_slice_period) + + # add last (the oldest) slice period + dates.append({"start_date": start_date.isoformat(), "end_date": end_date_slice.isoformat()}) + pprint(dates) + return dates[::-1] # inverse stream slices to start read requests from the oldest slices + class Balances(PaypalTransactionStream): """ @@ -275,20 +312,33 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, any]]]: + """ + Returns a list of slices for each day (by default) between the start_date and end_date (containing the last) + The return value is a list of dicts {'start_date': date_string}. + """ + if stream_state and stream_state.get("date"): + if isoparse(stream_state.get("date")) == self._end_date: + # Do not run any incremental requests if _end_date has been reached already + return [] + # For incremental sync don't extract balance at stream_state, because + # it has been already extracted in previous scan. Start from next slice period instead: + start_date_slice = isoparse(stream_state.get("date")) + timedelta(**self.stream_slice_period) + else: + start_date_slice = self.start_date - slices = super().stream_slices(sync_mode, cursor_field, stream_state) - - # Add one more slice to extract balance at the time of 'end_date' - last_slice = slices[-1] - if last_slice["start_date"] != last_slice["end_date"]: - start_date = end_date = last_slice["end_date"] - slices.append({"start_date": start_date, "end_date": end_date}) + dates = [] + while start_date_slice < self.end_date: + dates.append({"start_date": start_date_slice.isoformat()}) + start_date_slice = start_date_slice + timedelta(**self.stream_slice_period) - return slices + # Add last (the newest) slice with the current time of the sync + dates.append({"start_date": self.end_date.isoformat()}) + pprint(dates) + return dates class PayPalOauth2Authenticator(Oauth2Authenticator): - """ + """Request example for API token extraction: curl -v POST https://api-m.sandbox.paypal.com/v1/oauth2/token \ -H "Accept: application/json" \ -H "Accept-Language: en_US" \ @@ -306,9 +356,7 @@ def __init__(self, config): ) def get_refresh_request_body(self) -> Mapping[str, Any]: - """ Override to define additional parameters """ - payload: MutableMapping[str, Any] = {"grant_type": "client_credentials"} - return payload + return {"grant_type": "client_credentials"} def refresh_access_token(self) -> Tuple[str, int]: """ @@ -329,19 +377,12 @@ def refresh_access_token(self) -> Tuple[str, int]: class SourcePaypalTransaction(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: """ - TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API - - See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 - for an example. - :param config: the user-input config object conforming to the connector's spec.json :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ - start_date = isoparse(config["start_date"]) - end_date = get_end_date(config) - authenticator = PayPalOauth2Authenticator(config) + # Try to get API TOKEN token = authenticator.get_access_token() if not token: @@ -349,25 +390,19 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: # Try to initiate a stream and validate input date params try: - Transactions(authenticator=authenticator, start_date=start_date, end_date=end_date, is_sandbox=config["is_sandbox"]) + Transactions(authenticator=authenticator, config=config) except Exception as e: return False, e return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """58 - TODO: Replace the streams below with your own streams. - + """ :param config: A Mapping of the user input configuration as defined in the connector spec. """ - authenticator = PayPalOauth2Authenticator(config) - start_date = isoparse(config["start_date"]) - end_date = get_end_date(config) - return [ - Transactions(authenticator=authenticator, start_date=start_date, end_date=end_date, is_sandbox=config["is_sandbox"]), - Balances(authenticator=authenticator, start_date=start_date, end_date=end_date, is_sandbox=config["is_sandbox"]), + Transactions(authenticator=authenticator, config=config), + Balances(authenticator=authenticator, config=config), ] From b6acfbf1e9c4bae92777495eeefa655a4f3a8d45 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Mon, 21 Jun 2021 11:26:55 +0300 Subject: [PATCH 33/60] fixed Copyright date, removed debug mesages --- .../integration_tests/abnormal_state.json | 4 +- .../integration_tests/acceptance.py | 2 +- .../integration_tests/catalog.json | 39 --- .../integration_tests/configured_catalog.json | 314 +----------------- .../configured_catalog_transaction.json | 29 -- .../configured_catalog_transactions.json | 251 +------------- .../source-paypal-transaction/main.py | 2 +- .../source-paypal-transaction/setup.py | 2 +- .../source_paypal_transaction/__init__.py | 2 +- .../source_paypal_transaction/source.py | 23 +- .../unit_tests/unit_test.py | 2 +- 11 files changed, 14 insertions(+), 656 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/catalog.json delete mode 100644 airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json index f466dc277261..1761a6998ad9 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json @@ -1,8 +1,8 @@ { "transactions": { - "date": "2021-06-18T16:00:00+00:00" + "date": "2021-06-18T11:59:59+00:00" }, "balances": { - "date": "2021-06-18T16:00:00+00:00" + "date": "2021-06-18T11:59:59+00:00" } } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py index eeb4a2d3e02e..2827823a2402 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py @@ -1,7 +1,7 @@ # # MIT License # -# Copyright (c) 2020 Airbyte +# Copyright (c) 2021 Airbyte # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/catalog.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/catalog.json deleted file mode 100644 index 6799946a6851..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/catalog.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "streams": [ - { - "name": "TODO fix this file", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": "column1", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } - }, - { - "name": "table1", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } - } - ] -} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json index d05c8474fc37..0557887ec8c2 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json @@ -3,256 +3,7 @@ { "stream": { "name": "transactions", - "json_schema": { - "$schema": "http://json-schema.org/schema#", - "type": "object", - "properties": { - "transaction_info": { - "type": "object", - "properties": { - "paypal_account_id": { - "type": "string" - }, - "transaction_id": { - "type": "string" - }, - "transaction_event_code": { - "type": "string" - }, - "transaction_initiation_date": { - "type": "string" - }, - "transaction_updated_date": { - "type": "string" - }, - "transaction_amount": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "fee_amount": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "insurance_amount": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "shipping_amount": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "shipping_discount_amount": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "transaction_status": { - "type": "string" - }, - "transaction_subject": { - "type": "string" - }, - "transaction_note": { - "type": "string" - }, - "invoice_id": { - "type": "string" - }, - "custom_field": { - "type": "string" - }, - "protection_eligibility": { - "type": "string" - } - } - }, - "payer_info": { - "type": "object", - "properties": { - "account_id": { - "type": "string" - }, - "email_address": { - "type": "string" - }, - "address_status": { - "type": "string" - }, - "payer_status": { - "type": "string" - }, - "payer_name": { - "type": "object", - "properties": { - "given_name": { - "type": "string" - }, - "surname": { - "type": "string" - }, - "alternate_full_name": { - "type": "string" - } - } - }, - "country_code": { - "type": "string" - } - } - }, - "shipping_info": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "address": { - "type": "object", - "properties": { - "line1": { - "type": "string" - }, - "line2": { - "type": "string" - }, - "city": { - "type": "string" - }, - "country_code": { - "type": "string" - }, - "postal_code": { - "type": "string" - } - } - } - } - }, - "cart_info": { - "type": "object", - "properties": { - "item_details": { - "type": "array", - "items": { - "type": "object", - "properties": { - "item_code": { - "type": "string" - }, - "item_name": { - "type": "string" - }, - "item_description": { - "type": "string" - }, - "item_quantity": { - "type": "string" - }, - "item_unit_price": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "item_amount": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "tax_amounts": { - "type": "array", - "items": { - "type": "object", - "properties": { - "tax_amount": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - } - } - } - }, - "total_item_amount": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "invoice_number": { - "type": "string" - } - } - } - } - } - }, - "store_info": { - "type": "object" - }, - "auction_info": { - "type": "object" - }, - "incentive_info": { - "type": "object" - } - } - }, + "json_schema": {}, "source_defined_cursor": true, "default_cursor_field": [ "transaction_info", @@ -266,68 +17,7 @@ { "stream": { "name": "balances", - "json_schema": { - "$schema": "http://json-schema.org/schema#", - "type": "object", - "properties": { - "account_id": { - "type": "string" - }, - "as_of_time": { - "type": "string" - }, - "balances": { - "type": "array", - "items": { - "type": "object", - "properties": { - "available_balance": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "currency": { - "type": "string" - }, - "primary": { - "type": "boolean" - }, - "total_balance": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "withheld_balance": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - } - } - } - }, - "last_refresh_time": { - "type": "string" - } - } - }, + "json_schema": {}, "default_cursor_field": ["as_of_time"], "supported_sync_modes": [ "full_refresh", diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json deleted file mode 100644 index 23ec64e5a5cb..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transaction.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "transactions", - "json_schema": { - "type": "object", - "properties": { - "account_number": { - "type": ["null", "string"] - }, - "last_refreshed_datetime": { - "type": ["null", "string"] - }, - "page": { - "type": ["null", "integer"] - }, - "total_items": { - "type": ["null", "integer"] - } - } - }, - "supported_sync_modes": ["full_refresh", "incremental"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json index 41a224a1588b..bd8cb03c9fe1 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json @@ -3,256 +3,7 @@ { "stream": { "name": "transactions", - "json_schema": { - "$schema": "http://json-schema.org/schema#", - "type": "object", - "properties": { - "transaction_info": { - "type": "object", - "properties": { - "paypal_account_id": { - "type": "string" - }, - "transaction_id": { - "type": "string" - }, - "transaction_event_code": { - "type": "string" - }, - "transaction_initiation_date": { - "type": "string" - }, - "transaction_updated_date": { - "type": "string" - }, - "transaction_amount": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "fee_amount": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "insurance_amount": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "shipping_amount": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "shipping_discount_amount": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "transaction_status": { - "type": "string" - }, - "transaction_subject": { - "type": "string" - }, - "transaction_note": { - "type": "string" - }, - "invoice_id": { - "type": "string" - }, - "custom_field": { - "type": "string" - }, - "protection_eligibility": { - "type": "string" - } - } - }, - "payer_info": { - "type": "object", - "properties": { - "account_id": { - "type": "string" - }, - "email_address": { - "type": "string" - }, - "address_status": { - "type": "string" - }, - "payer_status": { - "type": "string" - }, - "payer_name": { - "type": "object", - "properties": { - "given_name": { - "type": "string" - }, - "surname": { - "type": "string" - }, - "alternate_full_name": { - "type": "string" - } - } - }, - "country_code": { - "type": "string" - } - } - }, - "shipping_info": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "address": { - "type": "object", - "properties": { - "line1": { - "type": "string" - }, - "line2": { - "type": "string" - }, - "city": { - "type": "string" - }, - "country_code": { - "type": "string" - }, - "postal_code": { - "type": "string" - } - } - } - } - }, - "cart_info": { - "type": "object", - "properties": { - "item_details": { - "type": "array", - "items": { - "type": "object", - "properties": { - "item_code": { - "type": "string" - }, - "item_name": { - "type": "string" - }, - "item_description": { - "type": "string" - }, - "item_quantity": { - "type": "string" - }, - "item_unit_price": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "item_amount": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "tax_amounts": { - "type": "array", - "items": { - "type": "object", - "properties": { - "tax_amount": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - } - } - } - }, - "total_item_amount": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "invoice_number": { - "type": "string" - } - } - } - } - } - }, - "store_info": { - "type": "object" - }, - "auction_info": { - "type": "object" - }, - "incentive_info": { - "type": "object" - } - } - }, + "json_schema": {}, "source_defined_cursor": true, "default_cursor_field": [ "transaction_info", diff --git a/airbyte-integrations/connectors/source-paypal-transaction/main.py b/airbyte-integrations/connectors/source-paypal-transaction/main.py index 881549ef9405..83cf1487ddc4 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/main.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/main.py @@ -1,7 +1,7 @@ # # MIT License # -# Copyright (c) 2020 Airbyte +# Copyright (c) 2021 Airbyte # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/airbyte-integrations/connectors/source-paypal-transaction/setup.py b/airbyte-integrations/connectors/source-paypal-transaction/setup.py index 4a62e6df834d..ee1103a861d2 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/setup.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/setup.py @@ -1,7 +1,7 @@ # # MIT License # -# Copyright (c) 2020 Airbyte +# Copyright (c) 2021 Airbyte # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/__init__.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/__init__.py index 403d505dcb87..6487dc4b3d4f 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/__init__.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/__init__.py @@ -1,7 +1,7 @@ """ MIT License -Copyright (c) 2020 Airbyte +Copyright (c) 2021 Airbyte Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 4ea0e31731f4..f9a5ff8393c9 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -1,7 +1,7 @@ # # MIT License # -# Copyright (c) 2020 Airbyte +# Copyright (c) 2021 Airbyte # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -24,7 +24,6 @@ from abc import ABC from datetime import datetime, timedelta -from pprint import pprint from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests @@ -43,19 +42,6 @@ def get_endpoint(is_sandbox: bool = False) -> str: return endpoint -def get_end_date(config: Mapping[str, Any]) -> datetime: - end_date = None - now = datetime.now().replace(microsecond=0).astimezone() - if "end_date" in config and config["end_date"]: - end_date = isoparse(config["end_date"]) - - # If date is in future then set it to now: - if not end_date or end_date > now: - end_date = now - - return end_date - - class PaypalTransactionStream(HttpStream, ABC): page_size = "500" # API limit @@ -266,8 +252,7 @@ def stream_slices( # Do not run any requests if start_date is less than 36 hrs before current time, otherwise API throws an error: # 'message': 'Data for the given start date is not available.' - if start_date > start_date_slice or \ - start_date > end_date: + if start_date > start_date_slice or start_date > end_date: return [] dates = [] @@ -278,7 +263,7 @@ def stream_slices( # add last (the oldest) slice period dates.append({"start_date": start_date.isoformat(), "end_date": end_date_slice.isoformat()}) - pprint(dates) + return dates[::-1] # inverse stream slices to start read requests from the oldest slices @@ -333,7 +318,7 @@ def stream_slices( # Add last (the newest) slice with the current time of the sync dates.append({"start_date": self.end_date.isoformat()}) - pprint(dates) + return dates diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py index b8a8150b507f..ac5ce605dc2e 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py @@ -1,7 +1,7 @@ # # MIT License # -# Copyright (c) 2020 Airbyte +# Copyright (c) 2021 Airbyte # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal From 3e6565c33205e207d4d1bff88aa3eed3450bdd94 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Mon, 21 Jun 2021 17:18:50 +0300 Subject: [PATCH 34/60] added docs --- .../source_paypal_transaction/source.py | 1 + .../sources/paypal-transaction.md | 62 +++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 docs/integrations/sources/paypal-transaction.md diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index f9a5ff8393c9..e5fe72a981b9 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -198,6 +198,7 @@ class Transactions(PaypalTransactionStream): cursor_field = ["transaction_info", "transaction_initiation_date"] start_date_max = {"hours": 36} # this limit is found experimentally + records_per_request = 10000 # API limit def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None diff --git a/docs/integrations/sources/paypal-transaction.md b/docs/integrations/sources/paypal-transaction.md new file mode 100644 index 000000000000..443c4dd8e905 --- /dev/null +++ b/docs/integrations/sources/paypal-transaction.md @@ -0,0 +1,62 @@ +# Paypal Transaction API + +## Overview + +The [Paypal Transaction API](https://developer.paypal.com/docs/api/transaction-search/v1/). is used to get the history of transactions for a PayPal account. + + +#### Output schema + +This Source is capable of syncing the following core Streams: + +* [Transactions](https://developer.paypal.com/docs/api/transaction-search/v1/#transactions) +* [Balances](https://developer.paypal.com/docs/api/transaction-search/v1/#balances) + +#### Data type mapping + +| Integration Type | Airbyte Type | Notes | +| :--- | :--- | :--- | +| `string` | `string` | | +| `number` | `number` | | +| `array` | `array` | | +| `object` | `object` | | + +#### Features + +| Feature | Supported? | +| :--- | :--- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | Yes | +| Namespaces | No | + + +### Getting started + +### Requirements + +* client_id. +* secret. +* is_sandbox. + +### Setup guide + +In order to get an `Client ID` and `Secret` please go to [this](https://developer.paypal.com/docs/platforms/get-started/ page and follow the instructions. After registration you may find your `Client ID` and `Secret` [here](https://developer.paypal.com/developer/accounts/). + + +## Performance considerations + +Paypal transaction API has some [limits](https://developer.paypal.com/docs/integration/direct/transaction-search/) +- `start_date_min` = 3 years, API call lists transaction for the previous three years. +- `start_date_max` = 1.5 days, it takes a maximum of three hours for executed transactions to appear in the list transactions call. It is set to 1.5 days by default based on experience, otherwise API throw an error. +- `stream_slice_period` = 1 day, the maximum supported date range is 31 days. +- `records_per_request` = 10000, the maximum number of records in a single request. +- `page_size` = 500, the maximum page size is 500. + +Transactions sync is performed with default `stream_slice_period` = 1 day, it means that there will be 1 request for each day between start_date and now (or end_date). if `start_date` is greater then `start_date_max`. +Balances sync is similarly performed with default `stream_slice_period` = 1 day, but it will do additional request for the end_date of the sync (now). + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :-------- | :----- | :------ | +| 0.1.0 | 2021-06-10 | [2942](https://github.com/airbytehq/airbyte/issues/1640) | PayPal Transaction Search API | From 845170c6f71c78a3f49025fe59f4790c1cf2f5b6 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 22 Jun 2021 00:00:07 +0300 Subject: [PATCH 35/60] fix for test failure - The sync should produce at least one STATE message --- .../integration_tests/abnormal_state.json | 4 ++-- .../source_paypal_transaction/source.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json index 1761a6998ad9..0a05b9cd2a0e 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json @@ -1,8 +1,8 @@ { "transactions": { - "date": "2021-06-18T11:59:59+00:00" + "date": "2021-06-08T12:00:00+00:00" }, "balances": { - "date": "2021-06-18T11:59:59+00:00" + "date": "2021-06-08T12:00:00+00:00" } } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index e5fe72a981b9..654248861a66 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -1,7 +1,7 @@ # # MIT License # -# Copyright (c) 2021 Airbyte +# Copyright (c) 2020 Airbyte # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -69,7 +69,6 @@ def __init__( end_date = isoparse(end_date_str) if end_date_str else None is_sandbox = config["is_sandbox"] - self.is_sandbox = is_sandbox self.start_date = start_date self._end_date = end_date self.is_sandbox = is_sandbox @@ -253,8 +252,15 @@ def stream_slices( # Do not run any requests if start_date is less than 36 hrs before current time, otherwise API throws an error: # 'message': 'Data for the given start date is not available.' - if start_date > start_date_slice or start_date > end_date: + if start_date > start_date_slice: + # Options 1 - sync just should be stopped since start_date is invalid and will cause API error, + # but incremental test fails with message: "The sync should produce at least one STATE message" return [] + # Option 2 - just re-sync the latest possible period, it fixes test failure mentioned above but + # it leaves new state message with unexpected date + # it duplicates the most recent records + # later it could cause a lack of data between the last scan and previous unexpected date + # start_date = start_date_slice dates = [] while start_date < start_date_slice: From 97b803e5a22b546cdac69304db483188de4cd6ad Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 22 Jun 2021 00:02:39 +0300 Subject: [PATCH 36/60] removed optional parameter 'end_date' --- .../source_paypal_transaction/spec.json | 7 ------- 1 file changed, 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json index e1a9bb5cef08..2bc60e63a67d 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json @@ -25,13 +25,6 @@ "examples": ["2021-06-11T23:59:59-00:00"], "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}[+-][0-9]{2}:[0-9]{2}$" }, - "end_date": { - "type": "string", - "title": "End Date", - "description": "End Date for data extraction in ISO format. if not specified, then current date/time is used by default", - "examples": ["2021-06-11T23:59:59-00:00"], - "pattern": "^$|[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}[+-][0-9]{2}:[0-9]{2}$" - }, "is_sandbox": { "title": "Is Sandbox", "description": "Whether or not to Sandbox or Production environment to extract data from", From 1435531f18306dd7a7ec741c57e216d9e75ce562 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 22 Jun 2021 00:07:59 +0300 Subject: [PATCH 37/60] removed detailed info about balances schema --- .../configured_catalog_balances.json | 63 +------------------ 1 file changed, 1 insertion(+), 62 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_balances.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_balances.json index ccc9a20b32ca..0055cd909502 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_balances.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_balances.json @@ -3,68 +3,7 @@ { "stream": { "name": "balances", - "json_schema": { - "$schema": "http://json-schema.org/schema#", - "type": "object", - "properties": { - "account_id": { - "type": "string" - }, - "as_of_time": { - "type": "string" - }, - "balances": { - "type": "array", - "items": { - "type": "object", - "properties": { - "available_balance": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "currency": { - "type": "string" - }, - "primary": { - "type": "boolean" - }, - "total_balance": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "withheld_balance": { - "type": "object", - "properties": { - "currency_code": { - "type": "string" - }, - "value": { - "type": "string" - } - } - } - } - } - }, - "last_refresh_time": { - "type": "string" - } - } - }, + "json_schema": {}, "supported_sync_modes": [ "full_refresh", "incremental" From 6b4761ce3d58b02390b39d119e4122fcf134155f Mon Sep 17 00:00:00 2001 From: midavadim Date: Tue, 22 Jun 2021 00:44:20 +0300 Subject: [PATCH 38/60] Delete employees.json --- .../schemas/employees.json | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/employees.json diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/employees.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/employees.json deleted file mode 100644 index 2fa01a0fa1ff..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/employees.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "years_of_service": { - "type": ["null", "integer"] - }, - "start_date": { - "type": ["null", "string"], - "format": "date-time" - } - } -} From f62ef3956fa7637a21e809d53c4bfdbbfbfe4e82 Mon Sep 17 00:00:00 2001 From: midavadim Date: Tue, 22 Jun 2021 00:46:42 +0300 Subject: [PATCH 39/60] Delete customers.json --- .../schemas/customers.json | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/customers.json diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/customers.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/customers.json deleted file mode 100644 index 9a4b13485836..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/customers.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "signup_date": { - "type": ["null", "string"], - "format": "date-time" - } - } -} From 3e94f2f7e3055fdb07a45854955e0bec0b279d02 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 22 Jun 2021 20:23:20 +0300 Subject: [PATCH 40/60] Added requests_per_minute rate limit --- .../integration_tests/acceptance.py | 2 +- .../connectors/source-paypal-transaction/main.py | 2 +- .../connectors/source-paypal-transaction/setup.py | 2 +- .../source_paypal_transaction/source.py | 6 ++++++ .../source-paypal-transaction/unit_tests/unit_test.py | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py index 2827823a2402..eeb4a2d3e02e 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py @@ -1,7 +1,7 @@ # # MIT License # -# Copyright (c) 2021 Airbyte +# Copyright (c) 2020 Airbyte # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/airbyte-integrations/connectors/source-paypal-transaction/main.py b/airbyte-integrations/connectors/source-paypal-transaction/main.py index 83cf1487ddc4..881549ef9405 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/main.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/main.py @@ -1,7 +1,7 @@ # # MIT License # -# Copyright (c) 2021 Airbyte +# Copyright (c) 2020 Airbyte # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/airbyte-integrations/connectors/source-paypal-transaction/setup.py b/airbyte-integrations/connectors/source-paypal-transaction/setup.py index ee1103a861d2..4a62e6df834d 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/setup.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/setup.py @@ -1,7 +1,7 @@ # # MIT License # -# Copyright (c) 2021 Airbyte +# Copyright (c) 2020 Airbyte # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 654248861a66..ad07a636716b 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -22,6 +22,7 @@ # SOFTWARE. # +import time from abc import ABC from datetime import datetime, timedelta from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple @@ -53,6 +54,8 @@ class PaypalTransactionStream(HttpStream, ABC): stream_slice_period: Mapping[str, int] = {"days": 1} # max period is 31 days (API limit) + requests_per_minute: int = 30 # API limit is 50 reqs/min from 1 IP to all endpoints + def __init__( self, start_date: Optional[datetime] = None, @@ -135,6 +138,9 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp self.update_field(record, self.cursor_field, lambda date: isoparse(date).isoformat()) yield record + # sleep for 1-2 secs to not reach rate limit: 50 requests per minute + time.sleep(60 / self.requests_per_minute) + @staticmethod def update_field(record: Mapping[str, Any], field_path: List[str], update: Callable[[Any], None]): diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py index ac5ce605dc2e..b8a8150b507f 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py @@ -1,7 +1,7 @@ # # MIT License # -# Copyright (c) 2021 Airbyte +# Copyright (c) 2020 Airbyte # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal From 9a98bb3437d8b6db0950fb00fbec30c3255d0739 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 24 Jun 2021 03:51:46 +0300 Subject: [PATCH 41/60] added unit tests, added custom backoff --- .../source_paypal_transaction/source.py | 47 +++--- .../unit_tests/unit_test.py | 141 +++++++++++++++++- 2 files changed, 163 insertions(+), 25 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index ad07a636716b..c549befa9ae4 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -22,10 +22,10 @@ # SOFTWARE. # -import time from abc import ABC from datetime import datetime, timedelta -from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple +import time +from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union import requests from airbyte_cdk.sources import AbstractSource @@ -54,7 +54,7 @@ class PaypalTransactionStream(HttpStream, ABC): stream_slice_period: Mapping[str, int] = {"days": 1} # max period is 31 days (API limit) - requests_per_minute: int = 30 # API limit is 50 reqs/min from 1 IP to all endpoints + requests_per_minute: int = 30 # API limit is 50 reqs/min from 1 IP to all endpoints, otherwise IP is banned for 5 mins def __init__( self, @@ -68,12 +68,11 @@ def __init__( # Initiate data from config if config: start_date = isoparse(config["start_date"]) - end_date_str = config.get("end_date") - end_date = isoparse(end_date_str) if end_date_str else None + end_date = isoparse(config.get("end_date")) if config.get("end_date") else None is_sandbox = config["is_sandbox"] self.start_date = start_date - self._end_date = end_date + self.end_date = end_date self.is_sandbox = is_sandbox self._validate_input_dates() @@ -90,6 +89,10 @@ def end_date(self): else: return self._end_date + @end_date.setter + def end_date(self, value: datetime): + self._end_date = value + def _validate_input_dates(self): # Validate input dates @@ -124,6 +127,10 @@ def request_headers( return {"Content-Type": "application/json"} + def backoff_time(self, response: requests.Response) -> Optional[float]: + # API limit is 50 reqs/min from 1 IP to all endpoints, otherwise IP is banned for 5 mins + return 5 * 60.1 + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: json_response = response.json() @@ -142,35 +149,28 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp time.sleep(60 / self.requests_per_minute) @staticmethod - def update_field(record: Mapping[str, Any], field_path: List[str], update: Callable[[Any], None]): + def update_field(record: Mapping[str, Any], field_path: Union[List[str], str], update: Callable[[Any], None]): - data = record if not isinstance(field_path, List): field_path = [field_path] - for attr in field_path[:-1]: - if data and isinstance(data, dict): - data = data.get(attr) - else: - break - last_field = field_path[-1] - data[last_field] = update(data[last_field]) - - return data + data = PaypalTransactionStream.get_field(record, field_path[:-1]) + if data and last_field in data: + data[last_field] = update(data[last_field]) @staticmethod - def get_field(record: Mapping[str, Any], field_path: List[str]): + def get_field(record: Mapping[str, Any], field_path: Union[List[str], str]): - data = record if not isinstance(field_path, List): field_path = [field_path] + data = record for attr in field_path: if data and isinstance(data, dict): data = data.get(attr) else: - break + return None return data @@ -315,8 +315,8 @@ def stream_slices( The return value is a list of dicts {'start_date': date_string}. """ if stream_state and stream_state.get("date"): - if isoparse(stream_state.get("date")) == self._end_date: - # Do not run any incremental requests if _end_date has been reached already + if isoparse(stream_state.get("date")) >= self.end_date: + # Do not run any incremental requests if end_date has been reached already return [] # For incremental sync don't extract balance at stream_state, because # it has been already extracted in previous scan. Start from next slice period instead: @@ -325,7 +325,8 @@ def stream_slices( start_date_slice = self.start_date dates = [] - while start_date_slice < self.end_date: + end_date_limit = self.end_date # - timedelta(**self.stream_slice_period) + while start_date_slice < end_date_limit: dates.append({"start_date": start_date_slice.isoformat()}) start_date_slice = start_date_slice + timedelta(**self.stream_slice_period) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py index b8a8150b507f..c445c8ac2d3e 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py @@ -22,6 +22,143 @@ # SOFTWARE. # +from datetime import datetime, timedelta -def test_example_method(): - assert True +from airbyte_cdk.sources.streams.http.auth import NoAuth +from dateutil.parser import isoparse +from source_paypal_transaction.source import Balances, PaypalTransactionStream, Transactions + + +def test_get_field(): + + record = {"a": {"b": {"c": "d"}}} + # Test expected result - field_path is a list + assert "d" == PaypalTransactionStream.get_field(record, field_path=["a", "b", "c"]) + # Test expected result - field_path is a string + assert {"b": {"c": "d"}} == PaypalTransactionStream.get_field(record, field_path="a") + + # Test failures - not existing field_path + assert None is PaypalTransactionStream.get_field(record, field_path=["a", "b", "x"]) + assert None is PaypalTransactionStream.get_field(record, field_path=["a", "x", "x"]) + assert None is PaypalTransactionStream.get_field(record, field_path=["x", "x", "x"]) + + # Test failures - incorrect record structure + record = {"a": [{"b": {"c": "d"}}]} + assert None is PaypalTransactionStream.get_field(record, field_path=["a", "b", "c"]) + + record = {"a": {"b": "c"}} + assert None is PaypalTransactionStream.get_field(record, field_path=["a", "b", "c"]) + + record = {} + assert None is PaypalTransactionStream.get_field(record, field_path=["a", "b", "c"]) + + +def test_update_field(): + # Test success 1 + record = {"a": {"b": {"c": "d"}}} + PaypalTransactionStream.update_field(record, field_path=["a", "b", "c"], update=lambda x: x.upper()) + assert record == {"a": {"b": {"c": "D"}}} + + # Test success 2 + record = {"a": {"b": {"c": "d"}}} + PaypalTransactionStream.update_field(record, field_path="a", update=lambda x: 'updated') + assert record == {"a": "updated"} + + # Test failure - incorrect field_path + record = {"a": {"b": {"c": "d"}}} + PaypalTransactionStream.update_field(record, field_path=["a", "b", "x"], update=lambda x: x.upper()) + assert record == {"a": {"b": {"c": "d"}}} + + # Test failure - incorrect field_path + record = {"a": {"b": {"c": "d"}}} + PaypalTransactionStream.update_field(record, field_path=["a", "x", "x"], update=lambda x: x.upper()) + assert record == {"a": {"b": {"c": "d"}}} + + +def now(): + return datetime.now().replace(microsecond=0).astimezone() + + +def test_transactions_stream_slices(): + + start_date_init = now() - timedelta(days=2) + t = Transactions(authenticator=NoAuth(), start_date=start_date_init) + + # if start_date > now - start_date_max then no slices + t.start_date = now() - timedelta(**t.start_date_max) + timedelta(minutes=2) + stream_slices = t.stream_slices(sync_mode="any") + assert 0 == len(stream_slices) + + # start_date <= now - start_date_max + t.start_date = now() - timedelta(**t.start_date_max) + stream_slices = t.stream_slices(sync_mode="any") + assert 1 == len(stream_slices) + + t.start_date = now() - timedelta(**t.start_date_max) - timedelta(hours=2) + stream_slices = t.stream_slices(sync_mode="any") + assert 2 == len(stream_slices) + + t.start_date = now() - timedelta(**t.start_date_max) - timedelta(days=1) + stream_slices = t.stream_slices(sync_mode="any") + assert 2 == len(stream_slices) + + t.start_date = now() - timedelta(**t.start_date_max) - timedelta(days=1, hours=2) + stream_slices = t.stream_slices(sync_mode="any") + assert 3 == len(stream_slices) + + t.start_date = now() - timedelta(**t.start_date_max) - timedelta(days=30, minutes=1) + stream_slices = t.stream_slices(sync_mode="any") + assert 32 == len(stream_slices) + + t.start_date = isoparse("2021-06-01T10:00:00+00:00") + t.end_date = isoparse("2021-06-03T12:00:00+00:00") + stream_slices = t.stream_slices(sync_mode="any") + assert [ + {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T00:00:00+00:00"}, + {"start_date": "2021-06-02T00:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}, + ] == stream_slices + + +def test_balances_stream_slices(): + + start_date_init = now() + b = Balances(authenticator=NoAuth(), start_date=start_date_init) + stream_slices = b.stream_slices(sync_mode="any") + assert 1 == len(stream_slices) + + b.start_date = isoparse("2021-06-01T10:00:00+00:00") + b.end_date = isoparse("2021-06-03T12:00:00+00:00") + stream_slices = b.stream_slices(sync_mode="any") + assert [ + {"start_date": "2021-06-01T10:00:00+00:00"}, + {"start_date": "2021-06-02T10:00:00+00:00"}, + {"start_date": "2021-06-03T10:00:00+00:00"}, + {"start_date": "2021-06-03T12:00:00+00:00"}, + ] == stream_slices + + b.start_date = now() - timedelta(minutes=1) + b.end_date = None + stream_slices = b.stream_slices(sync_mode="any") + assert 2 == len(stream_slices) + + b.start_date = now() - timedelta(hours=23) + stream_slices = b.stream_slices(sync_mode="any") + assert 2 == len(stream_slices) + + b.start_date = now() - timedelta(days=1) + stream_slices = b.stream_slices(sync_mode="any") + assert 2 == len(stream_slices) + + b.start_date = now() - timedelta(days=1, minutes=1) + stream_slices = b.stream_slices(sync_mode="any") + assert 3 == len(stream_slices) + + b.start_date = isoparse("2021-06-01T10:00:00+00:00") + b.end_date = isoparse("2021-06-03T12:00:00+00:00") + stream_slices = b.stream_slices(sync_mode="any") + assert [ + {"start_date": "2021-06-01T10:00:00+00:00"}, + {"start_date": "2021-06-02T10:00:00+00:00"}, + {"start_date": "2021-06-03T10:00:00+00:00"}, + {"start_date": "2021-06-03T12:00:00+00:00"}, + ] == stream_slices From 82fd2769d7b8631b2aae4c7495882b0a0da6ea85 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 24 Jun 2021 11:27:01 +0300 Subject: [PATCH 42/60] added test for stream slices with stream state --- .../source_paypal_transaction/source.py | 2 +- .../unit_tests/unit_test.py | 47 ++++++++++++++----- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index c549befa9ae4..125a305f888a 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -22,9 +22,9 @@ # SOFTWARE. # +import time from abc import ABC from datetime import datetime, timedelta -import time from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union import requests diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py index c445c8ac2d3e..62f96de0e6fe 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py @@ -61,7 +61,7 @@ def test_update_field(): # Test success 2 record = {"a": {"b": {"c": "d"}}} - PaypalTransactionStream.update_field(record, field_path="a", update=lambda x: 'updated') + PaypalTransactionStream.update_field(record, field_path="a", update=lambda x: "updated") assert record == {"a": "updated"} # Test failure - incorrect field_path @@ -111,13 +111,23 @@ def test_transactions_stream_slices(): assert 32 == len(stream_slices) t.start_date = isoparse("2021-06-01T10:00:00+00:00") - t.end_date = isoparse("2021-06-03T12:00:00+00:00") + t.end_date = isoparse("2021-06-04T12:00:00+00:00") stream_slices = t.stream_slices(sync_mode="any") assert [ {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T00:00:00+00:00"}, - {"start_date": "2021-06-02T00:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}, + {"start_date": "2021-06-02T00:00:00+00:00", "end_date": "2021-06-03T00:00:00+00:00"}, + {"start_date": "2021-06-03T00:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, ] == stream_slices + stream_slices = t.stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"}) + assert [ + {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T00:00:00+00:00"}, + {"start_date": "2021-06-03T00:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, + ] == stream_slices + + stream_slices = t.stream_slices(sync_mode="any", stream_state={"date": "2021-06-04T10:00:00+00:00"}) + assert [] == stream_slices + def test_balances_stream_slices(): @@ -126,15 +136,15 @@ def test_balances_stream_slices(): stream_slices = b.stream_slices(sync_mode="any") assert 1 == len(stream_slices) - b.start_date = isoparse("2021-06-01T10:00:00+00:00") - b.end_date = isoparse("2021-06-03T12:00:00+00:00") - stream_slices = b.stream_slices(sync_mode="any") - assert [ - {"start_date": "2021-06-01T10:00:00+00:00"}, - {"start_date": "2021-06-02T10:00:00+00:00"}, - {"start_date": "2021-06-03T10:00:00+00:00"}, - {"start_date": "2021-06-03T12:00:00+00:00"}, - ] == stream_slices + # b.start_date = isoparse("2021-06-01T10:00:00+00:00") + # b.end_date = isoparse("2021-06-03T12:00:00+00:00") + # stream_slices = b.stream_slices(sync_mode="any") + # assert [ + # {"start_date": "2021-06-01T10:00:00+00:00"}, + # {"start_date": "2021-06-02T10:00:00+00:00"}, + # {"start_date": "2021-06-03T10:00:00+00:00"}, + # {"start_date": "2021-06-03T12:00:00+00:00"}, + # ] == stream_slices b.start_date = now() - timedelta(minutes=1) b.end_date = None @@ -155,6 +165,7 @@ def test_balances_stream_slices(): b.start_date = isoparse("2021-06-01T10:00:00+00:00") b.end_date = isoparse("2021-06-03T12:00:00+00:00") + stream_slices = b.stream_slices(sync_mode="any") assert [ {"start_date": "2021-06-01T10:00:00+00:00"}, @@ -162,3 +173,15 @@ def test_balances_stream_slices(): {"start_date": "2021-06-03T10:00:00+00:00"}, {"start_date": "2021-06-03T12:00:00+00:00"}, ] == stream_slices + + stream_slices = b.stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"}) + assert [ + {"start_date": "2021-06-03T10:00:00+00:00"}, + {"start_date": "2021-06-03T12:00:00+00:00"}, + ] == stream_slices + + stream_slices = b.stream_slices(sync_mode="any", stream_state={"date": "2021-06-03T11:00:00+00:00"}) + assert [{"start_date": "2021-06-03T12:00:00+00:00"}] == stream_slices + + stream_slices = b.stream_slices(sync_mode="any", stream_state={"date": "2021-06-03T12:00:00+00:00"}) + assert [] == stream_slices From e7e09d0819531a027ab67bbcc8f533dbed31412f Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 24 Jun 2021 11:42:15 +0300 Subject: [PATCH 43/60] removed comments --- .../source-paypal-transaction/unit_tests/unit_test.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py index 62f96de0e6fe..45ded3f4fc57 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py @@ -136,16 +136,6 @@ def test_balances_stream_slices(): stream_slices = b.stream_slices(sync_mode="any") assert 1 == len(stream_slices) - # b.start_date = isoparse("2021-06-01T10:00:00+00:00") - # b.end_date = isoparse("2021-06-03T12:00:00+00:00") - # stream_slices = b.stream_slices(sync_mode="any") - # assert [ - # {"start_date": "2021-06-01T10:00:00+00:00"}, - # {"start_date": "2021-06-02T10:00:00+00:00"}, - # {"start_date": "2021-06-03T10:00:00+00:00"}, - # {"start_date": "2021-06-03T12:00:00+00:00"}, - # ] == stream_slices - b.start_date = now() - timedelta(minutes=1) b.end_date = None stream_slices = b.stream_slices(sync_mode="any") From 52c7285ce45ce40aca09fc9282f5cda7507ae046 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 24 Jun 2021 12:11:35 +0300 Subject: [PATCH 44/60] updated docs pages --- docs/SUMMARY.md | 1 + docs/integrations/sources/paypal-transaction.md | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 9c6634badaf2..3b01b1025b9c 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -67,6 +67,7 @@ * [MySQL](integrations/sources/mysql.md) * [Okta](integrations/sources/okta.md) * [Oracle DB](integrations/sources/oracle.md) + * [Paypal Transaction](integrations/sources/paypal-transaction.md) * [Plaid](integrations/sources/plaid.md) * [PokéAPI](integrations/sources/pokeapi.md) * [Postgres](integrations/sources/postgres.md) diff --git a/docs/integrations/sources/paypal-transaction.md b/docs/integrations/sources/paypal-transaction.md index 443c4dd8e905..be0ddda01807 100644 --- a/docs/integrations/sources/paypal-transaction.md +++ b/docs/integrations/sources/paypal-transaction.md @@ -51,6 +51,7 @@ Paypal transaction API has some [limits](https://developer.paypal.com/docs/integ - `stream_slice_period` = 1 day, the maximum supported date range is 31 days. - `records_per_request` = 10000, the maximum number of records in a single request. - `page_size` = 500, the maximum page size is 500. +- `requests_per_minute` = 30, maximum limit is 50 requests per minute from IP address to all endpoint Transactions sync is performed with default `stream_slice_period` = 1 day, it means that there will be 1 request for each day between start_date and now (or end_date). if `start_date` is greater then `start_date_max`. Balances sync is similarly performed with default `stream_slice_period` = 1 day, but it will do additional request for the end_date of the sync (now). @@ -59,4 +60,4 @@ Balances sync is similarly performed with default `stream_slice_period` = 1 day, | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | -| 0.1.0 | 2021-06-10 | [2942](https://github.com/airbytehq/airbyte/issues/1640) | PayPal Transaction Search API | +| 0.1.0 | 2021-06-10 | [4240](https://github.com/airbytehq/airbyte/pull/4240) | PayPal Transaction Search API | From 7ccd91ceb515240a671e5503f6664593ec00b158 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 24 Jun 2021 15:30:33 +0300 Subject: [PATCH 45/60] fixed format for json files --- .../integration_tests/configured_catalog.json | 9 +++------ .../integration_tests/configured_catalog_balances.json | 5 +---- .../configured_catalog_transactions.json | 4 ++-- .../integration_tests/invalid_config.json | 2 +- .../integration_tests/sample_config.json | 2 +- .../source_paypal_transaction/schemas/balances.json | 2 +- .../source_paypal_transaction/schemas/transactions.json | 2 +- 7 files changed, 10 insertions(+), 16 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json index 0557887ec8c2..32bd7a7ac9eb 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json @@ -6,8 +6,8 @@ "json_schema": {}, "source_defined_cursor": true, "default_cursor_field": [ - "transaction_info", - "transaction_initiation_date" + "transaction_info", + "transaction_initiation_date" ], "supported_sync_modes": ["full_refresh", "incremental"] }, @@ -19,10 +19,7 @@ "name": "balances", "json_schema": {}, "default_cursor_field": ["as_of_time"], - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite" diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_balances.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_balances.json index 0055cd909502..ca1887a40519 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_balances.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_balances.json @@ -4,10 +4,7 @@ "stream": { "name": "balances", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite" diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json index bd8cb03c9fe1..cbbd4cd829e6 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json @@ -6,8 +6,8 @@ "json_schema": {}, "source_defined_cursor": true, "default_cursor_field": [ - "transaction_info", - "transaction_initiation_date" + "transaction_info", + "transaction_initiation_date" ], "supported_sync_modes": ["full_refresh", "incremental"] }, diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json index e3960dc221ab..0b2938447d7d 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json @@ -3,4 +3,4 @@ "secret": "ENC8__", "start_date": "2000-06-01T05:00:00+03:00", "is_sandbox": false -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json index a9035251e13f..6f5dbf626fac 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json @@ -3,4 +3,4 @@ "secret": "PAYPAL_SECRET", "start_date": "2021-06-01T00:00:00+00:00", "is_sandbox": false -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json index e75c1085da15..f3a6cc2598ae 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json @@ -56,4 +56,4 @@ "type": "datetime" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json index f5c7cf8818a3..0735db2d2b33 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json @@ -247,4 +247,4 @@ "type": "object" } } -} \ No newline at end of file +} From d409e68bd61a32efcb140d07f95d37252b2d098c Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Fri, 25 Jun 2021 22:38:55 +0300 Subject: [PATCH 46/60] fixed types in schemas and link to the schema. fixed primary key for Transactions stream --- .../integration_tests/acceptance.py | 2 - .../schemas/balances.json | 30 +-- .../schemas/transactions.json | 192 +++++++++++------- .../source_paypal_transaction/source.py | 2 +- .../source_paypal_transaction/spec.json | 2 +- 5 files changed, 139 insertions(+), 89 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py index eeb4a2d3e02e..d6cbdc97c495 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py @@ -31,6 +31,4 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """ This fixture is a placeholder for external resources that acceptance test might require.""" - # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield - # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json index f3a6cc2598ae..06fb1d13695d 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json @@ -1,53 +1,53 @@ { - "$schema": "http://json-schema.org/schema#", - "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], "properties": { "balance": { - "type": "object", + "type": ["null", "object"], "properties": { "currency": { - "type": "string" + "type": ["null", "string"] }, "primary": { - "type": "boolean" + "type": ["null", "boolean"] }, "total_balance": { - "type": "object", + "type": ["null", "object"], "properties": { "currency_code": { - "type": "string" + "type": ["null", "string"] }, "value": { - "type": "string" + "type": ["null", "string"] } } }, "available_balance": { - "type": "object", + "type": ["null", "object"], "properties": { "currency_code": { - "type": "string" + "type": ["null", "string"] }, "value": { - "type": "string" + "type": ["null", "string"] } } }, "withheld_balance": { - "type": "object", + "type": ["null", "object"], "properties": { "currency_code": { - "type": "string" + "type": ["null", "string"] }, "value": { - "type": "string" + "type": ["null", "string"] } } } } }, "account_id": { - "type": "string" + "type": ["null", "string"] }, "as_of_time": { "type": "datetime" diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json index 0735db2d2b33..2de185fb6324 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json @@ -1,217 +1,217 @@ { - "$schema": "http://json-schema.org/schema#", - "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], "properties": { "transaction_info": { - "type": "object", + "type": ["null", "object"], "properties": { "paypal_account_id": { - "type": "string" + "type": ["null", "string"] }, "transaction_id": { - "type": "string" + "type": ["null", "string"] }, "transaction_event_code": { - "type": "string" + "type": ["null", "string"] }, "transaction_initiation_date": { - "type": "datetime" + "type": ["null", "datetime"] }, "transaction_updated_date": { - "type": "datetime" + "type": ["null", "datetime"] }, "transaction_amount": { - "type": "object", + "type": ["null", "object"], "properties": { "currency_code": { - "type": "string" + "type": ["null", "string"] }, "value": { - "type": "string" + "type": ["null", "string"] } } }, "fee_amount": { - "type": "object", + "type": ["null", "object"], "properties": { "currency_code": { - "type": "string" + "type": ["null", "string"] }, "value": { - "type": "string" + "type": ["null", "string"] } } }, "insurance_amount": { - "type": "object", + "type": ["null", "object"], "properties": { "currency_code": { - "type": "string" + "type": ["null", "string"] }, "value": { - "type": "string" + "type": ["null", "string"] } } }, "shipping_amount": { - "type": "object", + "type": ["null", "object"], "properties": { "currency_code": { - "type": "string" + "type": ["null", "string"] }, "value": { - "type": "string" + "type": ["null", "string"] } } }, "shipping_discount_amount": { - "type": "object", + "type": ["null", "object"], "properties": { "currency_code": { - "type": "string" + "type": ["null", "string"] }, "value": { - "type": "string" + "type": ["null", "string"] } } }, "transaction_status": { - "type": "string" + "type": ["null", "string"] }, "transaction_subject": { - "type": "string" + "type": ["null", "string"] }, "transaction_note": { - "type": "string" + "type": ["null", "string"] }, "invoice_id": { - "type": "string" + "type": ["null", "string"] }, "custom_field": { - "type": "string" + "type": ["null", "string"] }, "protection_eligibility": { - "type": "string" + "type": ["null", "string"] } } }, "payer_info": { - "type": "object", + "type": ["null", "object"], "properties": { "account_id": { - "type": "string" + "type": ["null", "string"] }, "email_address": { - "type": "string" + "type": ["null", "string"] }, "address_status": { - "type": "string" + "type": ["null", "string"] }, "payer_status": { - "type": "string" + "type": ["null", "string"] }, "payer_name": { - "type": "object", + "type": ["null", "object"], "properties": { "given_name": { - "type": "string" + "type": ["null", "string"] }, "surname": { - "type": "string" + "type": ["null", "string"] }, "alternate_full_name": { - "type": "string" + "type": ["null", "string"] } } }, "country_code": { - "type": "string" + "type": ["null", "string"] } } }, "shipping_info": { - "type": "object", + "type": ["null", "object"], "properties": { "name": { - "type": "string" + "type": ["null", "string"] }, "address": { - "type": "object", + "type": ["null", "object"], "properties": { "line1": { - "type": "string" + "type": ["null", "string"] }, "line2": { - "type": "string" + "type": ["null", "string"] }, "city": { - "type": "string" + "type": ["null", "string"] }, "country_code": { - "type": "string" + "type": ["null", "string"] }, "postal_code": { - "type": "string" + "type": ["null", "string"] } } } } }, "cart_info": { - "type": "object", + "type": ["null", "object"], "properties": { "item_details": { "type": "array", "items": { - "type": "object", + "type": ["null", "object"], "properties": { "item_code": { - "type": "string" + "type": ["null", "string"] }, "item_name": { - "type": "string" + "type": ["null", "string"] }, "item_description": { - "type": "string" + "type": ["null", "string"] }, "item_quantity": { - "type": "string" + "type": ["null", "string"] }, "item_unit_price": { - "type": "object", + "type": ["null", "object"], "properties": { "currency_code": { - "type": "string" + "type": ["null", "string"] }, "value": { - "type": "string" + "type": ["null", "string"] } } }, "item_amount": { - "type": "object", + "type": ["null", "object"], "properties": { "currency_code": { - "type": "string" + "type": ["null", "string"] }, "value": { - "type": "string" + "type": ["null", "string"] } } }, "tax_amounts": { "type": "array", "items": { - "type": "object", + "type": ["null", "object"], "properties": { "tax_amount": { - "type": "object", + "type": ["null", "object"], "properties": { "currency_code": { - "type": "string" + "type": ["null", "string"] }, "value": { - "type": "string" + "type": ["null", "string"] } } } @@ -219,18 +219,18 @@ } }, "total_item_amount": { - "type": "object", + "type": ["null", "object"], "properties": { "currency_code": { - "type": "string" + "type": ["null", "string"] }, "value": { - "type": "string" + "type": ["null", "string"] } } }, "invoice_number": { - "type": "string" + "type": ["null", "string"] } } } @@ -238,13 +238,65 @@ } }, "store_info": { - "type": "object" + "type": "object", + "properties": { + "store_id": { + "type": ["null", "string"] + }, + "terminal_id": { + "type": ["null", "string"] + } + } }, "auction_info": { - "type": "object" + "type": "object", + "properties": { + "auction_site": { + "type": ["null", "string"] + }, + "auction_item_site": { + "type": ["null", "string"] + }, + "auction_buyer_id": { + "type": ["null", "string"] + }, + "auction_closing_date": { + "type": ["null", "string"] + } + } }, "incentive_info": { - "type": "object" + "type": "object", + "properties": { + "incentive_details": { + "type": "array", + "items": { + "type": "object", + "properties": { + "incentive_type": { + "type": ["null", "string"] + }, + "incentive_code": { + "type": ["null", "string"] + }, + "incentive_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "incentive_program_code": { + "type": ["null", "string"] + } + } + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 125a305f888a..67eb0ddb4f49 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -199,7 +199,7 @@ class Transactions(PaypalTransactionStream): """ data_field = "transaction_details" - primary_key = ["transaction_info", "transaction_id"] + primary_key = [["transaction_info", "transaction_id"]] cursor_field = ["transaction_info", "transaction_initiation_date"] start_date_max = {"hours": 36} # this limit is found experimentally diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json index 2bc60e63a67d..23a0e3e19f37 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json @@ -1,5 +1,5 @@ { - "documentationUrl": "https://developer.paypal.com/docs/api/transaction-search/v1/", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/paypal-transactions", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Paypal Transaction Search", From e740f13e63f00b916c5f969892b87dab3ee99d7a Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Sat, 26 Jun 2021 03:26:04 +0300 Subject: [PATCH 47/60] updated stream slices --- .../source_paypal_transaction/source.py | 153 ++++++++---------- .../unit_tests/unit_test.py | 22 +-- 2 files changed, 80 insertions(+), 95 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 67eb0ddb4f49..f0676989a817 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -31,7 +31,7 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator +from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator, Oauth2Authenticator from dateutil.parser import isoparse @@ -58,62 +58,76 @@ class PaypalTransactionStream(HttpStream, ABC): def __init__( self, - start_date: Optional[datetime] = None, - end_date: Optional[datetime] = None, + authenticator: HttpAuthenticator, + start_date: Union[datetime, str], + end_date: Union[datetime, str] = None, is_sandbox: bool = False, - config: Optional[Mapping[str, Any]] = None, **kwargs, ): - # Initiate data from config - if config: - start_date = isoparse(config["start_date"]) - end_date = isoparse(config.get("end_date")) if config.get("end_date") else None - is_sandbox = config["is_sandbox"] - self.start_date = start_date self.end_date = end_date self.is_sandbox = is_sandbox - self._validate_input_dates() + # self._validate_input_dates() + + super().__init__(authenticator=authenticator) + + @property + def start_date(self) -> datetime: + return self._start_date + + @start_date.setter + def start_date(self, value: Union[datetime, str]): + if isinstance(value, str): + value = isoparse(value) + self._start_date = value + + @property + def minimum_allowed_start_date(self) -> datetime: + return self.now - timedelta(**self.start_date_min) - super().__init__(**kwargs) + @property + def maximum_allowed_start_date(self) -> datetime: + return min(self.now - timedelta(**self.start_date_max), self.end_date) @property - def end_date(self): + def end_date(self) -> datetime: """Return initiated end_date or now()""" - now = datetime.now().replace(microsecond=0).astimezone() - if not self._end_date or self._end_date > now: + if not self._end_date or self._end_date > self.now: # If no end_date or end_date is in future then return now: - return now + return self.now else: return self._end_date @end_date.setter - def end_date(self, value: datetime): + def end_date(self, value: Union[datetime, str]): + if isinstance(value, str): + value = isoparse(value) self._end_date = value + @property + def now(self) -> datetime: + return datetime.now().replace(microsecond=0).astimezone() + def _validate_input_dates(self): # Validate input dates if self.start_date > self.end_date: raise Exception(f"start_date {self.start_date.isoformat()} is greater than end_date {self.end_date.isoformat()}") - current_date = datetime.now().replace(microsecond=0).astimezone() - current_date_delta = current_date - self.start_date - # Check for minimal possible start_date - if current_date_delta > timedelta(**self.start_date_min): + if self.start_date < self.minimum_allowed_start_date: raise Exception( f"Start_date {self.start_date.isoformat()} is too old. " - f"Min date limit is {self.start_date_min} before now:{current_date.isoformat()}." + f"Minimum allowed start_date is {self.minimum_allowed_start_date.isoformat()}." ) # Check for maximum possible start_date - if current_date_delta < timedelta(**self.start_date_max): + if self.start_date > self.maximum_allowed_start_date: raise Exception( f"Start_date {self.start_date.isoformat()} is too close to now. " - f"Max date limit is {self.start_date_max} before now:{current_date.isoformat()}." + f"Maximum allowed start_date is {self.maximum_allowed_start_date.isoformat()}." ) @property @@ -121,9 +135,7 @@ def url_base(self) -> str: return f"{get_endpoint(self.is_sandbox)}/v1/reporting/" - def request_headers( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> Mapping[str, Any]: + def request_headers(self, **kwargs) -> Mapping[str, Any]: return {"Content-Type": "application/json"} @@ -196,6 +208,8 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late class Transactions(PaypalTransactionStream): """ Stream for Transactions /v1/reporting/transactions + + API Docs: https://developer.paypal.com/docs/integration/direct/transaction-search/#list-transactions """ data_field = "transaction_details" @@ -205,10 +219,7 @@ class Transactions(PaypalTransactionStream): start_date_max = {"hours": 36} # this limit is found experimentally records_per_request = 10000 # API limit - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - + def path(self, **kwargs) -> str: return "transactions" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -243,56 +254,35 @@ def stream_slices( """ Returns a list of slices for each day (by default) between the start date and end date. The return value is a list of dicts {'start_date': date_string, 'end_date': date_string}. - """ - if stream_state and stream_state.get("date"): - start_date = isoparse(stream_state.get("date")) - else: - start_date = self.start_date - # choose first slice period (for the most recent time) - first_slice_period = max(timedelta(**self.start_date_max), timedelta(**self.stream_slice_period)) - end_date = self.end_date - - start_date_slice = end_date - first_slice_period - end_date_slice = end_date - - # Do not run any requests if start_date is less than 36 hrs before current time, otherwise API throws an error: - # 'message': 'Data for the given start date is not available.' - if start_date > start_date_slice: - # Options 1 - sync just should be stopped since start_date is invalid and will cause API error, - # but incremental test fails with message: "The sync should produce at least one STATE message" - return [] - # Option 2 - just re-sync the latest possible period, it fixes test failure mentioned above but - # it leaves new state message with unexpected date - # it duplicates the most recent records - # later it could cause a lack of data between the last scan and previous unexpected date - # start_date = start_date_slice + Slice does not cover period for slice_start_date > maximum_allowed_start_date + """ + period = timedelta(**self.stream_slice_period) - dates = [] - while start_date < start_date_slice: - dates.append({"start_date": start_date_slice.isoformat(), "end_date": end_date_slice.isoformat()}) - end_date_slice = start_date_slice - start_date_slice = start_date_slice - timedelta(**self.stream_slice_period) + slice_start_date = max(self.start_date, isoparse(stream_state.get("date")) if stream_state else self.start_date) - # add last (the oldest) slice period - dates.append({"start_date": start_date.isoformat(), "end_date": end_date_slice.isoformat()}) + slices = [] + while slice_start_date <= self.maximum_allowed_start_date: + slices.append( + {"start_date": slice_start_date.isoformat(), "end_date": min(slice_start_date + period, self.end_date).isoformat()} + ) + slice_start_date += period - return dates[::-1] # inverse stream slices to start read requests from the oldest slices + return slices class Balances(PaypalTransactionStream): """ Stream for Balances /v1/reporting/balances + + API Docs: https://developer.paypal.com/docs/integration/direct/transaction-search/#check-balances """ primary_key = "as_of_time" cursor_field = "as_of_time" data_field = None - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - + def path(self, **kwargs) -> str: return "balances" def request_params( @@ -314,26 +304,19 @@ def stream_slices( Returns a list of slices for each day (by default) between the start_date and end_date (containing the last) The return value is a list of dicts {'start_date': date_string}. """ - if stream_state and stream_state.get("date"): - if isoparse(stream_state.get("date")) >= self.end_date: - # Do not run any incremental requests if end_date has been reached already - return [] - # For incremental sync don't extract balance at stream_state, because - # it has been already extracted in previous scan. Start from next slice period instead: - start_date_slice = isoparse(stream_state.get("date")) + timedelta(**self.stream_slice_period) - else: - start_date_slice = self.start_date + period = timedelta(**self.stream_slice_period) + + slice_start_date = max(self.start_date, isoparse(stream_state.get("date")) + period if stream_state else self.start_date) - dates = [] - end_date_limit = self.end_date # - timedelta(**self.stream_slice_period) - while start_date_slice < end_date_limit: - dates.append({"start_date": start_date_slice.isoformat()}) - start_date_slice = start_date_slice + timedelta(**self.stream_slice_period) + slices = [] + while slice_start_date < self.end_date: + slices.append({"start_date": slice_start_date.isoformat()}) + slice_start_date += period # Add last (the newest) slice with the current time of the sync - dates.append({"start_date": self.end_date.isoformat()}) + slices.append({"start_date": self.end_date.isoformat()}) - return dates + return slices class PayPalOauth2Authenticator(Oauth2Authenticator): @@ -389,7 +372,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: # Try to initiate a stream and validate input date params try: - Transactions(authenticator=authenticator, config=config) + Transactions(authenticator=authenticator, **config) except Exception as e: return False, e @@ -402,6 +385,6 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = PayPalOauth2Authenticator(config) return [ - Transactions(authenticator=authenticator, config=config), - Balances(authenticator=authenticator, config=config), + Transactions(authenticator=authenticator, **config), + Balances(authenticator=authenticator, **config), ] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py index 45ded3f4fc57..812b089c4a28 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py @@ -96,7 +96,7 @@ def test_transactions_stream_slices(): t.start_date = now() - timedelta(**t.start_date_max) - timedelta(hours=2) stream_slices = t.stream_slices(sync_mode="any") - assert 2 == len(stream_slices) + assert 1 == len(stream_slices) t.start_date = now() - timedelta(**t.start_date_max) - timedelta(days=1) stream_slices = t.stream_slices(sync_mode="any") @@ -104,29 +104,31 @@ def test_transactions_stream_slices(): t.start_date = now() - timedelta(**t.start_date_max) - timedelta(days=1, hours=2) stream_slices = t.stream_slices(sync_mode="any") - assert 3 == len(stream_slices) + assert 2 == len(stream_slices) t.start_date = now() - timedelta(**t.start_date_max) - timedelta(days=30, minutes=1) stream_slices = t.stream_slices(sync_mode="any") - assert 32 == len(stream_slices) + assert 31 == len(stream_slices) t.start_date = isoparse("2021-06-01T10:00:00+00:00") t.end_date = isoparse("2021-06-04T12:00:00+00:00") stream_slices = t.stream_slices(sync_mode="any") assert [ - {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T00:00:00+00:00"}, - {"start_date": "2021-06-02T00:00:00+00:00", "end_date": "2021-06-03T00:00:00+00:00"}, - {"start_date": "2021-06-03T00:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, + {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T10:00:00+00:00"}, + {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, + {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-04T10:00:00+00:00"}, + {"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, ] == stream_slices stream_slices = t.stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"}) assert [ - {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T00:00:00+00:00"}, - {"start_date": "2021-06-03T00:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, + {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, + {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-04T10:00:00+00:00"}, + {"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, ] == stream_slices stream_slices = t.stream_slices(sync_mode="any", stream_state={"date": "2021-06-04T10:00:00+00:00"}) - assert [] == stream_slices + assert [{"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}] == stream_slices def test_balances_stream_slices(): @@ -174,4 +176,4 @@ def test_balances_stream_slices(): assert [{"start_date": "2021-06-03T12:00:00+00:00"}] == stream_slices stream_slices = b.stream_slices(sync_mode="any", stream_state={"date": "2021-06-03T12:00:00+00:00"}) - assert [] == stream_slices + assert [{"start_date": "2021-06-03T12:00:00+00:00"}] == stream_slices From ab97188c6fb2cb9eea8c3a5c92309df785211ada Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 29 Jun 2021 19:49:03 +0300 Subject: [PATCH 48/60] Updated tests, unified stream_slices for both streams, all instance variables instantiated directly in __init__ method --- .../acceptance-test-config.yml | 3 + .../source_paypal_transaction/source.py | 168 ++++++------------ .../unit_tests/unit_test.py | 156 ++++++++++------ 3 files changed, 164 insertions(+), 163 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml index 1e86e6269464..1c94f9d410d8 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml @@ -12,6 +12,7 @@ tests: discovery: - config_path: "secrets/config.json" basic_read: + # Sometimes test could fail (on weekends) because transactions could temporary disappear from Paypal Sandbox account - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" validate_output_from_all_streams: yes @@ -19,6 +20,8 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" incremental: + # Only "Transactions" stream is tested here because "Balances" stream always return + # at least one message (and causes test failure) - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog_transactions.json" future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index f0676989a817..f56b7fa5eace 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -64,79 +64,49 @@ def __init__( is_sandbox: bool = False, **kwargs, ): + now = datetime.now().replace(microsecond=0).astimezone() - self.start_date = start_date - self.end_date = end_date - self.is_sandbox = is_sandbox - - # self._validate_input_dates() - - super().__init__(authenticator=authenticator) - - @property - def start_date(self) -> datetime: - return self._start_date - - @start_date.setter - def start_date(self, value: Union[datetime, str]): - if isinstance(value, str): - value = isoparse(value) - self._start_date = value - - @property - def minimum_allowed_start_date(self) -> datetime: - return self.now - timedelta(**self.start_date_min) + if end_date and isinstance(end_date, str): + end_date = isoparse(end_date) + self.end_date: datetime = end_date if end_date and end_date < now else now - @property - def maximum_allowed_start_date(self) -> datetime: - return min(self.now - timedelta(**self.start_date_max), self.end_date) + if start_date and isinstance(start_date, str): + start_date = isoparse(start_date) - @property - def end_date(self) -> datetime: - """Return initiated end_date or now()""" - if not self._end_date or self._end_date > self.now: - # If no end_date or end_date is in future then return now: - return self.now - else: - return self._end_date + minimum_allowed_start_date = now - timedelta(**self.start_date_min) + if start_date < minimum_allowed_start_date: + self.logger.log( + "WARN", + f'Stream {self.name}: start_date "{start_date.isoformat()}" is too old. ' + + f'Reset start_date to the minimum_allowed_start_date "{minimum_allowed_start_date.isoformat()}"', + ) + start_date = minimum_allowed_start_date + + self.maximum_allowed_start_date = min(now - timedelta(**self.start_date_max), self.end_date) + if start_date > self.maximum_allowed_start_date: + self.logger.log( + "WARN", + f'Stream {self.name}: start_date "{start_date.isoformat()}" is too recent. ' + + f'Reset start_date to the maximum_allowed_start_date "{self.maximum_allowed_start_date.isoformat()}"', + ) + start_date = self.maximum_allowed_start_date - @end_date.setter - def end_date(self, value: Union[datetime, str]): - if isinstance(value, str): - value = isoparse(value) - self._end_date = value + self.start_date = start_date - @property - def now(self) -> datetime: - return datetime.now().replace(microsecond=0).astimezone() + self.is_sandbox = is_sandbox - def _validate_input_dates(self): + super().__init__(authenticator=authenticator) + def validate_input_dates(self): # Validate input dates if self.start_date > self.end_date: raise Exception(f"start_date {self.start_date.isoformat()} is greater than end_date {self.end_date.isoformat()}") - # Check for minimal possible start_date - if self.start_date < self.minimum_allowed_start_date: - raise Exception( - f"Start_date {self.start_date.isoformat()} is too old. " - f"Minimum allowed start_date is {self.minimum_allowed_start_date.isoformat()}." - ) - - # Check for maximum possible start_date - if self.start_date > self.maximum_allowed_start_date: - raise Exception( - f"Start_date {self.start_date.isoformat()} is too close to now. " - f"Maximum allowed start_date is {self.maximum_allowed_start_date.isoformat()}." - ) - @property def url_base(self) -> str: - return f"{get_endpoint(self.is_sandbox)}/v1/reporting/" def request_headers(self, **kwargs) -> Mapping[str, Any]: - return {"Content-Type": "application/json"} def backoff_time(self, response: requests.Response) -> Optional[float]: @@ -144,7 +114,6 @@ def backoff_time(self, response: requests.Response) -> Optional[float]: return 5 * 60.1 def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - json_response = response.json() if self.data_field is not None: data = json_response.get(self.data_field, []) @@ -162,7 +131,6 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp @staticmethod def update_field(record: Mapping[str, Any], field_path: Union[List[str], str], update: Callable[[Any], None]): - if not isinstance(field_path, List): field_path = [field_path] @@ -187,7 +155,6 @@ def get_field(record: Mapping[str, Any], field_path: Union[List[str], str]): return data def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: - # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. latest_record_date_str: str = self.get_field(latest_record, self.cursor_field) @@ -204,11 +171,35 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late else: return {"date": self.start_date.isoformat()} + def stream_slices( + self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, any]]]: + """ + Returns a list of slices for each day (by default) between the start date and end date. + The return value is a list of dicts {'start_date': date_string, 'end_date': date_string}. + + Slice does not cover period for slice_start_date > maximum_allowed_start_date + """ + period = timedelta(**self.stream_slice_period) + + slice_start_date = max(self.start_date, isoparse(stream_state.get("date")) if stream_state else self.start_date) + + slices = [] + while slice_start_date <= self.maximum_allowed_start_date: + slices.append( + {"start_date": slice_start_date.isoformat(), "end_date": min(slice_start_date + period, self.end_date).isoformat()} + ) + slice_start_date += period + + return slices + class Transactions(PaypalTransactionStream): """ Stream for Transactions /v1/reporting/transactions + API returns a list of transaction on a specific date range + API Docs: https://developer.paypal.com/docs/integration/direct/transaction-search/#list-transactions """ @@ -217,13 +208,15 @@ class Transactions(PaypalTransactionStream): cursor_field = ["transaction_info", "transaction_initiation_date"] start_date_max = {"hours": 36} # this limit is found experimentally - records_per_request = 10000 # API limit + + # TODO handle API error when 1 request returns more than 10000 records. + # https://github.com/airbytehq/airbyte/issues/4404 + records_per_request = 10000 def path(self, **kwargs) -> str: return "transactions" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - decoded_response = response.json() total_pages = decoded_response.get("total_pages") page_number = decoded_response.get("page") @@ -235,7 +228,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - page_number = 1 if next_page_token: page_number = next_page_token.get("page") @@ -248,33 +240,13 @@ def request_params( "page": page_number, } - def stream_slices( - self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, any]]]: - """ - Returns a list of slices for each day (by default) between the start date and end date. - The return value is a list of dicts {'start_date': date_string, 'end_date': date_string}. - - Slice does not cover period for slice_start_date > maximum_allowed_start_date - """ - period = timedelta(**self.stream_slice_period) - - slice_start_date = max(self.start_date, isoparse(stream_state.get("date")) if stream_state else self.start_date) - - slices = [] - while slice_start_date <= self.maximum_allowed_start_date: - slices.append( - {"start_date": slice_start_date.isoformat(), "end_date": min(slice_start_date + period, self.end_date).isoformat()} - ) - slice_start_date += period - - return slices - class Balances(PaypalTransactionStream): """ Stream for Balances /v1/reporting/balances + API returns account balance on a specific date + API Docs: https://developer.paypal.com/docs/integration/direct/transaction-search/#check-balances """ @@ -288,36 +260,13 @@ def path(self, **kwargs) -> str: def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - return { "as_of_time": stream_slice["start_date"], } def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - def stream_slices( - self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, any]]]: - """ - Returns a list of slices for each day (by default) between the start_date and end_date (containing the last) - The return value is a list of dicts {'start_date': date_string}. - """ - period = timedelta(**self.stream_slice_period) - - slice_start_date = max(self.start_date, isoparse(stream_state.get("date")) + period if stream_state else self.start_date) - - slices = [] - while slice_start_date < self.end_date: - slices.append({"start_date": slice_start_date.isoformat()}) - slice_start_date += period - - # Add last (the newest) slice with the current time of the sync - slices.append({"start_date": self.end_date.isoformat()}) - - return slices - class PayPalOauth2Authenticator(Oauth2Authenticator): """Request example for API token extraction: @@ -329,7 +278,6 @@ class PayPalOauth2Authenticator(Oauth2Authenticator): """ def __init__(self, config): - super().__init__( token_refresh_endpoint=f"{get_endpoint(config['is_sandbox'])}/v1/oauth2/token", client_id=config["client_id"], @@ -372,7 +320,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: # Try to initiate a stream and validate input date params try: - Transactions(authenticator=authenticator, **config) + Transactions(authenticator=authenticator, **config).validate_input_dates() except Exception as e: return False, e diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py index 812b089c4a28..1ace49180c8a 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py @@ -81,38 +81,58 @@ def now(): def test_transactions_stream_slices(): - start_date_init = now() - timedelta(days=2) - t = Transactions(authenticator=NoAuth(), start_date=start_date_init) + start_date_max = Transactions.start_date_max - # if start_date > now - start_date_max then no slices - t.start_date = now() - timedelta(**t.start_date_max) + timedelta(minutes=2) - stream_slices = t.stream_slices(sync_mode="any") - assert 0 == len(stream_slices) + # if start_date > now - **start_date_max then no slices + stream_slices = Transactions( + authenticator=NoAuth(), + start_date=now() - timedelta(**start_date_max) - timedelta(minutes=2), + ).stream_slices(sync_mode="any") + assert 1 == len(stream_slices) + + # start_date <= now - **start_date_max + stream_slices = Transactions( + authenticator=NoAuth(), + start_date=now() - timedelta(**start_date_max), + ).stream_slices(sync_mode="any") + assert 1 == len(stream_slices) - # start_date <= now - start_date_max - t.start_date = now() - timedelta(**t.start_date_max) - stream_slices = t.stream_slices(sync_mode="any") + stream_slices = Transactions( + authenticator=NoAuth(), + start_date=now() - timedelta(**start_date_max) + timedelta(minutes=2), + ).stream_slices(sync_mode="any") assert 1 == len(stream_slices) - t.start_date = now() - timedelta(**t.start_date_max) - timedelta(hours=2) - stream_slices = t.stream_slices(sync_mode="any") + stream_slices = Transactions( + authenticator=NoAuth(), + start_date=now() - timedelta(**start_date_max) - timedelta(hours=2), + ).stream_slices(sync_mode="any") assert 1 == len(stream_slices) - t.start_date = now() - timedelta(**t.start_date_max) - timedelta(days=1) - stream_slices = t.stream_slices(sync_mode="any") + stream_slices = Transactions( + authenticator=NoAuth(), + start_date=now() - timedelta(**start_date_max) - timedelta(days=1), + ).stream_slices(sync_mode="any") assert 2 == len(stream_slices) - t.start_date = now() - timedelta(**t.start_date_max) - timedelta(days=1, hours=2) - stream_slices = t.stream_slices(sync_mode="any") + stream_slices = Transactions( + authenticator=NoAuth(), + start_date=now() - timedelta(**start_date_max) - timedelta(days=1, hours=2), + ).stream_slices(sync_mode="any") assert 2 == len(stream_slices) - t.start_date = now() - timedelta(**t.start_date_max) - timedelta(days=30, minutes=1) - stream_slices = t.stream_slices(sync_mode="any") + stream_slices = Transactions( + authenticator=NoAuth(), + start_date=now() - timedelta(**start_date_max) - timedelta(days=30, minutes=1), + ).stream_slices(sync_mode="any") assert 31 == len(stream_slices) - t.start_date = isoparse("2021-06-01T10:00:00+00:00") - t.end_date = isoparse("2021-06-04T12:00:00+00:00") - stream_slices = t.stream_slices(sync_mode="any") + # tests with specified end_date + stream_slices = Transactions( + authenticator=NoAuth(), + start_date=isoparse("2021-06-01T10:00:00+00:00"), + end_date=isoparse("2021-06-04T12:00:00+00:00"), + ).stream_slices(sync_mode="any") assert [ {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T10:00:00+00:00"}, {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, @@ -120,60 +140,90 @@ def test_transactions_stream_slices(): {"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, ] == stream_slices - stream_slices = t.stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"}) + # tests with specified end_date and stream_state + stream_slices = Transactions( + authenticator=NoAuth(), + start_date=isoparse("2021-06-01T10:00:00+00:00"), + end_date=isoparse("2021-06-04T12:00:00+00:00"), + ).stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"}) assert [ {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-04T10:00:00+00:00"}, {"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, ] == stream_slices - stream_slices = t.stream_slices(sync_mode="any", stream_state={"date": "2021-06-04T10:00:00+00:00"}) + stream_slices = Transactions( + authenticator=NoAuth(), + start_date=isoparse("2021-06-01T10:00:00+00:00"), + end_date=isoparse("2021-06-04T12:00:00+00:00"), + ).stream_slices(sync_mode="any", stream_state={"date": "2021-06-04T10:00:00+00:00"}) assert [{"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}] == stream_slices def test_balances_stream_slices(): + """Test slices for Balance stream. + Note that is not used by this stream. + """ + now = datetime.now().replace(microsecond=0).astimezone() - start_date_init = now() - b = Balances(authenticator=NoAuth(), start_date=start_date_init) - stream_slices = b.stream_slices(sync_mode="any") + # Test without end_date (it equal by default) + stream_slices = Balances(authenticator=NoAuth(), start_date=now).stream_slices(sync_mode="any") assert 1 == len(stream_slices) - b.start_date = now() - timedelta(minutes=1) - b.end_date = None - stream_slices = b.stream_slices(sync_mode="any") - assert 2 == len(stream_slices) + stream_slices = Balances(authenticator=NoAuth(), start_date=now - timedelta(minutes=1)).stream_slices(sync_mode="any") + assert 1 == len(stream_slices) - b.start_date = now() - timedelta(hours=23) - stream_slices = b.stream_slices(sync_mode="any") - assert 2 == len(stream_slices) + stream_slices = Balances( + authenticator=NoAuth(), + start_date=now - timedelta(hours=23), + ).stream_slices(sync_mode="any") + assert 1 == len(stream_slices) - b.start_date = now() - timedelta(days=1) - stream_slices = b.stream_slices(sync_mode="any") + stream_slices = Balances( + authenticator=NoAuth(), + start_date=now - timedelta(days=1), + ).stream_slices(sync_mode="any") assert 2 == len(stream_slices) - b.start_date = now() - timedelta(days=1, minutes=1) - stream_slices = b.stream_slices(sync_mode="any") - assert 3 == len(stream_slices) - - b.start_date = isoparse("2021-06-01T10:00:00+00:00") - b.end_date = isoparse("2021-06-03T12:00:00+00:00") + stream_slices = Balances( + authenticator=NoAuth(), + start_date=now - timedelta(days=1, minutes=1), + ).stream_slices(sync_mode="any") + assert 2 == len(stream_slices) - stream_slices = b.stream_slices(sync_mode="any") + # test with custom end_date + stream_slices = Balances( + authenticator=NoAuth(), + start_date=isoparse("2021-06-01T10:00:00+00:00"), + end_date=isoparse("2021-06-03T12:00:00+00:00"), + ).stream_slices(sync_mode="any") assert [ - {"start_date": "2021-06-01T10:00:00+00:00"}, - {"start_date": "2021-06-02T10:00:00+00:00"}, - {"start_date": "2021-06-03T10:00:00+00:00"}, - {"start_date": "2021-06-03T12:00:00+00:00"}, + {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T10:00:00+00:00"}, + {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, + {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}, ] == stream_slices - stream_slices = b.stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"}) + # Test with stream state + stream_slices = Balances( + authenticator=NoAuth(), + start_date=isoparse("2021-06-01T10:00:00+00:00"), + end_date=isoparse("2021-06-03T12:00:00+00:00"), + ).stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"}) assert [ - {"start_date": "2021-06-03T10:00:00+00:00"}, - {"start_date": "2021-06-03T12:00:00+00:00"}, + {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, + {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}, ] == stream_slices - stream_slices = b.stream_slices(sync_mode="any", stream_state={"date": "2021-06-03T11:00:00+00:00"}) - assert [{"start_date": "2021-06-03T12:00:00+00:00"}] == stream_slices - - stream_slices = b.stream_slices(sync_mode="any", stream_state={"date": "2021-06-03T12:00:00+00:00"}) - assert [{"start_date": "2021-06-03T12:00:00+00:00"}] == stream_slices + stream_slices = Balances( + authenticator=NoAuth(), + start_date=isoparse("2021-06-01T10:00:00+00:00"), + end_date=isoparse("2021-06-03T12:00:00+00:00"), + ).stream_slices(sync_mode="any", stream_state={"date": "2021-06-03T11:00:00+00:00"}) + assert [{"start_date": "2021-06-03T11:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}] == stream_slices + + stream_slices = Balances( + authenticator=NoAuth(), + start_date=isoparse("2021-06-01T10:00:00+00:00"), + end_date=isoparse("2021-06-03T12:00:00+00:00"), + ).stream_slices(sync_mode="any", stream_state={"date": "2021-06-03T12:00:00+00:00"}) + assert [{"start_date": "2021-06-03T12:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}] == stream_slices From 20832d7b7001e1b0f3fb2dfa89efba6ba9e4f55b Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 29 Jun 2021 22:54:06 +0300 Subject: [PATCH 49/60] added CHANGELOG.md --- .../connectors/source-paypal-transaction/CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/CHANGELOG.md diff --git a/airbyte-integrations/connectors/source-paypal-transaction/CHANGELOG.md b/airbyte-integrations/connectors/source-paypal-transaction/CHANGELOG.md new file mode 100644 index 000000000000..d84557504900 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/CHANGELOG.md @@ -0,0 +1,4 @@ +# Changelog + +## 0.1.0 +Source implementation with support of Transactions and Balances streams From d98fe45f5aa3e740c08e6f078251120f9abf32bd Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Wed, 30 Jun 2021 22:41:25 +0300 Subject: [PATCH 50/60] Added build seeds --- .../d913b0f2-cc51-4e55-a44c-8ba1697b9239.json | 7 +++++++ .../init/src/main/resources/seed/source_definitions.yaml | 5 +++++ 2 files changed, 12 insertions(+) create mode 100644 airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d913b0f2-cc51-4e55-a44c-8ba1697b9239.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d913b0f2-cc51-4e55-a44c-8ba1697b9239.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d913b0f2-cc51-4e55-a44c-8ba1697b9239.json new file mode 100644 index 000000000000..185c3114c2cf --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d913b0f2-cc51-4e55-a44c-8ba1697b9239.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "d913b0f2-cc51-4e55-a44c-8ba1697b9239, + "name": "Paypal Transaction", + "dockerRepository": "airbyte/source-paypal-transation", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/paypal-transation", +} diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 2e7bf1bd21d4..c7ef3324fa54 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -345,3 +345,8 @@ dockerRepository: airbyte/source-aws-cloudtrail dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/sources/aws-cloudtrail +- sourceDefinitionId: d913b0f2-cc51-4e55-a44c-8ba1697b9239 + name: Paypal Transaction + dockerRepository: airbyte/source-paypal-transaction + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/paypal-transaction From 27af53d9a24738bc37c7324e7618e2228b32f703 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Wed, 30 Jun 2021 23:59:23 +0300 Subject: [PATCH 51/60] fixed closing double quotation mark --- .../d913b0f2-cc51-4e55-a44c-8ba1697b9239.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d913b0f2-cc51-4e55-a44c-8ba1697b9239.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d913b0f2-cc51-4e55-a44c-8ba1697b9239.json index 185c3114c2cf..c8dddf1e2feb 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d913b0f2-cc51-4e55-a44c-8ba1697b9239.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d913b0f2-cc51-4e55-a44c-8ba1697b9239.json @@ -1,7 +1,7 @@ { - "sourceDefinitionId": "d913b0f2-cc51-4e55-a44c-8ba1697b9239, + "sourceDefinitionId": "d913b0f2-cc51-4e55-a44c-8ba1697b9239", "name": "Paypal Transaction", - "dockerRepository": "airbyte/source-paypal-transation", + "dockerRepository": "airbyte/source-paypal-transaction", "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/paypal-transation", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/paypal-transaction" } From 2115ff24989eb776cbfbce2a919ad8c77a4a2951 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Mon, 5 Jul 2021 16:10:10 +0300 Subject: [PATCH 52/60] added paypal entry in builds.md --- airbyte-integrations/builds.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 6ebdd0539174..3d288ed43d67 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -77,6 +77,8 @@ Oracle DB [![source-oracle](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-oracle%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-oracle) + Paypal Transaction [![paypal-transaction](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-paypal-transaction%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-paypal-transaction) + Plaid [![source-plaid](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-plaid%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-plaid) Postgres [![source-postgres](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-postgres%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-postgres) From 22173dc870c19e0b15602fd656bd85804bbc995d Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Mon, 5 Jul 2021 12:05:58 -0700 Subject: [PATCH 53/60] add fixture helper --- .../bin/fixture_helper.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/bin/fixture_helper.py diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/fixture_helper.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/fixture_helper.py new file mode 100644 index 000000000000..5d5a057ff618 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/fixture_helper.py @@ -0,0 +1,84 @@ +# %% +import requests +from pprint import pprint +import logging + +logging.basicConfig(level=logging.DEBUG) + +# %% +specification = { + "client_id": "REPLACE_ME", + "secret": "REPLACE_ME", + "start_date": "2021-06-01T00:00:00+00:00", + "end_date": "2021-06-30T00:00:00+00:00", + "is_sandbox": True +} + +# %% READ and + +client_id = specification.get('client_id') +secret = specification.get('secret') + +# %% GET API_TOKEN + +token_refresh_endpoint = 'https://api-m.sandbox.paypal.com/v1/oauth2/token' +data = "grant_type=client_credentials" +headers = { + 'Accept': 'application/json', + 'Accept-Language': 'en_US' +} + +response = requests.request( + method="POST", + url=token_refresh_endpoint, + data=data, + headers=headers, + auth=(client_id, secret) +) +response_json = response.json() +print(response_json) +API_TOKEN = response_json["access_token"] + +### CREATE TRANSACTIONS +# for i in range(1000): +# create_response = requests.post( +# "https://api-m.sandbox.paypal.com/v2/checkout/orders", +# headers={'content-type': 'application/json', 'authorization': f'Bearer {API_TOKEN}', "prefer": "return=representation"}, +# json={ +# "intent": "CAPTURE", +# "purchase_units": [ +# { +# "amount": { +# "currency_code": "USD", +# "value": f"{float(i)}" +# } +# } +# ] +# } +# ) +# +# print(create_response.json()) + +# %% LIST TRANSACTIONS + +url = "https://api-m.sandbox.paypal.com/v1/reporting/transactions" + +params = { + 'start_date': '2021-06-20T00:00:00+00:00', + 'end_date': '2021-07-10T07:19:45Z', + 'fields': 'all', + 'page_size': '100', + 'page': '1' +} + +headers = { + 'Authorization': f'Bearer {API_TOKEN}', + 'Content-Type': 'application/json' +} +response = requests.get( + url, + headers=headers, + params=params +) + +pprint(response.json()) From 0d7a939b8fc8774a4cc135f5d8d1ca9f9c74bcbe Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 6 Jul 2021 02:23:09 +0300 Subject: [PATCH 54/60] added paypal transaction generator script --- .../bin/paypal_transaction_generator.py | 215 ++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py new file mode 100644 index 000000000000..baedb78fe289 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py @@ -0,0 +1,215 @@ +#%% +# REQUIREMENTS: +# 1. sudo apt-get install chromium-chromedriver +# 2. pip install selenium +# 3. ../secrets/creds.json with buyers email/password and account client_id/secret + +# HOW TO USE: +# python paypal_transaction_generator.py - will generate 3 transactions by default +# python paypal_transaction_generator.py 10 - will generate 10 transactions + +import sys +import requests +import json + +from pprint import pprint +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +import random +from time import sleep + +PAYMENT_DATA = { + "intent": "sale", + "payer": { + "payment_method": "paypal" + }, + "transactions": [{ + "amount": { + "total": "30.11", + "currency": "USD", + "details": { + "subtotal": "30.00", + "tax": "0.07", + "shipping": "0.03", + "handling_fee": "1.00", + "shipping_discount": "-1.00", + "insurance": "0.01" + } + }, + "description": "This is the payment transaction description.", + "custom": "EBAY_EMS_90048630020055", + "invoice_number": "CHAMGE_IT", + "payment_options": { + "allowed_payment_method": "INSTANT_FUNDING_SOURCE" + }, + "soft_descriptor": "ECHI5786755", + "item_list": { + "items": [{ + "name": "hat", + "description": "Brown color hat", + "quantity": "5", + "price": "3", + "tax": "0.01", + "sku": "1", + "currency": "USD" + }, { + "name": "handbag", + "description": "Black color hand bag", + "quantity": "1", + "price": "15", + "tax": "0.02", + "sku": "product34", + "currency": "USD" + }], + "shipping_address": { + "recipient_name": "Hello World", + "line1": "4thFloor", + "line2": "unit#34", + "city": "SAn Jose", + "country_code": "US", + "postal_code": "95131", + "phone": "011862212345678", + "state": "CA" + } + } + }], + "note_to_payer": "Contact us for any questions on your order.", + "redirect_urls": { + "return_url": "https://example.com", + "cancel_url": "https://example.com" + } +} + +#%% +def read_json(filepath): + with open(filepath, "r") as f: + return json.loads(f.read()) + +def get_api_token(): + + client_id = CREDS.get('client_id') + secret = CREDS.get('secret') + + token_refresh_endpoint='https://api-m.sandbox.paypal.com/v1/oauth2/token' + data = "grant_type=client_credentials" + headers = { + 'Accept': 'application/json', + 'Accept-Language': 'en_US' + } + auth = (client_id, secret) + response = requests.request( + method="POST", + url=token_refresh_endpoint, + data=data, + headers=headers, + auth=auth) + response_json = response.json() + # print(response_json) + API_TOKEN = response_json["access_token"] + return API_TOKEN + + + +#%% +def random_digits(digits): + lower = 10**(digits-1) + upper = 10**digits - 1 + return random.randint(lower, upper) + +def make_payment(): + + # generate new invoice_number + PAYMENT_DATA['transactions'][0]['invoice_number'] = random_digits(11) + + response = requests.request( + method="POST", + url='https://api-m.sandbox.paypal.com/v1/payments/payment', + headers=headers, + data=json.dumps(PAYMENT_DATA) + ) + response_json = response.json() + # pprint(response_json) + + execute_url = '' + approval_url = '' + + for link in response_json['links']: + if link['rel'] == 'approval_url': + approval_url = link['href'] + elif link['rel'] == 'execute': + execute_url = link['href'] + elif link['rel'] == 'self': + self_url = link['href'] + + print(f'Payment made: {self_url}') + return approval_url, execute_url + +#%% APPROVE PAYMENT +def login(): + driver = webdriver.Chrome("/usr/bin/chromedriver") + + # SIGN_IN + driver.get("https://www.sandbox.paypal.com/ua/signin") + driver.find_element_by_id("email").send_keys(CREDS['buyer_username']) + driver.find_element_by_id("btnNext").click() + sleep(2) + driver.find_element_by_id("password").send_keys(CREDS['buyer_password']) + driver.find_element_by_id("btnLogin").click() + return driver + +def approve_payment(driver, url): + driver.get(url) + global cookies_accepted + + sleep(3) + if not cookies_accepted: + cookies = driver.find_element_by_id("acceptAllButton") + if cookies: + cookies.click() + + cookies_accepted = True + driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") + + element = WebDriverWait(driver, 20).until( + EC.presence_of_element_located((By.ID, "payment-submit-btn"))) + sleep(1) + element.click() + + # sleep(5) + # driver.find_element_by_id("payment-submit-btn").click() + + wait = WebDriverWait(driver, 5) + wait.until(EC.title_is('Example Domain')) + print(f'Payment approved: {driver.current_url}') + + +#%% +def execute_payment(url): + response = requests.request( + method="POST", + url=url, + data='{"payer_id": "ZE5533HZPGMC6"}', + headers=headers + ) + response_json = response.json() + print(f'Payment executed: {url} with STATE: {response_json["state"]}') + + +#%% +TOTAL_TRANSACTIONS = int(sys.argv[1]) if len(sys.argv) > 1 else 3 + +CREDS = read_json("../secrets/creds.json") +headers = { + 'Authorization': f'Bearer {get_api_token()}', + 'Content-Type': 'application/json' +} +driver = login() +cookies_accepted = False +for i in range(TOTAL_TRANSACTIONS): + print(f'Payment #{i}') + approval_url, execute_url = make_payment() + approve_payment(driver, approval_url) + execute_payment(execute_url) +driver.quit() From e0cf3c3d32c7bfdc8e4abd70b828fdba429ef2ae Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Wed, 7 Jul 2021 04:35:52 +0300 Subject: [PATCH 55/60] fixed styling --- .../bin/fixture_helper.py | 61 +++-- .../bin/paypal_transaction_generator.py | 242 +++++++++--------- 2 files changed, 166 insertions(+), 137 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/fixture_helper.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/fixture_helper.py index 5d5a057ff618..f717706b17d8 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/bin/fixture_helper.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/fixture_helper.py @@ -1,7 +1,32 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +import logging +from pprint import pprint + # %% import requests -from pprint import pprint -import logging logging.basicConfig(level=logging.DEBUG) @@ -11,21 +36,21 @@ "secret": "REPLACE_ME", "start_date": "2021-06-01T00:00:00+00:00", "end_date": "2021-06-30T00:00:00+00:00", - "is_sandbox": True + "is_sandbox": True, } # %% READ and -client_id = specification.get('client_id') -secret = specification.get('secret') +client_id = specification.get("client_id") +secret = specification.get("secret") # %% GET API_TOKEN -token_refresh_endpoint = 'https://api-m.sandbox.paypal.com/v1/oauth2/token' +token_refresh_endpoint = "https://api-m.sandbox.paypal.com/v1/oauth2/token" data = "grant_type=client_credentials" headers = { - 'Accept': 'application/json', - 'Accept-Language': 'en_US' + "Accept": "application/json", + "Accept-Language": "en_US", } response = requests.request( @@ -33,13 +58,13 @@ url=token_refresh_endpoint, data=data, headers=headers, - auth=(client_id, secret) + auth=(client_id, secret), ) response_json = response.json() print(response_json) API_TOKEN = response_json["access_token"] -### CREATE TRANSACTIONS +# CREATE TRANSACTIONS # for i in range(1000): # create_response = requests.post( # "https://api-m.sandbox.paypal.com/v2/checkout/orders", @@ -64,21 +89,21 @@ url = "https://api-m.sandbox.paypal.com/v1/reporting/transactions" params = { - 'start_date': '2021-06-20T00:00:00+00:00', - 'end_date': '2021-07-10T07:19:45Z', - 'fields': 'all', - 'page_size': '100', - 'page': '1' + "start_date": "2021-06-20T00:00:00+00:00", + "end_date": "2021-07-10T07:19:45Z", + "fields": "all", + "page_size": "100", + "page": "1", } headers = { - 'Authorization': f'Bearer {API_TOKEN}', - 'Content-Type': 'application/json' + "Authorization": f"Bearer {API_TOKEN}", + "Content-Type": "application/json", } response = requests.get( url, headers=headers, - params=params + params=params, ) pprint(response.json()) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py index baedb78fe289..ab90e4117fb6 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py @@ -1,4 +1,28 @@ -#%% +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +# # REQUIREMENTS: # 1. sudo apt-get install chromium-chromedriver # 2. pip install selenium @@ -8,157 +32,148 @@ # python paypal_transaction_generator.py - will generate 3 transactions by default # python paypal_transaction_generator.py 10 - will generate 10 transactions -import sys -import requests import json +import random +import sys +from time import sleep -from pprint import pprint +import requests from selenium import webdriver from selenium.webdriver.common.by import By -from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC -import random -from time import sleep +from selenium.webdriver.support.ui import WebDriverWait + +# from pprint import pprint + PAYMENT_DATA = { - "intent": "sale", - "payer": { - "payment_method": "paypal" - }, - "transactions": [{ - "amount": { - "total": "30.11", - "currency": "USD", - "details": { - "subtotal": "30.00", - "tax": "0.07", - "shipping": "0.03", - "handling_fee": "1.00", - "shipping_discount": "-1.00", - "insurance": "0.01" - } - }, - "description": "This is the payment transaction description.", - "custom": "EBAY_EMS_90048630020055", - "invoice_number": "CHAMGE_IT", - "payment_options": { - "allowed_payment_method": "INSTANT_FUNDING_SOURCE" - }, - "soft_descriptor": "ECHI5786755", - "item_list": { - "items": [{ - "name": "hat", - "description": "Brown color hat", - "quantity": "5", - "price": "3", - "tax": "0.01", - "sku": "1", - "currency": "USD" - }, { - "name": "handbag", - "description": "Black color hand bag", - "quantity": "1", - "price": "15", - "tax": "0.02", - "sku": "product34", - "currency": "USD" - }], - "shipping_address": { - "recipient_name": "Hello World", - "line1": "4thFloor", - "line2": "unit#34", - "city": "SAn Jose", - "country_code": "US", - "postal_code": "95131", - "phone": "011862212345678", - "state": "CA" - } - } - }], - "note_to_payer": "Contact us for any questions on your order.", - "redirect_urls": { - "return_url": "https://example.com", - "cancel_url": "https://example.com" - } + "intent": "sale", + "payer": {"payment_method": "paypal"}, + "transactions": [ + { + "amount": { + "total": "30.11", + "currency": "USD", + "details": { + "subtotal": "30.00", + "tax": "0.07", + "shipping": "0.03", + "handling_fee": "1.00", + "shipping_discount": "-1.00", + "insurance": "0.01", + }, + }, + "description": "This is the payment transaction description.", + "custom": "EBAY_EMS_90048630020055", + "invoice_number": "CHAMGE_IT", + "payment_options": {"allowed_payment_method": "INSTANT_FUNDING_SOURCE"}, + "soft_descriptor": "ECHI5786755", + "item_list": { + "items": [ + { + "name": "hat", + "description": "Brown color hat", + "quantity": "5", + "price": "3", + "tax": "0.01", + "sku": "1", + "currency": "USD", + }, + { + "name": "handbag", + "description": "Black color hand bag", + "quantity": "1", + "price": "15", + "tax": "0.02", + "sku": "product34", + "currency": "USD", + }, + ], + "shipping_address": { + "recipient_name": "Hello World", + "line1": "4thFloor", + "line2": "unit#34", + "city": "SAn Jose", + "country_code": "US", + "postal_code": "95131", + "phone": "011862212345678", + "state": "CA", + }, + }, + } + ], + "note_to_payer": "Contact us for any questions on your order.", + "redirect_urls": {"return_url": "https://example.com", "cancel_url": "https://example.com"}, } -#%% + def read_json(filepath): with open(filepath, "r") as f: return json.loads(f.read()) + def get_api_token(): - client_id = CREDS.get('client_id') - secret = CREDS.get('secret') + client_id = CREDS.get("client_id") + secret = CREDS.get("secret") - token_refresh_endpoint='https://api-m.sandbox.paypal.com/v1/oauth2/token' + token_refresh_endpoint = "https://api-m.sandbox.paypal.com/v1/oauth2/token" data = "grant_type=client_credentials" - headers = { - 'Accept': 'application/json', - 'Accept-Language': 'en_US' - } + headers = {"Accept": "application/json", "Accept-Language": "en_US"} auth = (client_id, secret) - response = requests.request( - method="POST", - url=token_refresh_endpoint, - data=data, - headers=headers, - auth=auth) + response = requests.request(method="POST", url=token_refresh_endpoint, data=data, headers=headers, auth=auth) response_json = response.json() # print(response_json) API_TOKEN = response_json["access_token"] return API_TOKEN - -#%% def random_digits(digits): - lower = 10**(digits-1) - upper = 10**digits - 1 + lower = 10 ** (digits - 1) + upper = 10 ** digits - 1 return random.randint(lower, upper) + def make_payment(): # generate new invoice_number - PAYMENT_DATA['transactions'][0]['invoice_number'] = random_digits(11) + PAYMENT_DATA["transactions"][0]["invoice_number"] = random_digits(11) response = requests.request( - method="POST", - url='https://api-m.sandbox.paypal.com/v1/payments/payment', - headers=headers, - data=json.dumps(PAYMENT_DATA) + method="POST", url="https://api-m.sandbox.paypal.com/v1/payments/payment", headers=headers, data=json.dumps(PAYMENT_DATA) ) response_json = response.json() # pprint(response_json) - execute_url = '' - approval_url = '' + execute_url = "" + approval_url = "" - for link in response_json['links']: - if link['rel'] == 'approval_url': - approval_url = link['href'] - elif link['rel'] == 'execute': - execute_url = link['href'] - elif link['rel'] == 'self': - self_url = link['href'] + for link in response_json["links"]: + if link["rel"] == "approval_url": + approval_url = link["href"] + elif link["rel"] == "execute": + execute_url = link["href"] + elif link["rel"] == "self": + self_url = link["href"] - print(f'Payment made: {self_url}') + print(f"Payment made: {self_url}") return approval_url, execute_url -#%% APPROVE PAYMENT + +# APPROVE PAYMENT def login(): driver = webdriver.Chrome("/usr/bin/chromedriver") # SIGN_IN driver.get("https://www.sandbox.paypal.com/ua/signin") - driver.find_element_by_id("email").send_keys(CREDS['buyer_username']) + driver.find_element_by_id("email").send_keys(CREDS["buyer_username"]) driver.find_element_by_id("btnNext").click() sleep(2) - driver.find_element_by_id("password").send_keys(CREDS['buyer_password']) + driver.find_element_by_id("password").send_keys(CREDS["buyer_password"]) driver.find_element_by_id("btnLogin").click() return driver + def approve_payment(driver, url): driver.get(url) global cookies_accepted @@ -172,8 +187,7 @@ def approve_payment(driver, url): cookies_accepted = True driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") - element = WebDriverWait(driver, 20).until( - EC.presence_of_element_located((By.ID, "payment-submit-btn"))) + element = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "payment-submit-btn"))) sleep(1) element.click() @@ -181,34 +195,24 @@ def approve_payment(driver, url): # driver.find_element_by_id("payment-submit-btn").click() wait = WebDriverWait(driver, 5) - wait.until(EC.title_is('Example Domain')) - print(f'Payment approved: {driver.current_url}') + wait.until(EC.title_is("Example Domain")) + print(f"Payment approved: {driver.current_url}") -#%% def execute_payment(url): - response = requests.request( - method="POST", - url=url, - data='{"payer_id": "ZE5533HZPGMC6"}', - headers=headers - ) + response = requests.request(method="POST", url=url, data='{"payer_id": "ZE5533HZPGMC6"}', headers=headers) response_json = response.json() print(f'Payment executed: {url} with STATE: {response_json["state"]}') -#%% TOTAL_TRANSACTIONS = int(sys.argv[1]) if len(sys.argv) > 1 else 3 CREDS = read_json("../secrets/creds.json") -headers = { - 'Authorization': f'Bearer {get_api_token()}', - 'Content-Type': 'application/json' -} +headers = {"Authorization": f"Bearer {get_api_token()}", "Content-Type": "application/json"} driver = login() cookies_accepted = False for i in range(TOTAL_TRANSACTIONS): - print(f'Payment #{i}') + print(f"Payment #{i}") approval_url, execute_url = make_payment() approve_payment(driver, approval_url) execute_payment(execute_url) From 506c02a5382d22a3a835a9283643850b6809debf Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Wed, 7 Jul 2021 04:36:48 +0300 Subject: [PATCH 56/60] maximum allowed start_date is extracted from API response now. --- .../integration_tests/abnormal_state.json | 4 +- .../source_paypal_transaction/source.py | 69 +++++++++++++++++-- .../unit_tests/unit_test.py | 2 +- 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json index 0a05b9cd2a0e..0f98bdb14a65 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json @@ -1,8 +1,8 @@ { "transactions": { - "date": "2021-06-08T12:00:00+00:00" + "date": "2021-07-10T23:46:44+00:00" }, "balances": { - "date": "2021-06-08T12:00:00+00:00" + "date": "2021-07-05T23:00:00+00:00" } } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index f56b7fa5eace..a0a72b6fff1d 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -49,9 +49,10 @@ class PaypalTransactionStream(HttpStream, ABC): # Date limits are needed to prevent API error: Data for the given start date is not available # API limit: (now() - start_date_min) < start_date < (now() - start_date_max) - start_date_min: Mapping[str, int] = {"days": 3 * 364} # API limit - 3 years - start_date_max: Mapping[str, int] = {"hours": 0} - + start_date_min: Mapping[str, int] = {"hours": 3 * 364} # API limit - 3 years + # Actual value can be found only in 'last_refreshed_datetime' attr of API response + # start_date_max: Mapping[str, int] = {"hours": 0} + last_refreshed_datetime: Optional[datetime] = None # extracted from API response. Indicate the latest possible start_date stream_slice_period: Mapping[str, int] = {"days": 1} # max period is 31 days (API limit) requests_per_minute: int = 30 # API limit is 50 reqs/min from 1 IP to all endpoints, otherwise IP is banned for 5 mins @@ -82,7 +83,8 @@ def __init__( ) start_date = minimum_allowed_start_date - self.maximum_allowed_start_date = min(now - timedelta(**self.start_date_max), self.end_date) + # self.maximum_allowed_start_date = min(now - timedelta(**self.start_date_max), self.end_date) + self.maximum_allowed_start_date = min(now, self.end_date) if start_date > self.maximum_allowed_start_date: self.logger.log( "WARN", @@ -115,6 +117,11 @@ def backoff_time(self, response: requests.Response) -> Optional[float]: def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: json_response = response.json() + + # Save extracted last_refreshed_datetime to use it as maximum allowed start_date + last_refreshed_datetime = json_response.get("last_refreshed_datetime") + self.last_refreshed_datetime = isoparse(last_refreshed_datetime) if last_refreshed_datetime else None + if self.data_field is not None: data = json_response.get(self.data_field, []) else: @@ -171,6 +178,9 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late else: return {"date": self.start_date.isoformat()} + def get_last_refreshed_datetime(self, sync_mode): + return None + def stream_slices( self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, any]]]: @@ -182,17 +192,45 @@ def stream_slices( """ period = timedelta(**self.stream_slice_period) - slice_start_date = max(self.start_date, isoparse(stream_state.get("date")) if stream_state else self.start_date) + # get last_refreshed_datetime from API response to use as maximum allowed start_date + last_refreshed_datetime = self.get_last_refreshed_datetime(sync_mode) + + if last_refreshed_datetime: + self.logger.info(f"Maximum possible start_date is {last_refreshed_datetime} based on info from API response") + self.maximum_allowed_start_date = min(last_refreshed_datetime, self.maximum_allowed_start_date) + + slice_start_date = self.start_date + + if stream_state: + # if stream_state_date is in the future (for example during tests) then reset it to now: + stream_state_date = min(isoparse(stream_state.get("date")), self.maximum_allowed_start_date) + + # but slice_start_date should be the most recent date: + slice_start_date = max(slice_start_date, stream_state_date) slices = [] while slice_start_date <= self.maximum_allowed_start_date: slices.append( - {"start_date": slice_start_date.isoformat(), "end_date": min(slice_start_date + period, self.end_date).isoformat()} + { + "start_date": slice_start_date.isoformat(), + "end_date": min(slice_start_date + period, self.end_date).isoformat(), + } ) slice_start_date += period return slices + def _send_request(self, request: requests.PreparedRequest) -> requests.Response: + try: + return super()._send_request(request) + except requests.exceptions.HTTPError as e: + error_message = e.response.text + if error_message: + self.logger.error(f"Stream {self.name}: {e.response.status_code} {e.response.reason} - {error_message}") + exit(1) + else: + raise e + class Transactions(PaypalTransactionStream): """ @@ -207,7 +245,9 @@ class Transactions(PaypalTransactionStream): primary_key = [["transaction_info", "transaction_id"]] cursor_field = ["transaction_info", "transaction_initiation_date"] - start_date_max = {"hours": 36} # this limit is found experimentally + # according to the docs, 3 hrs are needed for new transaction to appear in transaction list. + # In fact, this value can be up to 3 days. Actual value can be found only in 'last_refreshed_datetime' attr of API response + # start_date_max = {"hours": 3} # TODO handle API error when 1 request returns more than 10000 records. # https://github.com/airbytehq/airbyte/issues/4404 @@ -240,6 +280,21 @@ def request_params( "page": page_number, } + def get_last_refreshed_datetime(self, sync_mode): + # Run new stream just in order to extract last_refreshed_datetime from API response + slice = { + "start_date": self.start_date.isoformat(), + "end_date": self.start_date.isoformat(), + } + paypal_transaction = Transactions( + authenticator=self.authenticator, + start_date=self.start_date, + end_date=self.start_date, + is_sandbox=self.is_sandbox, + ) + list(paypal_transaction.read_records(sync_mode=sync_mode, stream_slice=slice)) + return paypal_transaction.last_refreshed_datetime + class Balances(PaypalTransactionStream): """ diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py index 1ace49180c8a..3199bc2eb081 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py @@ -79,7 +79,7 @@ def now(): return datetime.now().replace(microsecond=0).astimezone() -def test_transactions_stream_slices(): +def transactions_stream_slices(): start_date_max = Transactions.start_date_max From 45bdae5184fcc94738e33d8b0aff1cf06c3ab5e6 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Wed, 7 Jul 2021 10:32:51 +0300 Subject: [PATCH 57/60] fixed schemas --- .../source_paypal_transaction/schemas/balances.json | 2 +- .../source_paypal_transaction/schemas/transactions.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json index 06fb1d13695d..364b96ae5f09 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json @@ -53,7 +53,7 @@ "type": "datetime" }, "last_refresh_time": { - "type": "datetime" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json index 2de185fb6324..fcd85a867cef 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json @@ -18,7 +18,7 @@ "type": ["null", "datetime"] }, "transaction_updated_date": { - "type": ["null", "datetime"] + "type": ["null", "string"] }, "transaction_amount": { "type": ["null", "object"], From 290bb94ca63b7b5d1911f0bdb849ec4e352f1052 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Wed, 7 Jul 2021 11:42:37 +0300 Subject: [PATCH 58/60] fixed schemas - removed datetime --- .../source_paypal_transaction/schemas/balances.json | 2 +- .../source_paypal_transaction/schemas/transactions.json | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json index 364b96ae5f09..cfed43f13e47 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json @@ -50,7 +50,7 @@ "type": ["null", "string"] }, "as_of_time": { - "type": "datetime" + "type": "string" }, "last_refresh_time": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json index fcd85a867cef..7443e216f303 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json @@ -15,7 +15,7 @@ "type": ["null", "string"] }, "transaction_initiation_date": { - "type": ["null", "datetime"] + "type": ["null", "string"] }, "transaction_updated_date": { "type": ["null", "string"] @@ -238,7 +238,7 @@ } }, "store_info": { - "type": "object", + "type": ["null", "object"], "properties": { "store_id": { "type": ["null", "string"] @@ -249,7 +249,7 @@ } }, "auction_info": { - "type": "object", + "type": ["null", "object"], "properties": { "auction_site": { "type": ["null", "string"] @@ -266,7 +266,7 @@ } }, "incentive_info": { - "type": "object", + "type": ["null", "object"], "properties": { "incentive_details": { "type": "array", From 4a30c26188fb29f478105caf471011db8fede92e Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 8 Jul 2021 15:45:40 +0300 Subject: [PATCH 59/60] now maximum_allowed_start_date is identified by last_refreshed_datetime attr in API response. --- .../integration_tests/abnormal_state.json | 4 +- .../source_paypal_transaction/source.py | 113 +++++++++-------- .../unit_tests/unit_test.py | 114 ++++++++++++------ 3 files changed, 132 insertions(+), 99 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json index 0f98bdb14a65..f60c258e0e01 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json @@ -1,8 +1,8 @@ { "transactions": { - "date": "2021-07-10T23:46:44+00:00" + "date": "2021-07-11T23:00:00+00:00" }, "balances": { - "date": "2021-07-05T23:00:00+00:00" + "date": "2021-07-11T23:00:00+00:00" } } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index a0a72b6fff1d..ac28b724d379 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -44,15 +44,27 @@ def get_endpoint(is_sandbox: bool = False) -> str: class PaypalTransactionStream(HttpStream, ABC): + """Abstract class for Paypal Transaction Stream. + + Important note about 'start_date' params: + 'start_date' is one of required params, it comes from spec configuration or from stream state. + In both cases it must meet the following conditions: + + minimum_allowed_start_date <= start_date <= end_date <= last_refreshed_datetime <= now() + + otherwise API throws an "Data for the given start date is not available" error. + + So the prevent this error 'start_date' will be reset to: + minimum_allowed_start_date - if 'start_date' is too old + min(maximum_allowed_start_date, last_refreshed_datetime) - if 'start_date' is too recent + """ page_size = "500" # API limit - # Date limits are needed to prevent API error: Data for the given start date is not available - # API limit: (now() - start_date_min) < start_date < (now() - start_date_max) - start_date_min: Mapping[str, int] = {"hours": 3 * 364} # API limit - 3 years - # Actual value can be found only in 'last_refreshed_datetime' attr of API response - # start_date_max: Mapping[str, int] = {"hours": 0} - last_refreshed_datetime: Optional[datetime] = None # extracted from API response. Indicate the latest possible start_date + # Date limits are needed to prevent API error: "Data for the given start date is not available" + # API limit: (now() - start_date_min) <= start_date <= end_date <= last_refreshed_datetime <= now + start_date_min: Mapping[str, int] = {"hours": 3 * 365} # API limit - 3 years + last_refreshed_datetime: Optional[datetime] = None # extracted from API response. Indicate the most resent possible start_date stream_slice_period: Mapping[str, int] = {"days": 1} # max period is 31 days (API limit) requests_per_minute: int = 30 # API limit is 50 reqs/min from 1 IP to all endpoints, otherwise IP is banned for 5 mins @@ -83,7 +95,6 @@ def __init__( ) start_date = minimum_allowed_start_date - # self.maximum_allowed_start_date = min(now - timedelta(**self.start_date_max), self.end_date) self.maximum_allowed_start_date = min(now, self.end_date) if start_date > self.maximum_allowed_start_date: self.logger.log( @@ -179,7 +190,25 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late return {"date": self.start_date.isoformat()} def get_last_refreshed_datetime(self, sync_mode): - return None + """Get last_refreshed_datetime attribute from API response by running PaypalTransactionStream().read_records() + with 'empty' stream_slice (range=0) + + last_refreshed_datetime indicates the maximum available start_date for which API has data. + If request start_date > last_refreshed_datetime then API throws an error: + "Data for the given start date is not available" + """ + paypal_stream = self.__class__( + authenticator=self.authenticator, + start_date=self.start_date, + end_date=self.start_date, + is_sandbox=self.is_sandbox, + ) + stream_slice = { + "start_date": self.start_date.isoformat(), + "end_date": self.start_date.isoformat(), + } + list(paypal_stream.read_records(sync_mode=sync_mode, stream_slice=stream_slice)) + return paypal_stream.last_refreshed_datetime def stream_slices( self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None @@ -187,25 +216,22 @@ def stream_slices( """ Returns a list of slices for each day (by default) between the start date and end date. The return value is a list of dicts {'start_date': date_string, 'end_date': date_string}. - - Slice does not cover period for slice_start_date > maximum_allowed_start_date """ period = timedelta(**self.stream_slice_period) # get last_refreshed_datetime from API response to use as maximum allowed start_date - last_refreshed_datetime = self.get_last_refreshed_datetime(sync_mode) - - if last_refreshed_datetime: - self.logger.info(f"Maximum possible start_date is {last_refreshed_datetime} based on info from API response") - self.maximum_allowed_start_date = min(last_refreshed_datetime, self.maximum_allowed_start_date) + self.last_refreshed_datetime = self.get_last_refreshed_datetime(sync_mode) + if self.last_refreshed_datetime: + self.logger.info(f"Maximum allowed start_date is {self.last_refreshed_datetime} based on info from API response") + self.maximum_allowed_start_date = min(self.last_refreshed_datetime, self.maximum_allowed_start_date) slice_start_date = self.start_date if stream_state: - # if stream_state_date is in the future (for example during tests) then reset it to now: + # if stream_state_date is in the future (for example during tests) then reset it to maximum_allowed_start_date: stream_state_date = min(isoparse(stream_state.get("date")), self.maximum_allowed_start_date) - # but slice_start_date should be the most recent date: + # slice_start_date should be the most recent date: slice_start_date = max(slice_start_date, stream_state_date) slices = [] @@ -220,35 +246,17 @@ def stream_slices( return slices - def _send_request(self, request: requests.PreparedRequest) -> requests.Response: - try: - return super()._send_request(request) - except requests.exceptions.HTTPError as e: - error_message = e.response.text - if error_message: - self.logger.error(f"Stream {self.name}: {e.response.status_code} {e.response.reason} - {error_message}") - exit(1) - else: - raise e - class Transactions(PaypalTransactionStream): - """ - Stream for Transactions /v1/reporting/transactions - - API returns a list of transaction on a specific date range - + """List Paypal Transactions on a specific date range API Docs: https://developer.paypal.com/docs/integration/direct/transaction-search/#list-transactions + Endpoint: /v1/reporting/transactions """ data_field = "transaction_details" primary_key = [["transaction_info", "transaction_id"]] cursor_field = ["transaction_info", "transaction_initiation_date"] - # according to the docs, 3 hrs are needed for new transaction to appear in transaction list. - # In fact, this value can be up to 3 days. Actual value can be found only in 'last_refreshed_datetime' attr of API response - # start_date_max = {"hours": 3} - # TODO handle API error when 1 request returns more than 10000 records. # https://github.com/airbytehq/airbyte/issues/4404 records_per_request = 10000 @@ -280,29 +288,10 @@ def request_params( "page": page_number, } - def get_last_refreshed_datetime(self, sync_mode): - # Run new stream just in order to extract last_refreshed_datetime from API response - slice = { - "start_date": self.start_date.isoformat(), - "end_date": self.start_date.isoformat(), - } - paypal_transaction = Transactions( - authenticator=self.authenticator, - start_date=self.start_date, - end_date=self.start_date, - is_sandbox=self.is_sandbox, - ) - list(paypal_transaction.read_records(sync_mode=sync_mode, stream_slice=slice)) - return paypal_transaction.last_refreshed_datetime - class Balances(PaypalTransactionStream): - """ - Stream for Balances /v1/reporting/balances - - API returns account balance on a specific date - - API Docs: https://developer.paypal.com/docs/integration/direct/transaction-search/#check-balances + """Get account balance on a specific date + API Docs: https://developer.paypal.com/docs/integration/direct/transaction-search/#check-balancess """ primary_key = "as_of_time" @@ -351,7 +340,13 @@ def refresh_access_token(self) -> Tuple[str, int]: data = "grant_type=client_credentials" headers = {"Accept": "application/json", "Accept-Language": "en_US"} auth = (self.client_id, self.client_secret) - response = requests.request(method="POST", url=self.token_refresh_endpoint, data=data, headers=headers, auth=auth) + response = requests.request( + method="POST", + url=self.token_refresh_endpoint, + data=data, + headers=headers, + auth=auth, + ) response.raise_for_status() response_json = response.json() return response_json["access_token"], response_json["expires_in"] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py index 3199bc2eb081..78a4ff3c1ce8 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py @@ -79,60 +79,76 @@ def now(): return datetime.now().replace(microsecond=0).astimezone() -def transactions_stream_slices(): +def test_transactions_stream_slices(): - start_date_max = Transactions.start_date_max + start_date_max = {"hours": 0} # if start_date > now - **start_date_max then no slices - stream_slices = Transactions( + transactions = Transactions( authenticator=NoAuth(), start_date=now() - timedelta(**start_date_max) - timedelta(minutes=2), - ).stream_slices(sync_mode="any") + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any") assert 1 == len(stream_slices) # start_date <= now - **start_date_max - stream_slices = Transactions( + transactions = Transactions( authenticator=NoAuth(), start_date=now() - timedelta(**start_date_max), - ).stream_slices(sync_mode="any") + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any") assert 1 == len(stream_slices) - stream_slices = Transactions( + transactions = Transactions( authenticator=NoAuth(), start_date=now() - timedelta(**start_date_max) + timedelta(minutes=2), - ).stream_slices(sync_mode="any") + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any") assert 1 == len(stream_slices) - stream_slices = Transactions( + transactions = Transactions( authenticator=NoAuth(), start_date=now() - timedelta(**start_date_max) - timedelta(hours=2), - ).stream_slices(sync_mode="any") + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any") assert 1 == len(stream_slices) - stream_slices = Transactions( + transactions = Transactions( authenticator=NoAuth(), start_date=now() - timedelta(**start_date_max) - timedelta(days=1), - ).stream_slices(sync_mode="any") + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any") assert 2 == len(stream_slices) - stream_slices = Transactions( + transactions = Transactions( authenticator=NoAuth(), start_date=now() - timedelta(**start_date_max) - timedelta(days=1, hours=2), - ).stream_slices(sync_mode="any") + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any") assert 2 == len(stream_slices) - stream_slices = Transactions( + transactions = Transactions( authenticator=NoAuth(), start_date=now() - timedelta(**start_date_max) - timedelta(days=30, minutes=1), - ).stream_slices(sync_mode="any") + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any") assert 31 == len(stream_slices) # tests with specified end_date - stream_slices = Transactions( + transactions = Transactions( authenticator=NoAuth(), start_date=isoparse("2021-06-01T10:00:00+00:00"), end_date=isoparse("2021-06-04T12:00:00+00:00"), - ).stream_slices(sync_mode="any") + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any") assert [ {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T10:00:00+00:00"}, {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, @@ -141,22 +157,26 @@ def transactions_stream_slices(): ] == stream_slices # tests with specified end_date and stream_state - stream_slices = Transactions( + transactions = Transactions( authenticator=NoAuth(), start_date=isoparse("2021-06-01T10:00:00+00:00"), end_date=isoparse("2021-06-04T12:00:00+00:00"), - ).stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"}) + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"}) assert [ {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-04T10:00:00+00:00"}, {"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, ] == stream_slices - stream_slices = Transactions( + transactions = Transactions( authenticator=NoAuth(), start_date=isoparse("2021-06-01T10:00:00+00:00"), end_date=isoparse("2021-06-04T12:00:00+00:00"), - ).stream_slices(sync_mode="any", stream_state={"date": "2021-06-04T10:00:00+00:00"}) + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any", stream_state={"date": "2021-06-04T10:00:00+00:00"}) assert [{"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}] == stream_slices @@ -167,36 +187,48 @@ def test_balances_stream_slices(): now = datetime.now().replace(microsecond=0).astimezone() # Test without end_date (it equal by default) - stream_slices = Balances(authenticator=NoAuth(), start_date=now).stream_slices(sync_mode="any") + balance = Balances(authenticator=NoAuth(), start_date=now) + balance.get_last_refreshed_datetime = lambda x: None + stream_slices = balance.stream_slices(sync_mode="any") assert 1 == len(stream_slices) - stream_slices = Balances(authenticator=NoAuth(), start_date=now - timedelta(minutes=1)).stream_slices(sync_mode="any") + balance = Balances(authenticator=NoAuth(), start_date=now - timedelta(minutes=1)) + balance.get_last_refreshed_datetime = lambda x: None + stream_slices = balance.stream_slices(sync_mode="any") assert 1 == len(stream_slices) - stream_slices = Balances( + balance = Balances( authenticator=NoAuth(), start_date=now - timedelta(hours=23), - ).stream_slices(sync_mode="any") + ) + balance.get_last_refreshed_datetime = lambda x: None + stream_slices = balance.stream_slices(sync_mode="any") assert 1 == len(stream_slices) - stream_slices = Balances( + balance = Balances( authenticator=NoAuth(), start_date=now - timedelta(days=1), - ).stream_slices(sync_mode="any") + ) + balance.get_last_refreshed_datetime = lambda x: None + stream_slices = balance.stream_slices(sync_mode="any") assert 2 == len(stream_slices) - stream_slices = Balances( + balance = Balances( authenticator=NoAuth(), start_date=now - timedelta(days=1, minutes=1), - ).stream_slices(sync_mode="any") + ) + balance.get_last_refreshed_datetime = lambda x: None + stream_slices = balance.stream_slices(sync_mode="any") assert 2 == len(stream_slices) # test with custom end_date - stream_slices = Balances( + balance = Balances( authenticator=NoAuth(), start_date=isoparse("2021-06-01T10:00:00+00:00"), end_date=isoparse("2021-06-03T12:00:00+00:00"), - ).stream_slices(sync_mode="any") + ) + balance.get_last_refreshed_datetime = lambda x: None + stream_slices = balance.stream_slices(sync_mode="any") assert [ {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T10:00:00+00:00"}, {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, @@ -204,26 +236,32 @@ def test_balances_stream_slices(): ] == stream_slices # Test with stream state - stream_slices = Balances( + balance = Balances( authenticator=NoAuth(), start_date=isoparse("2021-06-01T10:00:00+00:00"), end_date=isoparse("2021-06-03T12:00:00+00:00"), - ).stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"}) + ) + balance.get_last_refreshed_datetime = lambda x: None + stream_slices = balance.stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"}) assert [ {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}, ] == stream_slices - stream_slices = Balances( + balance = Balances( authenticator=NoAuth(), start_date=isoparse("2021-06-01T10:00:00+00:00"), end_date=isoparse("2021-06-03T12:00:00+00:00"), - ).stream_slices(sync_mode="any", stream_state={"date": "2021-06-03T11:00:00+00:00"}) + ) + balance.get_last_refreshed_datetime = lambda x: None + stream_slices = balance.stream_slices(sync_mode="any", stream_state={"date": "2021-06-03T11:00:00+00:00"}) assert [{"start_date": "2021-06-03T11:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}] == stream_slices - stream_slices = Balances( + balance = Balances( authenticator=NoAuth(), start_date=isoparse("2021-06-01T10:00:00+00:00"), end_date=isoparse("2021-06-03T12:00:00+00:00"), - ).stream_slices(sync_mode="any", stream_state={"date": "2021-06-03T12:00:00+00:00"}) + ) + balance.get_last_refreshed_datetime = lambda x: None + stream_slices = balance.stream_slices(sync_mode="any", stream_state={"date": "2021-06-03T12:00:00+00:00"}) assert [{"start_date": "2021-06-03T12:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}] == stream_slices From 2a43d629b87163c5820ea00a6085dc3128543a04 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 8 Jul 2021 16:23:24 +0300 Subject: [PATCH 60/60] added possibility to specify additional properties --- .../source_paypal_transaction/spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json index 23a0e3e19f37..694d3bb9ff6a 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json @@ -5,7 +5,7 @@ "title": "Paypal Transaction Search", "type": "object", "required": ["client_id", "secret", "start_date", "is_sandbox"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "client_id": { "title": "Client ID",