diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 0e79bbe2f854..6dfb5913c895 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -108,6 +108,7 @@ jobs: MICROSOFT_TEAMS_TEST_CREDS: ${{ secrets.MICROSOFT_TEAMS_TEST_CREDS }} MIXPANEL_INTEGRATION_TEST_CREDS: ${{ secrets.MIXPANEL_INTEGRATION_TEST_CREDS }} MSSQL_RDS_TEST_CREDS: ${{ secrets.MSSQL_RDS_TEST_CREDS }} + PAYPAL_TRANSACTION_CREDS: ${{ secrets.SOURCE_PAYPAL_TRANSACTION_CREDS }} POSTHOG_TEST_CREDS: ${{ secrets.POSTHOG_TEST_CREDS }} RECHARGE_INTEGRATION_TEST_CREDS: ${{ secrets.RECHARGE_INTEGRATION_TEST_CREDS }} QUICKBOOKS_TEST_CREDS: ${{ secrets.QUICKBOOKS_TEST_CREDS }} diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 7b9f3983ee8c..545d70a568ca 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -107,6 +107,7 @@ jobs: MICROSOFT_TEAMS_TEST_CREDS: ${{ secrets.MICROSOFT_TEAMS_TEST_CREDS }} MIXPANEL_INTEGRATION_TEST_CREDS: ${{ secrets.MIXPANEL_INTEGRATION_TEST_CREDS }} MSSQL_RDS_TEST_CREDS: ${{ secrets.MSSQL_RDS_TEST_CREDS }} + PAYPAL_TRANSACTION_CREDS: ${{ secrets.SOURCE_PAYPAL_TRANSACTION_CREDS }} POSTHOG_TEST_CREDS: ${{ secrets.POSTHOG_TEST_CREDS }} RECHARGE_INTEGRATION_TEST_CREDS: ${{ secrets.RECHARGE_INTEGRATION_TEST_CREDS }} QUICKBOOKS_TEST_CREDS: ${{ secrets.QUICKBOOKS_TEST_CREDS }} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d913b0f2-cc51-4e55-a44c-8ba1697b9239.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d913b0f2-cc51-4e55-a44c-8ba1697b9239.json new file mode 100644 index 000000000000..c8dddf1e2feb --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d913b0f2-cc51-4e55-a44c-8ba1697b9239.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "d913b0f2-cc51-4e55-a44c-8ba1697b9239", + "name": "Paypal Transaction", + "dockerRepository": "airbyte/source-paypal-transaction", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/paypal-transaction" +} diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 608e0e723a5a..a984a2894e27 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -358,3 +358,8 @@ dockerRepository: airbyte/source-square dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/sources/square +- sourceDefinitionId: d913b0f2-cc51-4e55-a44c-8ba1697b9239 + name: Paypal Transaction + dockerRepository: airbyte/source-paypal-transaction + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/paypal-transaction diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index a38a0e751041..7366aed40e40 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -77,6 +77,8 @@ Oracle DB [![source-oracle](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-oracle%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-oracle) + Paypal Transaction [![paypal-transaction](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-paypal-transaction%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-paypal-transaction) + Plaid [![source-plaid](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-plaid%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-plaid) Postgres [![source-postgres](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-postgres%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-postgres) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/.dockerignore b/airbyte-integrations/connectors/source-paypal-transaction/.dockerignore new file mode 100644 index 000000000000..7d3fd691a105 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_paypal_transaction +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-paypal-transaction/CHANGELOG.md b/airbyte-integrations/connectors/source-paypal-transaction/CHANGELOG.md new file mode 100644 index 000000000000..d84557504900 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/CHANGELOG.md @@ -0,0 +1,4 @@ +# Changelog + +## 0.1.0 +Source implementation with support of Transactions and Balances streams diff --git a/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile b/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile new file mode 100644 index 000000000000..ccdfa2b4a372 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.7-slim + +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +WORKDIR /airbyte/integration_code +COPY source_paypal_transaction ./source_paypal_transaction +COPY main.py ./ +COPY setup.py ./ +RUN pip install . + +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-paypal-transaction diff --git a/airbyte-integrations/connectors/source-paypal-transaction/README.md b/airbyte-integrations/connectors/source-paypal-transaction/README.md new file mode 100644 index 000000000000..4b34bbf50cfe --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/README.md @@ -0,0 +1,129 @@ +# Paypal Transaction Source + +This is the repository for the Paypal Transaction source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/paypal-transaction). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-paypal-transaction:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/paypal-transaction) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_paypal_transaction/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source paypal-transaction test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-paypal-transaction:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-paypal-transaction:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-paypal-transaction:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paypal-transaction:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paypal-transaction:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-paypal-transaction:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](source-acceptance-tests.md) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-paypal-transaction:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-paypal-transaction:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml new file mode 100644 index 000000000000..1c94f9d410d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/source-acceptance-tests.md) +# for more information about how to configure these tests +connector_image: airbyte/source-paypal-transaction:dev +tests: + spec: + - spec_path: "source_paypal_transaction/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + # Sometimes test could fail (on weekends) because transactions could temporary disappear from Paypal Sandbox account + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + validate_output_from_all_streams: yes + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + incremental: + # Only "Transactions" stream is tested here because "Balances" stream always return + # at least one message (and causes test failure) + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_transactions.json" + future_state_path: "integration_tests/abnormal_state.json" + cursor_paths: + transactions: ["date"] + diff --git a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-docker.sh new file mode 100644 index 000000000000..1425ff74f151 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-docker.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/fixture_helper.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/fixture_helper.py new file mode 100644 index 000000000000..f717706b17d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/fixture_helper.py @@ -0,0 +1,109 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +import logging +from pprint import pprint + +# %% +import requests + +logging.basicConfig(level=logging.DEBUG) + +# %% +specification = { + "client_id": "REPLACE_ME", + "secret": "REPLACE_ME", + "start_date": "2021-06-01T00:00:00+00:00", + "end_date": "2021-06-30T00:00:00+00:00", + "is_sandbox": True, +} + +# %% READ and + +client_id = specification.get("client_id") +secret = specification.get("secret") + +# %% GET API_TOKEN + +token_refresh_endpoint = "https://api-m.sandbox.paypal.com/v1/oauth2/token" +data = "grant_type=client_credentials" +headers = { + "Accept": "application/json", + "Accept-Language": "en_US", +} + +response = requests.request( + method="POST", + url=token_refresh_endpoint, + data=data, + headers=headers, + auth=(client_id, secret), +) +response_json = response.json() +print(response_json) +API_TOKEN = response_json["access_token"] + +# CREATE TRANSACTIONS +# for i in range(1000): +# create_response = requests.post( +# "https://api-m.sandbox.paypal.com/v2/checkout/orders", +# headers={'content-type': 'application/json', 'authorization': f'Bearer {API_TOKEN}', "prefer": "return=representation"}, +# json={ +# "intent": "CAPTURE", +# "purchase_units": [ +# { +# "amount": { +# "currency_code": "USD", +# "value": f"{float(i)}" +# } +# } +# ] +# } +# ) +# +# print(create_response.json()) + +# %% LIST TRANSACTIONS + +url = "https://api-m.sandbox.paypal.com/v1/reporting/transactions" + +params = { + "start_date": "2021-06-20T00:00:00+00:00", + "end_date": "2021-07-10T07:19:45Z", + "fields": "all", + "page_size": "100", + "page": "1", +} + +headers = { + "Authorization": f"Bearer {API_TOKEN}", + "Content-Type": "application/json", +} +response = requests.get( + url, + headers=headers, + params=params, +) + +pprint(response.json()) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py new file mode 100644 index 000000000000..ab90e4117fb6 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py @@ -0,0 +1,219 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +# +# REQUIREMENTS: +# 1. sudo apt-get install chromium-chromedriver +# 2. pip install selenium +# 3. ../secrets/creds.json with buyers email/password and account client_id/secret + +# HOW TO USE: +# python paypal_transaction_generator.py - will generate 3 transactions by default +# python paypal_transaction_generator.py 10 - will generate 10 transactions + +import json +import random +import sys +from time import sleep + +import requests +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait + +# from pprint import pprint + + +PAYMENT_DATA = { + "intent": "sale", + "payer": {"payment_method": "paypal"}, + "transactions": [ + { + "amount": { + "total": "30.11", + "currency": "USD", + "details": { + "subtotal": "30.00", + "tax": "0.07", + "shipping": "0.03", + "handling_fee": "1.00", + "shipping_discount": "-1.00", + "insurance": "0.01", + }, + }, + "description": "This is the payment transaction description.", + "custom": "EBAY_EMS_90048630020055", + "invoice_number": "CHAMGE_IT", + "payment_options": {"allowed_payment_method": "INSTANT_FUNDING_SOURCE"}, + "soft_descriptor": "ECHI5786755", + "item_list": { + "items": [ + { + "name": "hat", + "description": "Brown color hat", + "quantity": "5", + "price": "3", + "tax": "0.01", + "sku": "1", + "currency": "USD", + }, + { + "name": "handbag", + "description": "Black color hand bag", + "quantity": "1", + "price": "15", + "tax": "0.02", + "sku": "product34", + "currency": "USD", + }, + ], + "shipping_address": { + "recipient_name": "Hello World", + "line1": "4thFloor", + "line2": "unit#34", + "city": "SAn Jose", + "country_code": "US", + "postal_code": "95131", + "phone": "011862212345678", + "state": "CA", + }, + }, + } + ], + "note_to_payer": "Contact us for any questions on your order.", + "redirect_urls": {"return_url": "https://example.com", "cancel_url": "https://example.com"}, +} + + +def read_json(filepath): + with open(filepath, "r") as f: + return json.loads(f.read()) + + +def get_api_token(): + + client_id = CREDS.get("client_id") + secret = CREDS.get("secret") + + token_refresh_endpoint = "https://api-m.sandbox.paypal.com/v1/oauth2/token" + data = "grant_type=client_credentials" + headers = {"Accept": "application/json", "Accept-Language": "en_US"} + auth = (client_id, secret) + response = requests.request(method="POST", url=token_refresh_endpoint, data=data, headers=headers, auth=auth) + response_json = response.json() + # print(response_json) + API_TOKEN = response_json["access_token"] + return API_TOKEN + + +def random_digits(digits): + lower = 10 ** (digits - 1) + upper = 10 ** digits - 1 + return random.randint(lower, upper) + + +def make_payment(): + + # generate new invoice_number + PAYMENT_DATA["transactions"][0]["invoice_number"] = random_digits(11) + + response = requests.request( + method="POST", url="https://api-m.sandbox.paypal.com/v1/payments/payment", headers=headers, data=json.dumps(PAYMENT_DATA) + ) + response_json = response.json() + # pprint(response_json) + + execute_url = "" + approval_url = "" + + for link in response_json["links"]: + if link["rel"] == "approval_url": + approval_url = link["href"] + elif link["rel"] == "execute": + execute_url = link["href"] + elif link["rel"] == "self": + self_url = link["href"] + + print(f"Payment made: {self_url}") + return approval_url, execute_url + + +# APPROVE PAYMENT +def login(): + driver = webdriver.Chrome("/usr/bin/chromedriver") + + # SIGN_IN + driver.get("https://www.sandbox.paypal.com/ua/signin") + driver.find_element_by_id("email").send_keys(CREDS["buyer_username"]) + driver.find_element_by_id("btnNext").click() + sleep(2) + driver.find_element_by_id("password").send_keys(CREDS["buyer_password"]) + driver.find_element_by_id("btnLogin").click() + return driver + + +def approve_payment(driver, url): + driver.get(url) + global cookies_accepted + + sleep(3) + if not cookies_accepted: + cookies = driver.find_element_by_id("acceptAllButton") + if cookies: + cookies.click() + + cookies_accepted = True + driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") + + element = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "payment-submit-btn"))) + sleep(1) + element.click() + + # sleep(5) + # driver.find_element_by_id("payment-submit-btn").click() + + wait = WebDriverWait(driver, 5) + wait.until(EC.title_is("Example Domain")) + print(f"Payment approved: {driver.current_url}") + + +def execute_payment(url): + response = requests.request(method="POST", url=url, data='{"payer_id": "ZE5533HZPGMC6"}', headers=headers) + response_json = response.json() + print(f'Payment executed: {url} with STATE: {response_json["state"]}') + + +TOTAL_TRANSACTIONS = int(sys.argv[1]) if len(sys.argv) > 1 else 3 + +CREDS = read_json("../secrets/creds.json") +headers = {"Authorization": f"Bearer {get_api_token()}", "Content-Type": "application/json"} +driver = login() +cookies_accepted = False +for i in range(TOTAL_TRANSACTIONS): + print(f"Payment #{i}") + approval_url, execute_url = make_payment() + approve_payment(driver, approval_url) + execute_payment(execute_url) +driver.quit() diff --git a/airbyte-integrations/connectors/source-paypal-transaction/build.gradle b/airbyte-integrations/connectors/source-paypal-transaction/build.gradle new file mode 100644 index 000000000000..4a6226795c51 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/build.gradle @@ -0,0 +1,14 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_paypal_transaction' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/__init__.py b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..f60c258e0e01 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json @@ -0,0 +1,8 @@ +{ + "transactions": { + "date": "2021-07-11T23:00:00+00:00" + }, + "balances": { + "date": "2021-07-11T23:00:00+00:00" + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py new file mode 100644 index 000000000000..d6cbdc97c495 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py @@ -0,0 +1,34 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """ This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..32bd7a7ac9eb --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json @@ -0,0 +1,28 @@ +{ + "streams": [ + { + "stream": { + "name": "transactions", + "json_schema": {}, + "source_defined_cursor": true, + "default_cursor_field": [ + "transaction_info", + "transaction_initiation_date" + ], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "balances", + "json_schema": {}, + "default_cursor_field": ["as_of_time"], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_balances.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_balances.json new file mode 100644 index 000000000000..ca1887a40519 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_balances.json @@ -0,0 +1,13 @@ +{ + "streams": [ + { + "stream": { + "name": "balances", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json new file mode 100644 index 000000000000..cbbd4cd829e6 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json @@ -0,0 +1,18 @@ +{ + "streams": [ + { + "stream": { + "name": "transactions", + "json_schema": {}, + "source_defined_cursor": true, + "default_cursor_field": [ + "transaction_info", + "transaction_initiation_date" + ], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json new file mode 100644 index 000000000000..0b2938447d7d --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json @@ -0,0 +1,6 @@ +{ + "client_id": "AWAz___", + "secret": "ENC8__", + "start_date": "2000-06-01T05:00:00+03:00", + "is_sandbox": false +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json new file mode 100644 index 000000000000..6f5dbf626fac --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json @@ -0,0 +1,6 @@ +{ + "client_id": "PAYPAL_CLIENT_ID", + "secret": "PAYPAL_SECRET", + "start_date": "2021-06-01T00:00:00+00:00", + "is_sandbox": false +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json new file mode 100644 index 000000000000..55e16864a243 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json @@ -0,0 +1,8 @@ +{ + "transactions": { + "date": "2021-06-04T17:34:43+00:00" + }, + "balances": { + "date": "2021-06-04T17:34:43+00:00" + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json new file mode 100644 index 000000000000..dc08b6e0fe0e --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json @@ -0,0 +1,8 @@ +{ + "transactions": { + "date": "2021-06-18T16:24:13+03:00" + }, + "balances": { + "date": "2021-06-18T16:24:13+03:00" + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/main.py b/airbyte-integrations/connectors/source-paypal-transaction/main.py new file mode 100644 index 000000000000..881549ef9405 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/main.py @@ -0,0 +1,33 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_paypal_transaction import SourcePaypalTransaction + +if __name__ == "__main__": + source = SourcePaypalTransaction() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/requirements.txt b/airbyte-integrations/connectors/source-paypal-transaction/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-paypal-transaction/setup.py b/airbyte-integrations/connectors/source-paypal-transaction/setup.py new file mode 100644 index 000000000000..4a62e6df834d --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/setup.py @@ -0,0 +1,48 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "source-acceptance-test", +] + +setup( + name="source_paypal_transaction", + description="Source implementation for Paypal Transaction.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/__init__.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/__init__.py new file mode 100644 index 000000000000..6487dc4b3d4f --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/__init__.py @@ -0,0 +1,27 @@ +""" +MIT License + +Copyright (c) 2021 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .source import SourcePaypalTransaction + +__all__ = ["SourcePaypalTransaction"] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/TODO.md b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/TODO.md new file mode 100644 index 000000000000..cf1efadb3c9c --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/TODO.md @@ -0,0 +1,25 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. + +The schema of a stream is the return value of `Stream.get_json_schema`. + +## Static schemas +By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. + +Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. + +## Dynamic schemas +If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). + +## Dynamically modifying static schemas +Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: +``` +def get_json_schema(self): + schema = super().get_json_schema() + schema['dynamically_determined_property'] = "property" + return schema +``` + +Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json new file mode 100644 index 000000000000..cfed43f13e47 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json @@ -0,0 +1,59 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "properties": { + "balance": { + "type": ["null", "object"], + "properties": { + "currency": { + "type": ["null", "string"] + }, + "primary": { + "type": ["null", "boolean"] + }, + "total_balance": { + "type": ["null", "object"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + }, + "available_balance": { + "type": ["null", "object"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + }, + "withheld_balance": { + "type": ["null", "object"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + } + }, + "account_id": { + "type": ["null", "string"] + }, + "as_of_time": { + "type": "string" + }, + "last_refresh_time": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json new file mode 100644 index 000000000000..7443e216f303 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json @@ -0,0 +1,302 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "properties": { + "transaction_info": { + "type": ["null", "object"], + "properties": { + "paypal_account_id": { + "type": ["null", "string"] + }, + "transaction_id": { + "type": ["null", "string"] + }, + "transaction_event_code": { + "type": ["null", "string"] + }, + "transaction_initiation_date": { + "type": ["null", "string"] + }, + "transaction_updated_date": { + "type": ["null", "string"] + }, + "transaction_amount": { + "type": ["null", "object"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + }, + "fee_amount": { + "type": ["null", "object"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + }, + "insurance_amount": { + "type": ["null", "object"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + }, + "shipping_amount": { + "type": ["null", "object"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + }, + "shipping_discount_amount": { + "type": ["null", "object"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + }, + "transaction_status": { + "type": ["null", "string"] + }, + "transaction_subject": { + "type": ["null", "string"] + }, + "transaction_note": { + "type": ["null", "string"] + }, + "invoice_id": { + "type": ["null", "string"] + }, + "custom_field": { + "type": ["null", "string"] + }, + "protection_eligibility": { + "type": ["null", "string"] + } + } + }, + "payer_info": { + "type": ["null", "object"], + "properties": { + "account_id": { + "type": ["null", "string"] + }, + "email_address": { + "type": ["null", "string"] + }, + "address_status": { + "type": ["null", "string"] + }, + "payer_status": { + "type": ["null", "string"] + }, + "payer_name": { + "type": ["null", "object"], + "properties": { + "given_name": { + "type": ["null", "string"] + }, + "surname": { + "type": ["null", "string"] + }, + "alternate_full_name": { + "type": ["null", "string"] + } + } + }, + "country_code": { + "type": ["null", "string"] + } + } + }, + "shipping_info": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "object"], + "properties": { + "line1": { + "type": ["null", "string"] + }, + "line2": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + } + } + } + } + }, + "cart_info": { + "type": ["null", "object"], + "properties": { + "item_details": { + "type": "array", + "items": { + "type": ["null", "object"], + "properties": { + "item_code": { + "type": ["null", "string"] + }, + "item_name": { + "type": ["null", "string"] + }, + "item_description": { + "type": ["null", "string"] + }, + "item_quantity": { + "type": ["null", "string"] + }, + "item_unit_price": { + "type": ["null", "object"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + }, + "item_amount": { + "type": ["null", "object"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + }, + "tax_amounts": { + "type": "array", + "items": { + "type": ["null", "object"], + "properties": { + "tax_amount": { + "type": ["null", "object"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + } + } + }, + "total_item_amount": { + "type": ["null", "object"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + }, + "invoice_number": { + "type": ["null", "string"] + } + } + } + } + } + }, + "store_info": { + "type": ["null", "object"], + "properties": { + "store_id": { + "type": ["null", "string"] + }, + "terminal_id": { + "type": ["null", "string"] + } + } + }, + "auction_info": { + "type": ["null", "object"], + "properties": { + "auction_site": { + "type": ["null", "string"] + }, + "auction_item_site": { + "type": ["null", "string"] + }, + "auction_buyer_id": { + "type": ["null", "string"] + }, + "auction_closing_date": { + "type": ["null", "string"] + } + } + }, + "incentive_info": { + "type": ["null", "object"], + "properties": { + "incentive_details": { + "type": "array", + "items": { + "type": "object", + "properties": { + "incentive_type": { + "type": ["null", "string"] + }, + "incentive_code": { + "type": ["null", "string"] + }, + "incentive_amount": { + "type": "object", + "properties": { + "currency_code": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "incentive_program_code": { + "type": ["null", "string"] + } + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py new file mode 100644 index 000000000000..ac28b724d379 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -0,0 +1,388 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +import time +from abc import ABC +from datetime import datetime, timedelta +from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union + +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator, Oauth2Authenticator +from dateutil.parser import isoparse + + +def get_endpoint(is_sandbox: bool = False) -> str: + if is_sandbox: + endpoint = "https://api-m.sandbox.paypal.com" + else: + endpoint = "https://api-m.paypal.com" + return endpoint + + +class PaypalTransactionStream(HttpStream, ABC): + """Abstract class for Paypal Transaction Stream. + + Important note about 'start_date' params: + 'start_date' is one of required params, it comes from spec configuration or from stream state. + In both cases it must meet the following conditions: + + minimum_allowed_start_date <= start_date <= end_date <= last_refreshed_datetime <= now() + + otherwise API throws an "Data for the given start date is not available" error. + + So the prevent this error 'start_date' will be reset to: + minimum_allowed_start_date - if 'start_date' is too old + min(maximum_allowed_start_date, last_refreshed_datetime) - if 'start_date' is too recent + """ + + page_size = "500" # API limit + + # Date limits are needed to prevent API error: "Data for the given start date is not available" + # API limit: (now() - start_date_min) <= start_date <= end_date <= last_refreshed_datetime <= now + start_date_min: Mapping[str, int] = {"hours": 3 * 365} # API limit - 3 years + last_refreshed_datetime: Optional[datetime] = None # extracted from API response. Indicate the most resent possible start_date + stream_slice_period: Mapping[str, int] = {"days": 1} # max period is 31 days (API limit) + + requests_per_minute: int = 30 # API limit is 50 reqs/min from 1 IP to all endpoints, otherwise IP is banned for 5 mins + + def __init__( + self, + authenticator: HttpAuthenticator, + start_date: Union[datetime, str], + end_date: Union[datetime, str] = None, + is_sandbox: bool = False, + **kwargs, + ): + now = datetime.now().replace(microsecond=0).astimezone() + + if end_date and isinstance(end_date, str): + end_date = isoparse(end_date) + self.end_date: datetime = end_date if end_date and end_date < now else now + + if start_date and isinstance(start_date, str): + start_date = isoparse(start_date) + + minimum_allowed_start_date = now - timedelta(**self.start_date_min) + if start_date < minimum_allowed_start_date: + self.logger.log( + "WARN", + f'Stream {self.name}: start_date "{start_date.isoformat()}" is too old. ' + + f'Reset start_date to the minimum_allowed_start_date "{minimum_allowed_start_date.isoformat()}"', + ) + start_date = minimum_allowed_start_date + + self.maximum_allowed_start_date = min(now, self.end_date) + if start_date > self.maximum_allowed_start_date: + self.logger.log( + "WARN", + f'Stream {self.name}: start_date "{start_date.isoformat()}" is too recent. ' + + f'Reset start_date to the maximum_allowed_start_date "{self.maximum_allowed_start_date.isoformat()}"', + ) + start_date = self.maximum_allowed_start_date + + self.start_date = start_date + + self.is_sandbox = is_sandbox + + super().__init__(authenticator=authenticator) + + def validate_input_dates(self): + # Validate input dates + if self.start_date > self.end_date: + raise Exception(f"start_date {self.start_date.isoformat()} is greater than end_date {self.end_date.isoformat()}") + + @property + def url_base(self) -> str: + return f"{get_endpoint(self.is_sandbox)}/v1/reporting/" + + def request_headers(self, **kwargs) -> Mapping[str, Any]: + return {"Content-Type": "application/json"} + + def backoff_time(self, response: requests.Response) -> Optional[float]: + # API limit is 50 reqs/min from 1 IP to all endpoints, otherwise IP is banned for 5 mins + return 5 * 60.1 + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + json_response = response.json() + + # Save extracted last_refreshed_datetime to use it as maximum allowed start_date + last_refreshed_datetime = json_response.get("last_refreshed_datetime") + self.last_refreshed_datetime = isoparse(last_refreshed_datetime) if last_refreshed_datetime else None + + if self.data_field is not None: + data = json_response.get(self.data_field, []) + else: + data = [json_response] + + for record in data: + # In order to support direct datetime string comparison (which is performed in incremental acceptance tests) + # convert any date format to python iso format string for date based cursors + self.update_field(record, self.cursor_field, lambda date: isoparse(date).isoformat()) + yield record + + # sleep for 1-2 secs to not reach rate limit: 50 requests per minute + time.sleep(60 / self.requests_per_minute) + + @staticmethod + def update_field(record: Mapping[str, Any], field_path: Union[List[str], str], update: Callable[[Any], None]): + if not isinstance(field_path, List): + field_path = [field_path] + + last_field = field_path[-1] + data = PaypalTransactionStream.get_field(record, field_path[:-1]) + if data and last_field in data: + data[last_field] = update(data[last_field]) + + @staticmethod + def get_field(record: Mapping[str, Any], field_path: Union[List[str], str]): + + if not isinstance(field_path, List): + field_path = [field_path] + + data = record + for attr in field_path: + if data and isinstance(data, dict): + data = data.get(attr) + else: + return None + + return data + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: + # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state + # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. + latest_record_date_str: str = self.get_field(latest_record, self.cursor_field) + + if current_stream_state and "date" in current_stream_state and latest_record_date_str: + # isoparse supports different formats, like: + # python iso format: 2021-06-04T00:00:00+03:00 + # format from transactions record: 2021-06-04T00:00:00+0300 + # format from balances record: 2021-06-02T00:00:00Z + latest_record_date = isoparse(latest_record_date_str) + current_parsed_date = isoparse(current_stream_state["date"]) + + return {"date": max(current_parsed_date, latest_record_date).isoformat()} + else: + return {"date": self.start_date.isoformat()} + + def get_last_refreshed_datetime(self, sync_mode): + """Get last_refreshed_datetime attribute from API response by running PaypalTransactionStream().read_records() + with 'empty' stream_slice (range=0) + + last_refreshed_datetime indicates the maximum available start_date for which API has data. + If request start_date > last_refreshed_datetime then API throws an error: + "Data for the given start date is not available" + """ + paypal_stream = self.__class__( + authenticator=self.authenticator, + start_date=self.start_date, + end_date=self.start_date, + is_sandbox=self.is_sandbox, + ) + stream_slice = { + "start_date": self.start_date.isoformat(), + "end_date": self.start_date.isoformat(), + } + list(paypal_stream.read_records(sync_mode=sync_mode, stream_slice=stream_slice)) + return paypal_stream.last_refreshed_datetime + + def stream_slices( + self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, any]]]: + """ + Returns a list of slices for each day (by default) between the start date and end date. + The return value is a list of dicts {'start_date': date_string, 'end_date': date_string}. + """ + period = timedelta(**self.stream_slice_period) + + # get last_refreshed_datetime from API response to use as maximum allowed start_date + self.last_refreshed_datetime = self.get_last_refreshed_datetime(sync_mode) + if self.last_refreshed_datetime: + self.logger.info(f"Maximum allowed start_date is {self.last_refreshed_datetime} based on info from API response") + self.maximum_allowed_start_date = min(self.last_refreshed_datetime, self.maximum_allowed_start_date) + + slice_start_date = self.start_date + + if stream_state: + # if stream_state_date is in the future (for example during tests) then reset it to maximum_allowed_start_date: + stream_state_date = min(isoparse(stream_state.get("date")), self.maximum_allowed_start_date) + + # slice_start_date should be the most recent date: + slice_start_date = max(slice_start_date, stream_state_date) + + slices = [] + while slice_start_date <= self.maximum_allowed_start_date: + slices.append( + { + "start_date": slice_start_date.isoformat(), + "end_date": min(slice_start_date + period, self.end_date).isoformat(), + } + ) + slice_start_date += period + + return slices + + +class Transactions(PaypalTransactionStream): + """List Paypal Transactions on a specific date range + API Docs: https://developer.paypal.com/docs/integration/direct/transaction-search/#list-transactions + Endpoint: /v1/reporting/transactions + """ + + data_field = "transaction_details" + primary_key = [["transaction_info", "transaction_id"]] + cursor_field = ["transaction_info", "transaction_initiation_date"] + + # TODO handle API error when 1 request returns more than 10000 records. + # https://github.com/airbytehq/airbyte/issues/4404 + records_per_request = 10000 + + def path(self, **kwargs) -> str: + return "transactions" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + decoded_response = response.json() + total_pages = decoded_response.get("total_pages") + page_number = decoded_response.get("page") + if page_number < total_pages: + return {"page": page_number + 1} + else: + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + page_number = 1 + if next_page_token: + page_number = next_page_token.get("page") + + return { + "start_date": stream_slice["start_date"], + "end_date": stream_slice["end_date"], + "fields": "all", + "page_size": self.page_size, + "page": page_number, + } + + +class Balances(PaypalTransactionStream): + """Get account balance on a specific date + API Docs: https://developer.paypal.com/docs/integration/direct/transaction-search/#check-balancess + """ + + primary_key = "as_of_time" + cursor_field = "as_of_time" + data_field = None + + def path(self, **kwargs) -> str: + return "balances" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return { + "as_of_time": stream_slice["start_date"], + } + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + +class PayPalOauth2Authenticator(Oauth2Authenticator): + """Request example for API token extraction: + curl -v POST https://api-m.sandbox.paypal.com/v1/oauth2/token \ + -H "Accept: application/json" \ + -H "Accept-Language: en_US" \ + -u "CLIENT_ID:SECRET" \ + -d "grant_type=client_credentials" + """ + + def __init__(self, config): + super().__init__( + token_refresh_endpoint=f"{get_endpoint(config['is_sandbox'])}/v1/oauth2/token", + client_id=config["client_id"], + client_secret=config["secret"], + refresh_token="", + ) + + def get_refresh_request_body(self) -> Mapping[str, Any]: + return {"grant_type": "client_credentials"} + + def refresh_access_token(self) -> Tuple[str, int]: + """ + returns a tuple of (access_token, token_lifespan_in_seconds) + """ + try: + data = "grant_type=client_credentials" + headers = {"Accept": "application/json", "Accept-Language": "en_US"} + auth = (self.client_id, self.client_secret) + response = requests.request( + method="POST", + url=self.token_refresh_endpoint, + data=data, + headers=headers, + auth=auth, + ) + response.raise_for_status() + response_json = response.json() + return response_json["access_token"], response_json["expires_in"] + except Exception as e: + raise Exception(f"Error while refreshing access token: {e}") from e + + +class SourcePaypalTransaction(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + """ + :param config: the user-input config object conforming to the connector's spec.json + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + authenticator = PayPalOauth2Authenticator(config) + + # Try to get API TOKEN + token = authenticator.get_access_token() + if not token: + return False, "Unable to fetch Paypal API token due to incorrect client_id or secret" + + # Try to initiate a stream and validate input date params + try: + Transactions(authenticator=authenticator, **config).validate_input_dates() + except Exception as e: + return False, e + + return True, None + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + authenticator = PayPalOauth2Authenticator(config) + + return [ + Transactions(authenticator=authenticator, **config), + Balances(authenticator=authenticator, **config), + ] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json new file mode 100644 index 000000000000..694d3bb9ff6a --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json @@ -0,0 +1,36 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/paypal-transactions", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Paypal Transaction Search", + "type": "object", + "required": ["client_id", "secret", "start_date", "is_sandbox"], + "additionalProperties": true, + "properties": { + "client_id": { + "title": "Client ID", + "type": "string", + "description": "The Paypal Client ID for API credentials" + }, + "secret": { + "title": "Secret", + "type": "string", + "description": "The Secret for a given Client ID.", + "airbyte_secret": true + }, + "start_date": { + "type": "string", + "title": "Start Date", + "description": "Start Date for data extraction in ISO format. Date must be in range from 3 years till 12 hrs before present time", + "examples": ["2021-06-11T23:59:59-00:00"], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}[+-][0-9]{2}:[0-9]{2}$" + }, + "is_sandbox": { + "title": "Is Sandbox", + "description": "Whether or not to Sandbox or Production environment to extract data from", + "type": "boolean", + "default": false + } + } + } +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py new file mode 100644 index 000000000000..78a4ff3c1ce8 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py @@ -0,0 +1,267 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from datetime import datetime, timedelta + +from airbyte_cdk.sources.streams.http.auth import NoAuth +from dateutil.parser import isoparse +from source_paypal_transaction.source import Balances, PaypalTransactionStream, Transactions + + +def test_get_field(): + + record = {"a": {"b": {"c": "d"}}} + # Test expected result - field_path is a list + assert "d" == PaypalTransactionStream.get_field(record, field_path=["a", "b", "c"]) + # Test expected result - field_path is a string + assert {"b": {"c": "d"}} == PaypalTransactionStream.get_field(record, field_path="a") + + # Test failures - not existing field_path + assert None is PaypalTransactionStream.get_field(record, field_path=["a", "b", "x"]) + assert None is PaypalTransactionStream.get_field(record, field_path=["a", "x", "x"]) + assert None is PaypalTransactionStream.get_field(record, field_path=["x", "x", "x"]) + + # Test failures - incorrect record structure + record = {"a": [{"b": {"c": "d"}}]} + assert None is PaypalTransactionStream.get_field(record, field_path=["a", "b", "c"]) + + record = {"a": {"b": "c"}} + assert None is PaypalTransactionStream.get_field(record, field_path=["a", "b", "c"]) + + record = {} + assert None is PaypalTransactionStream.get_field(record, field_path=["a", "b", "c"]) + + +def test_update_field(): + # Test success 1 + record = {"a": {"b": {"c": "d"}}} + PaypalTransactionStream.update_field(record, field_path=["a", "b", "c"], update=lambda x: x.upper()) + assert record == {"a": {"b": {"c": "D"}}} + + # Test success 2 + record = {"a": {"b": {"c": "d"}}} + PaypalTransactionStream.update_field(record, field_path="a", update=lambda x: "updated") + assert record == {"a": "updated"} + + # Test failure - incorrect field_path + record = {"a": {"b": {"c": "d"}}} + PaypalTransactionStream.update_field(record, field_path=["a", "b", "x"], update=lambda x: x.upper()) + assert record == {"a": {"b": {"c": "d"}}} + + # Test failure - incorrect field_path + record = {"a": {"b": {"c": "d"}}} + PaypalTransactionStream.update_field(record, field_path=["a", "x", "x"], update=lambda x: x.upper()) + assert record == {"a": {"b": {"c": "d"}}} + + +def now(): + return datetime.now().replace(microsecond=0).astimezone() + + +def test_transactions_stream_slices(): + + start_date_max = {"hours": 0} + + # if start_date > now - **start_date_max then no slices + transactions = Transactions( + authenticator=NoAuth(), + start_date=now() - timedelta(**start_date_max) - timedelta(minutes=2), + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any") + assert 1 == len(stream_slices) + + # start_date <= now - **start_date_max + transactions = Transactions( + authenticator=NoAuth(), + start_date=now() - timedelta(**start_date_max), + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any") + assert 1 == len(stream_slices) + + transactions = Transactions( + authenticator=NoAuth(), + start_date=now() - timedelta(**start_date_max) + timedelta(minutes=2), + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any") + assert 1 == len(stream_slices) + + transactions = Transactions( + authenticator=NoAuth(), + start_date=now() - timedelta(**start_date_max) - timedelta(hours=2), + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any") + assert 1 == len(stream_slices) + + transactions = Transactions( + authenticator=NoAuth(), + start_date=now() - timedelta(**start_date_max) - timedelta(days=1), + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any") + assert 2 == len(stream_slices) + + transactions = Transactions( + authenticator=NoAuth(), + start_date=now() - timedelta(**start_date_max) - timedelta(days=1, hours=2), + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any") + assert 2 == len(stream_slices) + + transactions = Transactions( + authenticator=NoAuth(), + start_date=now() - timedelta(**start_date_max) - timedelta(days=30, minutes=1), + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any") + assert 31 == len(stream_slices) + + # tests with specified end_date + transactions = Transactions( + authenticator=NoAuth(), + start_date=isoparse("2021-06-01T10:00:00+00:00"), + end_date=isoparse("2021-06-04T12:00:00+00:00"), + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any") + assert [ + {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T10:00:00+00:00"}, + {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, + {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-04T10:00:00+00:00"}, + {"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, + ] == stream_slices + + # tests with specified end_date and stream_state + transactions = Transactions( + authenticator=NoAuth(), + start_date=isoparse("2021-06-01T10:00:00+00:00"), + end_date=isoparse("2021-06-04T12:00:00+00:00"), + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"}) + assert [ + {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, + {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-04T10:00:00+00:00"}, + {"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, + ] == stream_slices + + transactions = Transactions( + authenticator=NoAuth(), + start_date=isoparse("2021-06-01T10:00:00+00:00"), + end_date=isoparse("2021-06-04T12:00:00+00:00"), + ) + transactions.get_last_refreshed_datetime = lambda x: None + stream_slices = transactions.stream_slices(sync_mode="any", stream_state={"date": "2021-06-04T10:00:00+00:00"}) + assert [{"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}] == stream_slices + + +def test_balances_stream_slices(): + """Test slices for Balance stream. + Note that is not used by this stream. + """ + now = datetime.now().replace(microsecond=0).astimezone() + + # Test without end_date (it equal by default) + balance = Balances(authenticator=NoAuth(), start_date=now) + balance.get_last_refreshed_datetime = lambda x: None + stream_slices = balance.stream_slices(sync_mode="any") + assert 1 == len(stream_slices) + + balance = Balances(authenticator=NoAuth(), start_date=now - timedelta(minutes=1)) + balance.get_last_refreshed_datetime = lambda x: None + stream_slices = balance.stream_slices(sync_mode="any") + assert 1 == len(stream_slices) + + balance = Balances( + authenticator=NoAuth(), + start_date=now - timedelta(hours=23), + ) + balance.get_last_refreshed_datetime = lambda x: None + stream_slices = balance.stream_slices(sync_mode="any") + assert 1 == len(stream_slices) + + balance = Balances( + authenticator=NoAuth(), + start_date=now - timedelta(days=1), + ) + balance.get_last_refreshed_datetime = lambda x: None + stream_slices = balance.stream_slices(sync_mode="any") + assert 2 == len(stream_slices) + + balance = Balances( + authenticator=NoAuth(), + start_date=now - timedelta(days=1, minutes=1), + ) + balance.get_last_refreshed_datetime = lambda x: None + stream_slices = balance.stream_slices(sync_mode="any") + assert 2 == len(stream_slices) + + # test with custom end_date + balance = Balances( + authenticator=NoAuth(), + start_date=isoparse("2021-06-01T10:00:00+00:00"), + end_date=isoparse("2021-06-03T12:00:00+00:00"), + ) + balance.get_last_refreshed_datetime = lambda x: None + stream_slices = balance.stream_slices(sync_mode="any") + assert [ + {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T10:00:00+00:00"}, + {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, + {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}, + ] == stream_slices + + # Test with stream state + balance = Balances( + authenticator=NoAuth(), + start_date=isoparse("2021-06-01T10:00:00+00:00"), + end_date=isoparse("2021-06-03T12:00:00+00:00"), + ) + balance.get_last_refreshed_datetime = lambda x: None + stream_slices = balance.stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"}) + assert [ + {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, + {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}, + ] == stream_slices + + balance = Balances( + authenticator=NoAuth(), + start_date=isoparse("2021-06-01T10:00:00+00:00"), + end_date=isoparse("2021-06-03T12:00:00+00:00"), + ) + balance.get_last_refreshed_datetime = lambda x: None + stream_slices = balance.stream_slices(sync_mode="any", stream_state={"date": "2021-06-03T11:00:00+00:00"}) + assert [{"start_date": "2021-06-03T11:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}] == stream_slices + + balance = Balances( + authenticator=NoAuth(), + start_date=isoparse("2021-06-01T10:00:00+00:00"), + end_date=isoparse("2021-06-03T12:00:00+00:00"), + ) + balance.get_last_refreshed_datetime = lambda x: None + stream_slices = balance.stream_slices(sync_mode="any", stream_state={"date": "2021-06-03T12:00:00+00:00"}) + assert [{"start_date": "2021-06-03T12:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}] == stream_slices diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index b8d1d021b20d..a36bab1efc37 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -73,6 +73,7 @@ * [MySQL](integrations/sources/mysql.md) * [Okta](integrations/sources/okta.md) * [Oracle DB](integrations/sources/oracle.md) + * [Paypal Transaction](integrations/sources/paypal-transaction.md) * [Plaid](integrations/sources/plaid.md) * [PokéAPI](integrations/sources/pokeapi.md) * [Postgres](integrations/sources/postgres.md) diff --git a/docs/integrations/sources/paypal-transaction.md b/docs/integrations/sources/paypal-transaction.md new file mode 100644 index 000000000000..be0ddda01807 --- /dev/null +++ b/docs/integrations/sources/paypal-transaction.md @@ -0,0 +1,63 @@ +# Paypal Transaction API + +## Overview + +The [Paypal Transaction API](https://developer.paypal.com/docs/api/transaction-search/v1/). is used to get the history of transactions for a PayPal account. + + +#### Output schema + +This Source is capable of syncing the following core Streams: + +* [Transactions](https://developer.paypal.com/docs/api/transaction-search/v1/#transactions) +* [Balances](https://developer.paypal.com/docs/api/transaction-search/v1/#balances) + +#### Data type mapping + +| Integration Type | Airbyte Type | Notes | +| :--- | :--- | :--- | +| `string` | `string` | | +| `number` | `number` | | +| `array` | `array` | | +| `object` | `object` | | + +#### Features + +| Feature | Supported? | +| :--- | :--- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | Yes | +| Namespaces | No | + + +### Getting started + +### Requirements + +* client_id. +* secret. +* is_sandbox. + +### Setup guide + +In order to get an `Client ID` and `Secret` please go to [this](https://developer.paypal.com/docs/platforms/get-started/ page and follow the instructions. After registration you may find your `Client ID` and `Secret` [here](https://developer.paypal.com/developer/accounts/). + + +## Performance considerations + +Paypal transaction API has some [limits](https://developer.paypal.com/docs/integration/direct/transaction-search/) +- `start_date_min` = 3 years, API call lists transaction for the previous three years. +- `start_date_max` = 1.5 days, it takes a maximum of three hours for executed transactions to appear in the list transactions call. It is set to 1.5 days by default based on experience, otherwise API throw an error. +- `stream_slice_period` = 1 day, the maximum supported date range is 31 days. +- `records_per_request` = 10000, the maximum number of records in a single request. +- `page_size` = 500, the maximum page size is 500. +- `requests_per_minute` = 30, maximum limit is 50 requests per minute from IP address to all endpoint + +Transactions sync is performed with default `stream_slice_period` = 1 day, it means that there will be 1 request for each day between start_date and now (or end_date). if `start_date` is greater then `start_date_max`. +Balances sync is similarly performed with default `stream_slice_period` = 1 day, but it will do additional request for the end_date of the sync (now). + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :-------- | :----- | :------ | +| 0.1.0 | 2021-06-10 | [4240](https://github.com/airbytehq/airbyte/pull/4240) | PayPal Transaction Search API | diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index a81384d060f2..56f58c9a0e4d 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -73,6 +73,7 @@ write_standard_creds source-mixpanel-singer "$MIXPANEL_INTEGRATION_TEST_CREDS" write_standard_creds source-mssql "$MSSQL_RDS_TEST_CREDS" write_standard_creds source-okta "$SOURCE_OKTA_TEST_CREDS" write_standard_creds source-plaid "$PLAID_INTEGRATION_TEST_CREDS" +write_standard_creds source-paypal-transaction "$PAYPAL_TRANSACTION_CREDS" write_standard_creds source-posthog "$POSTHOG_TEST_CREDS" write_standard_creds source-quickbooks-singer "$QUICKBOOKS_TEST_CREDS" write_standard_creds source-recharge "$RECHARGE_INTEGRATION_TEST_CREDS"