From 08106a209b9b78908e2ec2c1c394f88163497710 Mon Sep 17 00:00:00 2001 From: sarafonseca-123 Date: Sun, 23 Oct 2022 21:36:29 -0300 Subject: [PATCH 01/17] Add new source: RD Station Marketing --- airbyte-integrations/builds.md | 1 + .../source-rd-station-marketing/.dockerignore | 6 + .../source-rd-station-marketing/Dockerfile | 38 ++++ .../source-rd-station-marketing/README.md | 133 ++++++++++++ .../acceptance-test-config.yml | 22 ++ .../acceptance-test-docker.sh | 16 ++ .../source-rd-station-marketing/bootstrap.md | 29 +++ .../source-rd-station-marketing/build.gradle | 9 + .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 14 ++ .../integration_tests/acceptance.py | 14 ++ .../integration_tests/configured_catalog.json | 20 ++ .../integration_tests/invalid_config.json | 11 + .../integration_tests/sample_config.json | 11 + .../integration_tests/sample_state.json | 14 ++ .../source-rd-station-marketing/main.py | 13 ++ .../requirements.txt | 2 + .../source-rd-station-marketing/setup.py | 31 +++ .../source_rd_station_marketing/__init__.py | 8 + .../schemas/analytics_conversions.json | 30 +++ .../schemas/analytics_emails.json | 51 +++++ .../schemas/analytics_funnel.json | 24 +++ .../analytics_workflow_emails_statistics.json | 63 ++++++ .../schemas/emails.json | 56 +++++ .../schemas/embeddables.json | 24 +++ .../schemas/fields.json | 64 ++++++ .../schemas/landing_pages.json | 30 +++ .../schemas/popups.json | 27 +++ .../schemas/segmentations.json | 44 ++++ .../schemas/workflows.json | 24 +++ .../source_rd_station_marketing/source.py | 76 +++++++ .../source_rd_station_marketing/spec.json | 85 ++++++++ .../source_rd_station_marketing/streams.py | 203 ++++++++++++++++++ .../unit_tests/__init__.py | 3 + .../unit_tests/test_incremental_streams.py | 59 +++++ .../unit_tests/test_source.py | 67 ++++++ .../unit_tests/test_streams.py | 106 +++++++++ docs/integrations/README.md | 1 + .../sources/rd-station-marketing.md | 44 ++++ 39 files changed, 1476 insertions(+) create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/.dockerignore create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/Dockerfile create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/README.md create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/bootstrap.md create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/build.gradle create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/main.py create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/requirements.txt create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/setup.py create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/__init__.py create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_conversions.json create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_emails.json create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_funnel.json create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_workflow_emails_statistics.json create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/emails.json create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/embeddables.json create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/fields.json create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/landing_pages.json create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/popups.json create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/segmentations.json create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/workflows.json create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/source.py create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/spec.json create mode 100755 airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_incremental_streams.py create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_streams.py create mode 100644 docs/integrations/sources/rd-station-marketing.md diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 9d72e65aa1f1..c6c21922ba89 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -94,6 +94,7 @@ | Confluence | [![source-confluence](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-confluence%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-confluence) | | Qualaroo | [![source-qualaroo](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-qualaroo%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-qualaroo) | | QuickBooks | [![source-quickbooks-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-quickbooks-singer%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-quickbooks-singer) | +| RD Station Marketing | [![source-rd-station-marketing](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-rd-station-marketing%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-rd-station-marketing) | | Recharge | [![source-recharge](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-recharge%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-recharge) | | Recurly | [![source-recurly](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-recurly%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-recurly) | | Redshift | [![source-redshift](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-redshift%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-redshift) | diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/.dockerignore b/airbyte-integrations/connectors/source-rd-station-marketing/.dockerignore new file mode 100644 index 000000000000..1ee8b479485f --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_rd_station_marketing +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/Dockerfile b/airbyte-integrations/connectors/source-rd-station-marketing/Dockerfile new file mode 100644 index 000000000000..327eabb2010a --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_rd_station_marketing ./source_rd_station_marketing + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-rd-station-marketing diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/README.md b/airbyte-integrations/connectors/source-rd-station-marketing/README.md new file mode 100644 index 000000000000..8337e8a1f952 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/README.md @@ -0,0 +1,133 @@ +# RD Station Marketing Source + +This is the repository for the RD Station Marketing source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/rd-station-marketing). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python3 -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-rd-station-marketing:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/rd-station-marketing) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_rd_station_marketing/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source rd-station test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-rd-station-marketing:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-rd-station-marketing:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-rd-station-marketing:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-rd-station-marketing:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-rd-station-marketing:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-rd-station-marketing:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). + +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-rd-station-marketing:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-rd-station-marketing:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-rd-station-marketing/acceptance-test-config.yml new file mode 100644 index 000000000000..7315fa50eda0 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/acceptance-test-config.yml @@ -0,0 +1,22 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-rd-station-marketing:dev +tests: + spec: + - spec_path: "source_rd_station_marketing/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + timeout_seconds: 3600 + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 3600 diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-rd-station-marketing/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/bootstrap.md b/airbyte-integrations/connectors/source-rd-station-marketing/bootstrap.md new file mode 100644 index 000000000000..fa71c6f5ca39 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/bootstrap.md @@ -0,0 +1,29 @@ +# RD Station Marketing + +## Overview + +RD Station Marketing is the leading Marketing Automation tool in Latin America. It is a software application that helps your company carry out better campaigns, nurture Leads, generate qualified business opportunities and achieve more results. From social media to email, Landing Pages, Pop-ups, even Automations and Analytics. + +## Authentication + +RD Station Marketing uses Oauth2 to authenticate. To get the credentials, you need first to create an App for private use in this [link](https://appstore.rdstation.com/en/publisher) (needs to be loged in to access). After that, follow [these](https://developers.rdstation.com/reference/autenticacao?lng=en) instructions to create the client_id and client_secret. + +## Endpoints + +There are eleven endpoints in RD Station Marketing Connector: + +- [Analytics Conversions](https://developers.rdstation.com/reference/get_platform-analytics-conversions?lng=en): Responds with conversion statistics for campaings and other marketing assets. +- [Analytics Emails](https://developers.rdstation.com/reference/get_platform-analytics-emails?lng=en): Responds with statistics about the emails sent with this tool. +- [Analytics Funnel](https://developers.rdstation.com/reference/get_platform-analytics-funnel): Responds with the sales funnel for a given period, grouped by day. +- [Analytics Workflow Emails Statistics](https://developers.rdstation.com/reference/get_platform-analytics-workflow-emails): Responds with statistics about emails sent via an automation flow. +- [Emails](https://developers.rdstation.com/reference/get_platform-emails): List all sent emails. +- [Embeddables](https://developers.rdstation.com/reference/get_platform-embeddables): Returns a list of all forms for an account. +- [Fields](https://developers.rdstation.com/reference/get_platform-contacts-fields): Returns all fields, customized and default, and its attributes. +- [Landing Pages](https://developers.rdstation.com/reference/get_platform-landing-pages): Returns a list of all landing pages for an account. +- [Pop-ups](https://developers.rdstation.com/reference/get_platform-popups): Returns a list of all pop-ups for an account. +- [Segmentations](https://developers.rdstation.com/reference/get_platform-segmentations): List all segmentations, custom and default. +- [Workflows](https://developers.rdstation.com/reference/get_platform-workflows): Returns all automation flows. + +## Quick Notes + +- The analytics streams are only supported if you have a Pro or Enterprise RD Station Account. The usage is available only to these plans. diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/build.gradle b/airbyte-integrations/connectors/source-rd-station-marketing/build.gradle new file mode 100644 index 000000000000..171dd1799b82 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_rd_station_marketing' +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/__init__.py b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..0f178109f511 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/abnormal_state.json @@ -0,0 +1,14 @@ +{ + "analytics_emails": { + "send_at": "2217-06-26 21:20:07" + }, + "analytics_funnel": { + "reference_day": "2217-06-26 21:20:07" + }, + "analytics_conversions": { + "asset_updated_at": "2217-06-26 21:20:07" + }, + "analytics_workflow_emails_statistics": { + "asset_updated_at": "2217-06-26 21:20:07" + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..934f1a21f676 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/configured_catalog.json @@ -0,0 +1,20 @@ +{ + "streams": [ + { + "stream": { + "name": "analytics_emails", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "send_at" + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/invalid_config.json new file mode 100644 index 000000000000..178618cff3b3 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/invalid_config.json @@ -0,0 +1,11 @@ +{ + "authorization": + { + "auth_type": "Client", + "client_id": "fake-client-id", + "client_secret": "fake-client-secret", + "refresh_token": "fake-refresh-token" + }, + "replication_start_date": "2022-01-01T00:00:00Z", + "all_contacts_segmentation": "9999999999999" +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/sample_config.json new file mode 100644 index 000000000000..5149ad122f2a --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/sample_config.json @@ -0,0 +1,11 @@ +{ + "authorization": + { + "auth_type": "Client", + "client_id": "", + "client_secret": "", + "refresh_token": "" + }, + "replication_start_date": "2022-01-01T00:00:00Z", + "all_contacts_segmentation": "2050455" +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/sample_state.json new file mode 100644 index 000000000000..e72298059ea4 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/sample_state.json @@ -0,0 +1,14 @@ +{ + "analytics_emails": { + "send_at": "2022-06-26 21:20:07" + }, + "analytics_funnel": { + "reference_day": "2022-06-26 21:20:07" + }, + "analytics_conversions": { + "asset_updated_at": "2022-06-26 21:20:07" + }, + "analytics_workflow_emails_statistics": { + "updated_at": "2022-06-26 21:20:07" + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/main.py b/airbyte-integrations/connectors/source-rd-station-marketing/main.py new file mode 100644 index 000000000000..3e89331aff62 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_rd_station_marketing import SourceRDStationMarketing + +if __name__ == "__main__": + source = SourceRDStationMarketing() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/requirements.txt b/airbyte-integrations/connectors/source-rd-station-marketing/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/setup.py b/airbyte-integrations/connectors/source-rd-station-marketing/setup.py new file mode 100644 index 000000000000..095ffed27c16 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/setup.py @@ -0,0 +1,31 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1.56", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", + "responses~=0.13.3", + "requests-mock", +] + +setup( + name="source_rd_station_marketing", + description="Source implementation for Rd Station Marketing.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/__init__.py b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/__init__.py new file mode 100644 index 000000000000..c548f26f7235 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceRDStationMarketing + +__all__ = ["SourceRDStationMarketing"] diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_conversions.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_conversions.json new file mode 100644 index 000000000000..b285d069ca0d --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_conversions.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "asset_id": { + "type": ["null", "integer"] + }, + "asset_identifier": { + "type": ["null", "string"] + }, + "asset_created_at": { + "type": ["null", "string"] + }, + "asset_updated_at": { + "type": ["null", "string"] + }, + "asset_type": { + "type": ["null", "string"] + }, + "conversion_count": { + "type": ["null", "integer"] + }, + "visits_count": { + "type": ["null", "integer"] + }, + "conversion_rate": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_emails.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_emails.json new file mode 100644 index 000000000000..6b74fb2db79b --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_emails.json @@ -0,0 +1,51 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "send_at": { + "type": ["null", "string"] + }, + "campaign_id": { + "type": ["null", "integer"] + }, + "campaign_name": { + "type": ["null", "string"] + }, + "email_dropped_count": { + "type": ["null", "integer"] + }, + "email_delivered_count": { + "type": ["null", "integer"] + }, + "email_bounced_count": { + "type": ["null", "integer"] + }, + "email_opened_count": { + "type": ["null", "integer"] + }, + "email_clicked_count": { + "type": ["null", "integer"] + }, + "email_unsubscribed_count": { + "type": ["null", "integer"] + }, + "email_spam_reported_count": { + "type": ["null", "integer"] + }, + "email_delivered_rate": { + "type": ["null", "number"] + }, + "email_opened_rate": { + "type": ["null", "number"] + }, + "email_clicked_rate": { + "type": ["null", "number"] + }, + "email_spam_reported_rate": { + "type": ["null", "number"] + }, + "contacts_count": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_funnel.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_funnel.json new file mode 100644 index 000000000000..8f144ccfa7de --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_funnel.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "reference_day": { + "type": ["null", "string"] + }, + "contacts_count": { + "type": ["null", "integer"] + }, + "qualified_contacts_count": { + "type": ["null", "integer"] + }, + "opportunities_count": { + "type": ["null", "integer"] + }, + "sales_count": { + "type": ["null", "integer"] + }, + "visitors_count": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_workflow_emails_statistics.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_workflow_emails_statistics.json new file mode 100644 index 000000000000..ffe7866f912d --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/analytics_workflow_emails_statistics.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "workflow_name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "email_name": { + "type": ["null", "string"] + }, + "workflow_action_id": { + "type": ["null", "string"] + }, + "workflow_id": { + "type": ["null", "string"] + }, + "contacts_count": { + "type": ["null", "integer"] + }, + "count_processed": { + "type": ["null", "integer"] + }, + "email_delivered_count": { + "type": ["null", "integer"] + }, + "email_opened_unique_count": { + "type": ["null", "integer"] + }, + "email_clicked_unique_count": { + "type": ["null", "integer"] + }, + "email_dropped_count": { + "type": ["null", "integer"] + }, + "email_unsubscribed_count": { + "type": ["null", "integer"] + }, + "email_spam_reported_count": { + "type": ["null", "integer"] + }, + "email_delivered_rate": { + "type": ["null", "number"] + }, + "email_opened_rate": { + "type": ["null", "number"] + }, + "email_clicked_rate": { + "type": ["null", "number"] + }, + "email_spam_reported_rate": { + "type": ["null", "number"] + }, + "email_bounced_unique_count": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/emails.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/emails.json new file mode 100644 index 000000000000..d69b22ee0d19 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/emails.json @@ -0,0 +1,56 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "campaign_id": { + "type": ["null", "integer"] + }, + "behavior_score_info": { + "type": ["null", "object"], + "properties": { + "engaged": { + "type": ["null", "boolean"] + }, + "disengaged": { + "type": ["null", "boolean"] + }, + "indeterminate": { + "type": ["null", "boolean"] + } + } + }, + "send_at": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "sending_is_imminent": { + "type": ["null", "boolean"] + }, + "is_predictive_sending": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "component_template_id": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "leads_count": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/embeddables.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/embeddables.json new file mode 100644 index 000000000000..052bb931c79c --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/embeddables.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "conversion_identifier": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/fields.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/fields.json new file mode 100644 index 000000000000..d740af4b0d0c --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/fields.json @@ -0,0 +1,64 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "uuid": { + "type": ["null", "string"] + }, + "label": { + "type": ["null", "object"], + "properties": { + "en-UD": { + "type": ["null", "string"] + }, + "en-US": { + "type": ["null", "string"] + }, + "es-ES": { + "type": ["null", "string"] + }, + "pt-BR": { + "type": ["null", "string"] + }, + "default": { + "type": ["null", "string"] + } + } + }, + "name": { + "type": ["null", "object"], + "properties": { + "en-UD": { + "type": ["null", "string"] + }, + "en-US": { + "type": ["null", "string"] + }, + "es-ES": { + "type": ["null", "string"] + }, + "pt-BR": { + "type": ["null", "string"] + }, + "default": { + "type": ["null", "string"] + } + } + }, + "api_identifier": { + "type": ["null", "string"] + }, + "custom_field": { + "type": ["null", "boolean"] + }, + "validation_rules": { + "type": ["null", "object"] + }, + "presentation_type": { + "type": ["null", "string"] + }, + "data_type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/landing_pages.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/landing_pages.json new file mode 100644 index 000000000000..dc8e55175ab1 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/landing_pages.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "conversion_identifier": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "has_active_experiment": { + "type": ["null", "boolean"] + }, + "had_experiment": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/popups.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/popups.json new file mode 100644 index 000000000000..250310cc7748 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/popups.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "conversion_identifier": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "trigger": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/segmentations.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/segmentations.json new file mode 100644 index 000000000000..04a2d97658fb --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/segmentations.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "standard": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "process_status": { + "type": ["null", "string"] + }, + "links": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "rel": { + "type": ["null", "string"] + }, + "href": { + "type": ["null", "string"] + }, + "media": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/workflows.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/workflows.json new file mode 100644 index 000000000000..c9bd95a32671 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/schemas/workflows.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "user_email_created": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "user_email_updated": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/source.py b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/source.py new file mode 100644 index 000000000000..d6fc73ffc289 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/source.py @@ -0,0 +1,76 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from typing import Any, List, Mapping, Tuple + +import pendulum +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator +from source_rd_station_marketing.streams import ( + AnalyticsConversions, + AnalyticsEmails, + AnalyticsFunnel, + AnalyticsWorkflowEmailsStatistics, + Emails, + Embeddables, + Fields, + LandingPages, + Popups, + Segmentations, + Workflows, +) + + +class SourceRDStationMarketing(AbstractSource): + def check_connection( + self, logger: AirbyteLogger, config: Mapping[str, Any] + ) -> Tuple[bool, Any]: + try: + stream_kwargs = self.get_stream_kwargs(config) + segmentations = Segmentations(**stream_kwargs) + segmentations_gen = segmentations.read_records(sync_mode=SyncMode.full_refresh) + next(segmentations_gen) + return True, None + except Exception as error: + return ( + False, + f"Unable to connect to RD Station Marketing API with the provided credentials - {repr(error)}", + ) + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + stream_kwargs = self.get_stream_kwargs(config) + incremental_kwargs = {**stream_kwargs, "replication_start_date": pendulum.parse(config["replication_start_date"])} + streams = [ + AnalyticsEmails(**incremental_kwargs), + AnalyticsConversions(**incremental_kwargs), + AnalyticsFunnel(**incremental_kwargs), + AnalyticsWorkflowEmailsStatistics(**incremental_kwargs), + Emails(**stream_kwargs), + Embeddables(**stream_kwargs), + Fields(**stream_kwargs), + LandingPages(**stream_kwargs), + Popups(**stream_kwargs), + Segmentations(**stream_kwargs), + Workflows(**stream_kwargs) + ] + return streams + + @staticmethod + def get_stream_kwargs(config: Mapping[str, Any]) -> Mapping[str, Any]: + authorization = config.get("authorization", {}) + stream_kwargs = dict() + + stream_kwargs["authenticator"] = Oauth2Authenticator( + token_refresh_endpoint="https://api.rd.services/auth/token", + client_secret=authorization.get("client_secret"), + client_id=authorization.get("client_id"), + refresh_token=authorization.get("refresh_token"), + ) + return stream_kwargs diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/spec.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/spec.json new file mode 100644 index 000000000000..d75d2eca144c --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/spec.json @@ -0,0 +1,85 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/rd-station-marketing", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "RD Station Marketing Spec", + "type": "object", + "required": [ + "replication_start_date" + ], + "additionalProperties": true, + "properties": { + "authorization": { + "type": "object", + "title": "Authentication Type", + "description": "Choose one of the possible authorization method", + "oneOf": [ + { + "title": "Sign in via RD Station (OAuth)", + "type": "object", + "required": [ + "auth_type" + ], + "properties": { + "auth_type": { + "type": "string", + "const": "Client", + "order": 0 + }, + "client_id": { + "title": "Client ID", + "type": "string", + "description": "The Client ID of your RD Station developer application.", + "airbyte_secret": true + }, + "client_secret": { + "title": "Client Secret", + "type": "string", + "description": "The Client Secret of your RD Station developer application", + "airbyte_secret": true + }, + "refresh_token": { + "title": "Refresh Token", + "type": "string", + "description": "The token for obtaining the new access token.", + "airbyte_secret": true + } + } + } + ] + }, + "replication_start_date": { + "title": "Start Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. When specified and not None, then stream will behave as incremental", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "examples": [ + "2017-01-25T00:00:00Z" + ], + "type": "string" + } + } + }, + "supportsIncremental": true, + "authSpecification": { + "auth_type": "oauth2.0", + "oauth2Specification": { + "rootObject": [ + "authorization", + 0 + ], + "oauthFlowInitParameters": [ + [ + "client_id" + ], + [ + "client_secret" + ] + ], + "oauthFlowOutputParameters": [ + [ + "refresh_token" + ] + ] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py new file mode 100755 index 000000000000..ba254fc8da0c --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py @@ -0,0 +1,203 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# +from abc import ABC +from datetime import date +from typing import Any, Iterable, Mapping, MutableMapping, Optional + +import pendulum +import requests +from airbyte_cdk.sources.streams.http import HttpStream + + +class RDStationMarketingStream(HttpStream, ABC): + url_base = "https://api.rd.services" + primary_key = None + page = 2 + page_size=125 + extra_params = {} + data_field = None + + def __init__(self, authenticator, replication_start_date=None, **kwargs): + super().__init__(authenticator=authenticator, **kwargs) + self._replication_start_date = replication_start_date + + def path(self, **kwargs) -> str: + class_name = self.__class__.__name__ + return f"/platform/{class_name[0].lower()}{class_name[1:]}" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + if self.data_field: + json_response = response.json().get(self.data_field) + else: + json_response = response.json() + if json_response: + page_params = dict(page=self.page) + self.page = self.page + 1 + return page_params + else: + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + params = {"page_size": self.page_size} + if next_page_token: + params.update(**next_page_token) + return params + + def parse_response( + self, + response: requests.Response, + **kwargs + ) -> Iterable[Mapping]: + if self.data_field: + records = response.json().get(self.data_field) + else: + records = response.json() + for record in records: + yield record + + +class IncrementalRDStationMarketingStream(RDStationMarketingStream): + def path(self, **kwargs) -> str: + return f"/platform/analytics/{self.data_field}" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def request_params( + self, + stream_state: Mapping[str, Any], + **kwargs + ) -> MutableMapping[str, Any]: + replication_start_date = self._replication_start_date + + if replication_start_date: + if stream_state.get(self.cursor_field): + replication_start_date = max(pendulum.parse(stream_state[self.cursor_field]), replication_start_date) + + params = {} + params.update( + {"start_date": replication_start_date.strftime("%Y-%m-%d"), + "end_date": date.today().strftime("%Y-%m-%d"), + }) + + params.update(self.extra_params) + return params + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + latest_benchmark = latest_record[self.cursor_field] + if current_stream_state.get(self.cursor_field): + return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} + return {self.cursor_field: latest_benchmark} + + +class AnalyticsConversions(IncrementalRDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-analytics-conversions + """ + data_field = "conversions" + cursor_field = "asset_updated_at" + primary_key = "asset_id" + + def parse_response( + self, + response: requests.Response, + **kwargs + ) -> Iterable[Mapping]: + records = response.json().get(self.data_field)[0].get(self.data_field) + for record in records: + yield record + + +class AnalyticsEmails(IncrementalRDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-analytics-emails + """ + data_field = "emails" + cursor_field = "send_at" + primary_key = "campaign_id" + + +class AnalyticsFunnel(IncrementalRDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-analytics-funnel + """ + data_field = "funnel" + cursor_field = "reference_day" + primary_key = "reference_day" + + +class AnalyticsWorkflowEmailsStatistics(IncrementalRDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-analytics-workflow-emails + """ + data_field = "workflow_email_statistics" + cursor_field = "updated_at" + primary_key = "workflow_id" + + def path(self, **kwargs) -> str: + return f"/platform/analytics/workflow_emails_statistics" + + +class Emails(RDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-emails + """ + data_field = "items" + primary_key = "id" + + +class Embeddables(RDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-embeddables + """ + primary_key = "id" + + +class Fields(RDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-contacts-fields + """ + data_field = "fields" + primary_key = "uuid" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def path(self, **kwargs) -> str: + return f"/platform/contacts/fields" + + +class LandingPages(RDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-landing-pages + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "/platform/landing_pages" + + +class Popups(RDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-popups + """ + primary_key = "id" + + +class Segmentations(RDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-segmentations + """ + data_field = "segmentations" + primary_key = "id" + + +class Workflows(RDStationMarketingStream): + """ + API docs: https://developers.rdstation.com/reference/get_platform-workflows + """ + data_field = "workflows" + primary_key = "id" diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/__init__.py b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_incremental_streams.py new file mode 100644 index 000000000000..7e37cde8f5fe --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_incremental_streams.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_rd_station_marketing.streams import IncrementalRDStationMarketingStream + + +@fixture +def test_current_stream_state(): + return {"updated_time": "2021-10-22"} + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(IncrementalRDStationMarketingStream, "path", "v0/example_endpoint") + mocker.patch.object(IncrementalRDStationMarketingStream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalRDStationMarketingStream, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + stream = IncrementalRDStationMarketingStream(authenticator=MagicMock()) + expected_cursor_field = [] + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_base_class, test_current_stream_state, mocker): + mocker.patch.object(IncrementalRDStationMarketingStream, "cursor_field", "updated_time") + stream = IncrementalRDStationMarketingStream(authenticator=MagicMock()) + inputs = {"current_stream_state": test_current_stream_state, "latest_record": test_current_stream_state} + expected_state = {"updated_time": "2021-10-22"} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_stream_slices(patch_incremental_base_class): + stream = IncrementalRDStationMarketingStream(authenticator=MagicMock()) + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + expected_stream_slice = [None] + assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + mocker.patch.object(IncrementalRDStationMarketingStream, "cursor_field", "dummy_field") + stream = IncrementalRDStationMarketingStream(authenticator=MagicMock()) + assert stream.supports_incremental + + +def test_source_defined_cursor(patch_incremental_base_class): + stream = IncrementalRDStationMarketingStream(authenticator=MagicMock()) + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(patch_incremental_base_class): + stream = IncrementalRDStationMarketingStream(authenticator=MagicMock()) + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_source.py new file mode 100644 index 000000000000..cbf8de2f6e8f --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_source.py @@ -0,0 +1,67 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +import responses +from pytest import fixture +from source_rd_station_marketing.source import SourceRDStationMarketing + + +@fixture +def test_config(): + return {"authorization": { + "auth_type": "Client", + "client_id": "test_client_id", + "client_secret": "test_client_secret", + "refresh_token": "test_refresh_token", + }, + "replication_start_date": "2022-01-01T00:00:00Z" + } + +def setup_responses(): + responses.add( + responses.POST, + "https://api.rd.services/auth/token", + json={"access_token": "fake_access_token", "expires_in": 3600}, + ) + responses.add( + responses.GET, + "https://api.rd.services/platform/segmentations", + json={"segmentations": [ + { + "id": 71625167165, + "name": "A mock segmentation", + "standard": True, + "created_at": "2019-09-04T18:05:42.638-03:00", + "updated_at": "2019-09-04T18:05:42.638-03:00", + "process_status": "processed", + "links": [ + { + "rel": "SEGMENTATIONS.CONTACTS", + "href": "https://api.rd.services/platform/segmentations/71625167165/contacts", + "media": "application/json", + "type": "GET" + } + ] + } + ]} + ) + + +@responses.activate +def test_check_connection(test_config): + setup_responses() + source = SourceRDStationMarketing() + logger_mock = MagicMock() + assert source.check_connection(logger_mock, test_config) == (True, None) + + +@responses.activate +def test_streams(test_config): + setup_responses() + source = SourceRDStationMarketing() + streams = source.streams(test_config) + expected_streams_number = 11 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_streams.py new file mode 100644 index 000000000000..8b50776a475a --- /dev/null +++ b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_streams.py @@ -0,0 +1,106 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_rd_station_marketing.streams import RDStationMarketingStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(RDStationMarketingStream, "path", "v0/example_endpoint") + mocker.patch.object(RDStationMarketingStream, "primary_key", "test_primary_key") + mocker.patch.object(RDStationMarketingStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = RDStationMarketingStream(authenticator=MagicMock()) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {'page_size': 125} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = RDStationMarketingStream(authenticator=MagicMock()) + inputs = {"response": MagicMock()} + expected_token = {'page': 2} + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = RDStationMarketingStream(authenticator=MagicMock()) + response = MagicMock() + response.json.return_value = [{ + "id": 71625167165, + "name": "A mock segmentation", + "standard": True, + "created_at": "2019-09-04T18:05:42.638-03:00", + "updated_at": "2019-09-04T18:05:42.638-03:00", + "process_status": "processed", + "links": [ + { + "rel": "SEGMENTATIONS.CONTACTS", + "href": "https://api.rd.services/platform/segmentations/71625167165/contacts", + "media": "application/json", + "type": "GET" + } + ] + }] + inputs = {"response": response, "stream_state": None} + expected_parsed_object = { + "id": 71625167165, + "name": "A mock segmentation", + "standard": True, + "created_at": "2019-09-04T18:05:42.638-03:00", + "updated_at": "2019-09-04T18:05:42.638-03:00", + "process_status": "processed", + "links": [ + { + "rel": "SEGMENTATIONS.CONTACTS", + "href": "https://api.rd.services/platform/segmentations/71625167165/contacts", + "media": "application/json", + "type": "GET" + } + ] + } + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = RDStationMarketingStream(authenticator=MagicMock()) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = RDStationMarketingStream(authenticator=MagicMock()) + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = RDStationMarketingStream(authenticator=MagicMock()) + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = RDStationMarketingStream(authenticator=MagicMock()) + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 78dfeb98f0b1..1c344b98da32 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -134,6 +134,7 @@ For more information about the grading system, see [Product Release Stages](http | [PrestaShop](sources/presta-shop.md) | Alpha | Yes | | [Qualaroo](sources/qualaroo.md) | Alpha | Yes | | [QuickBooks](sources/quickbooks.md) | Alpha | No | +| [RD Station Marketing](sources/rd-station-marketing.md) | Alpha | No | | [Recharge](sources/recharge.md) | Beta | Yes | | [Recurly](sources/recurly.md) | Alpha | Yes | | [Redshift](sources/redshift.md) | Alpha | Yes | diff --git a/docs/integrations/sources/rd-station-marketing.md b/docs/integrations/sources/rd-station-marketing.md new file mode 100644 index 000000000000..4b90e4f2b699 --- /dev/null +++ b/docs/integrations/sources/rd-station-marketing.md @@ -0,0 +1,44 @@ +# RD Station Marketing + +RD Station Marketing is the leading Marketing Automation tool in Latin America. It is a software application that helps your company carry out better campaigns, nurture Leads, generate qualified business opportunities and achieve more results. From social media to email, Landing Pages, Pop-ups, even Automations and Analytics. + +## Prerequisites +* An RD Station account +* A callback URL to receive the first account credential (can be done using localhost) +* `client_id` and `client_secret` credentials. Access [this link](https://appstore.rdstation.com/en/publisher) to register a new application and start the authentication flow. + +## Airbyte Open Source +* Start Date +* client_id +* client_secret +* Refresh token + +## Supported sync modes + +The RD Station Marketing source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + - Full Refresh + - Incremental (for analytics endpoints) + +## Supported Streams + +* conversions (analytics endpoint) +* emails (analytics endpoint) +* funnel (analytics endpoint) +* workflow_emails_statistics (analytics endpoint) +* emails +* embeddables +* fields +* landing_pages +* popups +* segmentations +* workflows + +## Performance considerations + +Each endpoint has its own performance limitations, which also consider the account plan. For more informations, visit the page [API request limit](https://developers.rdstation.com/reference/limite-de-requisicoes-da-api?lng=en). + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------- | +| 0.1.0 | 2022-10-23 | [7092](https://github.com/airbytehq/airbyte/pull/7092) | Initial Release | From c209589031c3f79e668afaa6a40fb85ec1c72552 Mon Sep 17 00:00:00 2001 From: sarafonseca-123 Date: Sun, 23 Oct 2022 21:55:18 -0300 Subject: [PATCH 02/17] Update catalog with all streams --- .../integration_tests/configured_catalog.json | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/configured_catalog.json index 934f1a21f676..f677c9e0d490 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/configured_catalog.json @@ -15,6 +15,159 @@ }, "sync_mode": "incremental", "destination_sync_mode": "append" + }, + { + "stream": { + "name": "analytics_conversions", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "asset_updated_at" + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "analytics_funnel", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "reference_day" + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "analytics_workflow_emails_statistics", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "updated_at" + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "emails", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "update_time" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "embeddables", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "update_time" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "fields", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "update_time" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "landing_pages", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "update_time" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "popups", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "update_time" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "segmentations", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "update_time" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "workflows", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "update_time" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" } ] } \ No newline at end of file From 25c9b338049a9ce08577c599554ef14644aa9785 Mon Sep 17 00:00:00 2001 From: sarafonseca-123 Date: Sun, 23 Oct 2022 21:56:07 -0300 Subject: [PATCH 03/17] Update changelog with the rigth PR reference number --- docs/integrations/sources/rd-station-marketing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/rd-station-marketing.md b/docs/integrations/sources/rd-station-marketing.md index 4b90e4f2b699..1c5953a14d1d 100644 --- a/docs/integrations/sources/rd-station-marketing.md +++ b/docs/integrations/sources/rd-station-marketing.md @@ -41,4 +41,4 @@ Each endpoint has its own performance limitations, which also consider the accou | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------- | -| 0.1.0 | 2022-10-23 | [7092](https://github.com/airbytehq/airbyte/pull/7092) | Initial Release | +| 0.1.0 | 2022-10-23 | [18348](https://github.com/airbytehq/airbyte/pull/7092) | Initial Release | From 5dd5f34f9bcb736ac19b26011a6e44ef92844d3c Mon Sep 17 00:00:00 2001 From: sarafonseca-123 Date: Mon, 24 Oct 2022 19:38:35 -0300 Subject: [PATCH 04/17] Minor fixes to RD Station docs --- docs/integrations/sources/rd-station-marketing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrations/sources/rd-station-marketing.md b/docs/integrations/sources/rd-station-marketing.md index 1c5953a14d1d..0edaee561f8c 100644 --- a/docs/integrations/sources/rd-station-marketing.md +++ b/docs/integrations/sources/rd-station-marketing.md @@ -9,8 +9,8 @@ RD Station Marketing is the leading Marketing Automation tool in Latin America. ## Airbyte Open Source * Start Date -* client_id -* client_secret +* Client Id +* Client Secret * Refresh token ## Supported sync modes From c563d291e7bf6472c7d010a4cbff188d790fc6ff Mon Sep 17 00:00:00 2001 From: sarafonseca-123 Date: Mon, 24 Oct 2022 20:18:25 -0300 Subject: [PATCH 05/17] Change field replication_start_date to start_date --- .../source_rd_station_marketing/source.py | 2 +- .../source_rd_station_marketing/spec.json | 4 ++-- .../source_rd_station_marketing/streams.py | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/source.py b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/source.py index d6fc73ffc289..dd77e70c8afe 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/source.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/source.py @@ -46,7 +46,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: :param config: A Mapping of the user input configuration as defined in the connector spec. """ stream_kwargs = self.get_stream_kwargs(config) - incremental_kwargs = {**stream_kwargs, "replication_start_date": pendulum.parse(config["replication_start_date"])} + incremental_kwargs = {**stream_kwargs, "start_date": pendulum.parse(config["start_date"])} streams = [ AnalyticsEmails(**incremental_kwargs), AnalyticsConversions(**incremental_kwargs), diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/spec.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/spec.json index d75d2eca144c..72eeecef2168 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/spec.json +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/spec.json @@ -5,7 +5,7 @@ "title": "RD Station Marketing Spec", "type": "object", "required": [ - "replication_start_date" + "start_date" ], "additionalProperties": true, "properties": { @@ -48,7 +48,7 @@ } ] }, - "replication_start_date": { + "start_date": { "title": "Start Date", "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. When specified and not None, then stream will behave as incremental", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py index ba254fc8da0c..d40d283bbd49 100755 --- a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py @@ -18,9 +18,9 @@ class RDStationMarketingStream(HttpStream, ABC): extra_params = {} data_field = None - def __init__(self, authenticator, replication_start_date=None, **kwargs): + def __init__(self, authenticator, start_date=None, **kwargs): super().__init__(authenticator=authenticator, **kwargs) - self._replication_start_date = replication_start_date + self._start_date = start_date def path(self, **kwargs) -> str: class_name = self.__class__.__name__ @@ -71,15 +71,15 @@ def request_params( stream_state: Mapping[str, Any], **kwargs ) -> MutableMapping[str, Any]: - replication_start_date = self._replication_start_date + start_date = self._start_date - if replication_start_date: + if start_date: if stream_state.get(self.cursor_field): - replication_start_date = max(pendulum.parse(stream_state[self.cursor_field]), replication_start_date) + start_date = max(pendulum.parse(stream_state[self.cursor_field]), start_date) params = {} params.update( - {"start_date": replication_start_date.strftime("%Y-%m-%d"), + {"start_date": start_date.strftime("%Y-%m-%d"), "end_date": date.today().strftime("%Y-%m-%d"), }) From faaacc219645d1e15a2c6cfdbad9b95feb4b901c Mon Sep 17 00:00:00 2001 From: sarafonseca-123 Date: Mon, 24 Oct 2022 20:39:13 -0300 Subject: [PATCH 06/17] Fix unit tests after changing replication_start_date name --- .../source-rd-station-marketing/unit_tests/test_source.py | 2 +- .../source-rd-station-marketing/unit_tests/test_streams.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_source.py index cbf8de2f6e8f..c198e201bcf4 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_source.py @@ -17,7 +17,7 @@ def test_config(): "client_secret": "test_client_secret", "refresh_token": "test_refresh_token", }, - "replication_start_date": "2022-01-01T00:00:00Z" + "start_date": "2022-01-01T00:00:00Z" } def setup_responses(): diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_streams.py index 8b50776a475a..517a40008cbd 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_streams.py @@ -6,17 +6,20 @@ from unittest.mock import MagicMock import pytest -from source_rd_station_marketing.streams import RDStationMarketingStream +from source_rd_station_marketing.streams import RDStationMarketingStream, Segmentations @pytest.fixture def patch_base_class(mocker): # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(RDStationMarketingStream, "path", "v0/example_endpoint") mocker.patch.object(RDStationMarketingStream, "primary_key", "test_primary_key") mocker.patch.object(RDStationMarketingStream, "__abstractmethods__", set()) +def test_path(patch_base_class): + stream = Segmentations(authenticator=MagicMock()) + assert stream.path() == "/platform/segmentations" + def test_request_params(patch_base_class): stream = RDStationMarketingStream(authenticator=MagicMock()) inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} From 9fe5a60fee87d8500721293d303b173ba761eb81 Mon Sep 17 00:00:00 2001 From: sarafonseca-123 Date: Mon, 24 Oct 2022 20:42:09 -0300 Subject: [PATCH 07/17] Fix typo --- .../connectors/source-rd-station-marketing/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/setup.py b/airbyte-integrations/connectors/source-rd-station-marketing/setup.py index 095ffed27c16..cd18bafcfb56 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/setup.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/setup.py @@ -19,7 +19,7 @@ setup( name="source_rd_station_marketing", - description="Source implementation for Rd Station Marketing.", + description="Source implementation for RD Station Marketing.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), From efc824c439ad8507eac2831cff41a87efe10bc87 Mon Sep 17 00:00:00 2001 From: sarafonseca-123 Date: Mon, 24 Oct 2022 20:57:25 -0300 Subject: [PATCH 08/17] Refact next_page_token logic --- .../source_rd_station_marketing/streams.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py index d40d283bbd49..efe01d23f782 100755 --- a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py @@ -13,8 +13,7 @@ class RDStationMarketingStream(HttpStream, ABC): url_base = "https://api.rd.services" primary_key = None - page = 2 - page_size=125 + page = 1 extra_params = {} data_field = None @@ -32,18 +31,18 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, else: json_response = response.json() if json_response: - page_params = dict(page=self.page) self.page = self.page + 1 - return page_params + return {"next_page": self.page} else: return None def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - params = {"page_size": self.page_size} + params = {"page_size": 125, "page": self.page} if next_page_token: - params.update(**next_page_token) + params = {"page_size": 125, "page": next_page_token["next_page"]} + print(params) return params def parse_response( From 8339efb42642b01ed725072d090b721e37d91b45 Mon Sep 17 00:00:00 2001 From: sarafonseca-123 Date: Mon, 24 Oct 2022 21:03:45 -0300 Subject: [PATCH 09/17] Fix unit tests --- .../source-rd-station-marketing/unit_tests/test_streams.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_streams.py index 517a40008cbd..f89aa8785eef 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_streams.py @@ -23,14 +23,14 @@ def test_path(patch_base_class): def test_request_params(patch_base_class): stream = RDStationMarketingStream(authenticator=MagicMock()) inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {'page_size': 125} + expected_params = {'page': 1, 'page_size': 125} assert stream.request_params(**inputs) == expected_params def test_next_page_token(patch_base_class): stream = RDStationMarketingStream(authenticator=MagicMock()) inputs = {"response": MagicMock()} - expected_token = {'page': 2} + expected_token = {'next_page': 2} assert stream.next_page_token(**inputs) == expected_token From 1fa1cf712e8d36eecd03b81558eb479dda152456 Mon Sep 17 00:00:00 2001 From: sarafonseca-123 Date: Mon, 24 Oct 2022 21:04:42 -0300 Subject: [PATCH 10/17] Remove print function --- .../source_rd_station_marketing/streams.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py index efe01d23f782..d7930a161f31 100755 --- a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py @@ -42,7 +42,6 @@ def request_params( params = {"page_size": 125, "page": self.page} if next_page_token: params = {"page_size": 125, "page": next_page_token["next_page"]} - print(params) return params def parse_response( From a8982672ed07df645ba8767d61d83e8c1b090322 Mon Sep 17 00:00:00 2001 From: sarafonseca-123 Date: Mon, 24 Oct 2022 21:44:15 -0300 Subject: [PATCH 11/17] Update airbyte-cdk version to 0.2 --- .../connectors/source-rd-station-marketing/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/setup.py b/airbyte-integrations/connectors/source-rd-station-marketing/setup.py index cd18bafcfb56..b3a9ab68c439 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/setup.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1.56", + "airbyte-cdk~=0.2", ] TEST_REQUIREMENTS = [ From 0d6e9c152528273ae3a0473cda2d8b4bf1fdf81e Mon Sep 17 00:00:00 2001 From: sarafonseca Date: Wed, 26 Oct 2022 10:37:33 -0300 Subject: [PATCH 12/17] Update airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py Co-authored-by: Marcos Marx --- .../source_rd_station_marketing/streams.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py index d7930a161f31..b1e89324ddf0 100755 --- a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py @@ -71,8 +71,7 @@ def request_params( ) -> MutableMapping[str, Any]: start_date = self._start_date - if start_date: - if stream_state.get(self.cursor_field): + if start_date and stream_state.get(self.cursor_field): start_date = max(pendulum.parse(stream_state[self.cursor_field]), start_date) params = {} From 0aff6dd7133cadc26fe3ca4b0bc28524af024fc5 Mon Sep 17 00:00:00 2001 From: sarafonseca-123 Date: Wed, 26 Oct 2022 11:05:08 -0300 Subject: [PATCH 13/17] Change from yield to yield from --- .../source_rd_station_marketing/streams.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py index d7930a161f31..9eb4cc536526 100755 --- a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py @@ -11,11 +11,12 @@ class RDStationMarketingStream(HttpStream, ABC): - url_base = "https://api.rd.services" - primary_key = None - page = 1 - extra_params = {} data_field = None + extra_params = {} + page = 1 + page_size_limit = 125 + primary_key = None + url_base = "https://api.rd.services" def __init__(self, authenticator, start_date=None, **kwargs): super().__init__(authenticator=authenticator, **kwargs) @@ -39,9 +40,9 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - params = {"page_size": 125, "page": self.page} + params = {"page_size": self.page_size_limit, "page": self.page} if next_page_token: - params = {"page_size": 125, "page": next_page_token["next_page"]} + params = {"page_size": self.page_size_limit, "page": next_page_token["next_page"]} return params def parse_response( @@ -53,8 +54,7 @@ def parse_response( records = response.json().get(self.data_field) else: records = response.json() - for record in records: - yield record + yield from records class IncrementalRDStationMarketingStream(RDStationMarketingStream): @@ -105,8 +105,7 @@ def parse_response( **kwargs ) -> Iterable[Mapping]: records = response.json().get(self.data_field)[0].get(self.data_field) - for record in records: - yield record + yield from records class AnalyticsEmails(IncrementalRDStationMarketingStream): From 5bc38d4053c478d32fa2b67f526a75aa903131bc Mon Sep 17 00:00:00 2001 From: sarafonseca-123 Date: Thu, 27 Oct 2022 10:40:24 -0300 Subject: [PATCH 14/17] Apply format --- .../source_rd_station_marketing/source.py | 6 +- .../source_rd_station_marketing/streams.py | 58 ++++++++++--------- .../unit_tests/test_incremental_streams.py | 1 + .../unit_tests/test_source.py | 48 ++++++++------- .../unit_tests/test_streams.py | 45 +++++++------- 5 files changed, 83 insertions(+), 75 deletions(-) diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/source.py b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/source.py index dd77e70c8afe..ee56f5e7d7d6 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/source.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/source.py @@ -26,9 +26,7 @@ class SourceRDStationMarketing(AbstractSource): - def check_connection( - self, logger: AirbyteLogger, config: Mapping[str, Any] - ) -> Tuple[bool, Any]: + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: try: stream_kwargs = self.get_stream_kwargs(config) segmentations = Segmentations(**stream_kwargs) @@ -58,7 +56,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: LandingPages(**stream_kwargs), Popups(**stream_kwargs), Segmentations(**stream_kwargs), - Workflows(**stream_kwargs) + Workflows(**stream_kwargs), ] return streams diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py index 4f65d78c2091..25bca0799280 100755 --- a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/streams.py @@ -1,6 +1,7 @@ # # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + from abc import ABC from datetime import date from typing import Any, Iterable, Mapping, MutableMapping, Optional @@ -25,7 +26,7 @@ def __init__(self, authenticator, start_date=None, **kwargs): def path(self, **kwargs) -> str: class_name = self.__class__.__name__ return f"/platform/{class_name[0].lower()}{class_name[1:]}" - + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: if self.data_field: json_response = response.json().get(self.data_field) @@ -36,7 +37,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, return {"next_page": self.page} else: return None - + def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: @@ -45,11 +46,7 @@ def request_params( params = {"page_size": self.page_size_limit, "page": next_page_token["next_page"]} return params - def parse_response( - self, - response: requests.Response, - **kwargs - ) -> Iterable[Mapping]: + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: if self.data_field: records = response.json().get(self.data_field) else: @@ -60,25 +57,23 @@ def parse_response( class IncrementalRDStationMarketingStream(RDStationMarketingStream): def path(self, **kwargs) -> str: return f"/platform/analytics/{self.data_field}" - + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: return None - - def request_params( - self, - stream_state: Mapping[str, Any], - **kwargs - ) -> MutableMapping[str, Any]: + + def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: start_date = self._start_date - + if start_date and stream_state.get(self.cursor_field): - start_date = max(pendulum.parse(stream_state[self.cursor_field]), start_date) - + start_date = max(pendulum.parse(stream_state[self.cursor_field]), start_date) + params = {} params.update( - {"start_date": start_date.strftime("%Y-%m-%d"), - "end_date": date.today().strftime("%Y-%m-%d"), - }) + { + "start_date": start_date.strftime("%Y-%m-%d"), + "end_date": date.today().strftime("%Y-%m-%d"), + } + ) params.update(self.extra_params) return params @@ -94,15 +89,12 @@ class AnalyticsConversions(IncrementalRDStationMarketingStream): """ API docs: https://developers.rdstation.com/reference/get_platform-analytics-conversions """ + data_field = "conversions" cursor_field = "asset_updated_at" primary_key = "asset_id" - def parse_response( - self, - response: requests.Response, - **kwargs - ) -> Iterable[Mapping]: + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: records = response.json().get(self.data_field)[0].get(self.data_field) yield from records @@ -111,6 +103,7 @@ class AnalyticsEmails(IncrementalRDStationMarketingStream): """ API docs: https://developers.rdstation.com/reference/get_platform-analytics-emails """ + data_field = "emails" cursor_field = "send_at" primary_key = "campaign_id" @@ -120,6 +113,7 @@ class AnalyticsFunnel(IncrementalRDStationMarketingStream): """ API docs: https://developers.rdstation.com/reference/get_platform-analytics-funnel """ + data_field = "funnel" cursor_field = "reference_day" primary_key = "reference_day" @@ -129,18 +123,20 @@ class AnalyticsWorkflowEmailsStatistics(IncrementalRDStationMarketingStream): """ API docs: https://developers.rdstation.com/reference/get_platform-analytics-workflow-emails """ + data_field = "workflow_email_statistics" cursor_field = "updated_at" primary_key = "workflow_id" def path(self, **kwargs) -> str: - return f"/platform/analytics/workflow_emails_statistics" + return "/platform/analytics/workflow_emails_statistics" class Emails(RDStationMarketingStream): """ API docs: https://developers.rdstation.com/reference/get_platform-emails """ + data_field = "items" primary_key = "id" @@ -149,6 +145,7 @@ class Embeddables(RDStationMarketingStream): """ API docs: https://developers.rdstation.com/reference/get_platform-embeddables """ + primary_key = "id" @@ -156,20 +153,22 @@ class Fields(RDStationMarketingStream): """ API docs: https://developers.rdstation.com/reference/get_platform-contacts-fields """ + data_field = "fields" primary_key = "uuid" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: return None - + def path(self, **kwargs) -> str: - return f"/platform/contacts/fields" + return "/platform/contacts/fields" class LandingPages(RDStationMarketingStream): """ API docs: https://developers.rdstation.com/reference/get_platform-landing-pages """ + primary_key = "id" def path(self, **kwargs) -> str: @@ -180,6 +179,7 @@ class Popups(RDStationMarketingStream): """ API docs: https://developers.rdstation.com/reference/get_platform-popups """ + primary_key = "id" @@ -187,6 +187,7 @@ class Segmentations(RDStationMarketingStream): """ API docs: https://developers.rdstation.com/reference/get_platform-segmentations """ + data_field = "segmentations" primary_key = "id" @@ -195,5 +196,6 @@ class Workflows(RDStationMarketingStream): """ API docs: https://developers.rdstation.com/reference/get_platform-workflows """ + data_field = "workflows" primary_key = "id" diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_incremental_streams.py index 7e37cde8f5fe..6c99b32f1cc0 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_incremental_streams.py @@ -13,6 +13,7 @@ def test_current_stream_state(): return {"updated_time": "2021-10-22"} + @fixture def patch_incremental_base_class(mocker): # Mock abstract methods to enable instantiating abstract class diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_source.py index c198e201bcf4..9b11721310f0 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_source.py @@ -11,15 +11,17 @@ @fixture def test_config(): - return {"authorization": { - "auth_type": "Client", - "client_id": "test_client_id", - "client_secret": "test_client_secret", - "refresh_token": "test_refresh_token", - }, - "start_date": "2022-01-01T00:00:00Z" + return { + "authorization": { + "auth_type": "Client", + "client_id": "test_client_id", + "client_secret": "test_client_secret", + "refresh_token": "test_refresh_token", + }, + "start_date": "2022-01-01T00:00:00Z", } + def setup_responses(): responses.add( responses.POST, @@ -29,24 +31,26 @@ def setup_responses(): responses.add( responses.GET, "https://api.rd.services/platform/segmentations", - json={"segmentations": [ - { - "id": 71625167165, - "name": "A mock segmentation", - "standard": True, - "created_at": "2019-09-04T18:05:42.638-03:00", - "updated_at": "2019-09-04T18:05:42.638-03:00", - "process_status": "processed", - "links": [ + json={ + "segmentations": [ { - "rel": "SEGMENTATIONS.CONTACTS", - "href": "https://api.rd.services/platform/segmentations/71625167165/contacts", - "media": "application/json", - "type": "GET" + "id": 71625167165, + "name": "A mock segmentation", + "standard": True, + "created_at": "2019-09-04T18:05:42.638-03:00", + "updated_at": "2019-09-04T18:05:42.638-03:00", + "process_status": "processed", + "links": [ + { + "rel": "SEGMENTATIONS.CONTACTS", + "href": "https://api.rd.services/platform/segmentations/71625167165/contacts", + "media": "application/json", + "type": "GET", + } + ], } ] - } - ]} + }, ) diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_streams.py index f89aa8785eef..7f3199465464 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/unit_tests/test_streams.py @@ -20,24 +20,26 @@ def test_path(patch_base_class): stream = Segmentations(authenticator=MagicMock()) assert stream.path() == "/platform/segmentations" + def test_request_params(patch_base_class): stream = RDStationMarketingStream(authenticator=MagicMock()) inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {'page': 1, 'page_size': 125} + expected_params = {"page": 1, "page_size": 125} assert stream.request_params(**inputs) == expected_params def test_next_page_token(patch_base_class): stream = RDStationMarketingStream(authenticator=MagicMock()) inputs = {"response": MagicMock()} - expected_token = {'next_page': 2} + expected_token = {"next_page": 2} assert stream.next_page_token(**inputs) == expected_token def test_parse_response(patch_base_class): stream = RDStationMarketingStream(authenticator=MagicMock()) response = MagicMock() - response.json.return_value = [{ + response.json.return_value = [ + { "id": 71625167165, "name": "A mock segmentation", "standard": True, @@ -49,27 +51,28 @@ def test_parse_response(patch_base_class): "rel": "SEGMENTATIONS.CONTACTS", "href": "https://api.rd.services/platform/segmentations/71625167165/contacts", "media": "application/json", - "type": "GET" + "type": "GET", } - ] - }] + ], + } + ] inputs = {"response": response, "stream_state": None} expected_parsed_object = { - "id": 71625167165, - "name": "A mock segmentation", - "standard": True, - "created_at": "2019-09-04T18:05:42.638-03:00", - "updated_at": "2019-09-04T18:05:42.638-03:00", - "process_status": "processed", - "links": [ - { - "rel": "SEGMENTATIONS.CONTACTS", - "href": "https://api.rd.services/platform/segmentations/71625167165/contacts", - "media": "application/json", - "type": "GET" - } - ] - } + "id": 71625167165, + "name": "A mock segmentation", + "standard": True, + "created_at": "2019-09-04T18:05:42.638-03:00", + "updated_at": "2019-09-04T18:05:42.638-03:00", + "process_status": "processed", + "links": [ + { + "rel": "SEGMENTATIONS.CONTACTS", + "href": "https://api.rd.services/platform/segmentations/71625167165/contacts", + "media": "application/json", + "type": "GET", + } + ], + } assert next(stream.parse_response(**inputs)) == expected_parsed_object From ab0a0834fb1fa9834c549c0adb9984be1c4d20de Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Fri, 28 Oct 2022 09:32:36 -0300 Subject: [PATCH 15/17] add rd station to source def --- .../resources/seed/source_definitions.yaml | 7 + .../integration_tests/configured_catalog.json | 124 ------------------ 2 files changed, 7 insertions(+), 124 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index e4f28b0fc94a..cb6965ce2c86 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -933,6 +933,13 @@ icon: retently.svg sourceType: api releaseStage: alpha +- name: RD Station Marketing + sourceDefinitionId: fb141f29-be2a-450b-a4f2-2cd203a00f84 + dockerRepository: airbyte/source-rd-station-marketing + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/rd-station-marketing + sourceType: api + releaseStage: alpha - name: RKI Covid sourceDefinitionId: d78e5de0-aa44-4744-aa4f-74c818ccfe19 dockerRepository: airbyte/source-rki-covid diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/configured_catalog.json index f677c9e0d490..c76a23d355d2 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/configured_catalog.json @@ -1,69 +1,5 @@ { "streams": [ - { - "stream": { - "name": "analytics_emails", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "send_at" - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "analytics_conversions", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "asset_updated_at" - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "analytics_funnel", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "reference_day" - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "analytics_workflow_emails_statistics", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, { "stream": { "name": "emails", @@ -79,36 +15,6 @@ "sync_mode": "full_refresh", "destination_sync_mode": "append" }, - { - "stream": { - "name": "embeddables", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "update_time" - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "fields", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "update_time" - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append" - }, { "stream": { "name": "landing_pages", @@ -124,21 +30,6 @@ "sync_mode": "full_refresh", "destination_sync_mode": "append" }, - { - "stream": { - "name": "popups", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "update_time" - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append" - }, { "stream": { "name": "segmentations", @@ -153,21 +44,6 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "append" - }, - { - "stream": { - "name": "workflows", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "update_time" - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append" } ] } \ No newline at end of file From 47b9d3c5a74d6d48f649fa84abc7c119ca184b62 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Fri, 28 Oct 2022 09:33:33 -0300 Subject: [PATCH 16/17] update doc changelog --- docs/integrations/sources/rd-station-marketing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/rd-station-marketing.md b/docs/integrations/sources/rd-station-marketing.md index 0edaee561f8c..989d4e205499 100644 --- a/docs/integrations/sources/rd-station-marketing.md +++ b/docs/integrations/sources/rd-station-marketing.md @@ -41,4 +41,4 @@ Each endpoint has its own performance limitations, which also consider the accou | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------- | -| 0.1.0 | 2022-10-23 | [18348](https://github.com/airbytehq/airbyte/pull/7092) | Initial Release | +| 0.1.0 | 2022-10-23 | [18348](https://github.com/airbytehq/airbyte/pull/18348) | Initial Release | From ab074b46ec758efcae3f9616f16b5f2851c0c27a Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 28 Oct 2022 13:10:49 +0000 Subject: [PATCH 17/17] auto-bump connector version --- .../src/main/resources/seed/source_specs.yaml | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 348a435fc32e..47353fa9c581 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -9607,6 +9607,70 @@ path_in_connector_config: - "credentials" - "client_secret" +- dockerImage: "airbyte/source-rd-station-marketing:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/rd-station-marketing" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "RD Station Marketing Spec" + type: "object" + required: + - "start_date" + additionalProperties: true + properties: + authorization: + type: "object" + title: "Authentication Type" + description: "Choose one of the possible authorization method" + oneOf: + - title: "Sign in via RD Station (OAuth)" + type: "object" + required: + - "auth_type" + properties: + auth_type: + type: "string" + const: "Client" + order: 0 + client_id: + title: "Client ID" + type: "string" + description: "The Client ID of your RD Station developer application." + airbyte_secret: true + client_secret: + title: "Client Secret" + type: "string" + description: "The Client Secret of your RD Station developer application" + airbyte_secret: true + refresh_token: + title: "Refresh Token" + type: "string" + description: "The token for obtaining the new access token." + airbyte_secret: true + start_date: + title: "Start Date" + description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ + \ data before this date will not be replicated. When specified and not\ + \ None, then stream will behave as incremental" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + examples: + - "2017-01-25T00:00:00Z" + type: "string" + supportsIncremental: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] + authSpecification: + auth_type: "oauth2.0" + oauth2Specification: + rootObject: + - "authorization" + - "0" + oauthFlowInitParameters: + - - "client_id" + - - "client_secret" + oauthFlowOutputParameters: + - - "refresh_token" - dockerImage: "airbyte/source-rki-covid:0.1.1" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/rki-covid"