From f185c5b21a343f4943c6e943c59eccfb5d7a85c0 Mon Sep 17 00:00:00 2001 From: alexchouraki Date: Tue, 11 Oct 2022 11:48:45 +0200 Subject: [PATCH 01/25] first working connector - overwrite only --- .../source-zendesk-sell/.dockerignore | 6 + .../connectors/source-zendesk-sell/Dockerfile | 38 ++++ .../connectors/source-zendesk-sell/README.md | 132 +++++++++++ .../acceptance-test-config.yml | 30 +++ .../acceptance-test-docker.sh | 16 ++ .../source-zendesk-sell/build.gradle | 9 + .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 14 ++ .../integration_tests/configured_catalog.json | 60 +++++ .../integration_tests/invalid_config.json | 3 + .../integration_tests/sample_config.json | 3 + .../integration_tests/sample_state.json | 5 + .../connectors/source-zendesk-sell/main.py | 13 ++ .../source-zendesk-sell/requirements.txt | 2 + .../connectors/source-zendesk-sell/setup.py | 29 +++ .../source_zendesk_sell/__init__.py | 8 + .../source_zendesk_sell/schemas/TODO.md | 25 +++ .../source_zendesk_sell/schemas/contacts.json | 100 +++++++++ .../source_zendesk_sell/schemas/deals.json | 87 ++++++++ .../source_zendesk_sell/schemas/leads.json | 85 +++++++ .../schemas/pipelines.json | 25 +++ .../source_zendesk_sell/schemas/stages.json | 37 ++++ .../source_zendesk_sell/source.py | 209 ++++++++++++++++++ .../source_zendesk_sell/spec.yaml | 15 ++ .../unit_tests/__init__.py | 3 + .../unit_tests/test_incremental_streams.py | 59 +++++ .../unit_tests/test_source.py | 22 ++ .../unit_tests/test_streams.py | 83 +++++++ 29 files changed, 1126 insertions(+) create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/.dockerignore create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/Dockerfile create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/README.md create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/build.gradle create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/main.py create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/requirements.txt create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/setup.py create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/__init__.py create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/contacts.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deals.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/leads.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/pipelines.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/stages.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_incremental_streams.py create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-zendesk-sell/.dockerignore b/airbyte-integrations/connectors/source-zendesk-sell/.dockerignore new file mode 100644 index 000000000000..7c3c867373cf --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_zendesk_sell +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-zendesk-sell/Dockerfile b/airbyte-integrations/connectors/source-zendesk-sell/Dockerfile new file mode 100644 index 000000000000..381fff970479 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.13-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_zendesk_sell ./source_zendesk_sell + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-zendesk-sell diff --git a/airbyte-integrations/connectors/source-zendesk-sell/README.md b/airbyte-integrations/connectors/source-zendesk-sell/README.md new file mode 100644 index 000000000000..ef20d50da41f --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/README.md @@ -0,0 +1,132 @@ +# Zendesk Sell Source + +This is the repository for the Zendesk Sell source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/zendesk-sell). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-zendesk-sell:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/zendesk-sell) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zendesk_sell/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source zendesk-sell test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-zendesk-sell:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-zendesk-sell:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-zendesk-sell:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-sell:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-sell:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-zendesk-sell:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-zendesk-sell:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-zendesk-sell:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-config.yml new file mode 100644 index 000000000000..9e44467ce664 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-zendesk-sell:dev +tests: + spec: + - spec_path: "source_zendesk_sell/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.txt" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: # TODO if your connector does not implement incremental sync, remove this block + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-zendesk-sell/build.gradle b/airbyte-integrations/connectors/source-zendesk-sell/build.gradle new file mode 100644 index 000000000000..4b2aeaebf368 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_zendesk_sell' +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/__init__.py b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..871548c4aa59 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "contacts": { + "created_at": "33190962600" + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py new file mode 100644 index 000000000000..cd2dc4aaffb2 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..7083a2c1ec63 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/configured_catalog.json @@ -0,0 +1,60 @@ +{ + "streams": [ + { + "stream": { + "name": "pipelines", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "stages", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "deals", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "contacts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "leads", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/invalid_config.json new file mode 100644 index 000000000000..f65c5e3c6e87 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "api_token": "f23yhd630otl94y85a8bf384958473pto95847fd006da49382716or937ruw059" +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_config.json new file mode 100644 index 000000000000..9f6bd635126f --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "api_token": "c56afd675afe19b87a8bf810666927baa58854fd006da22795850ee957eec854" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_state.json new file mode 100644 index 000000000000..b431595f17b8 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "contacts": { + "created_at": "1604479286" + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/main.py b/airbyte-integrations/connectors/source-zendesk-sell/main.py new file mode 100644 index 000000000000..e5cc433b0d36 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zendesk_sell import SourceZendeskSell + +if __name__ == "__main__": + source = SourceZendeskSell() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/requirements.txt b/airbyte-integrations/connectors/source-zendesk-sell/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-zendesk-sell/setup.py b/airbyte-integrations/connectors/source-zendesk-sell/setup.py new file mode 100644 index 000000000000..0900235594ab --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1.56", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_zendesk_sell", + description="Source implementation for Zendesk Sell.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/__init__.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/__init__.py new file mode 100644 index 000000000000..153d0c6b7b8e --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceZendeskSell + +__all__ = ["SourceZendeskSell"] diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/TODO.md b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/TODO.md new file mode 100644 index 000000000000..cf1efadb3c9c --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/TODO.md @@ -0,0 +1,25 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. + +The schema of a stream is the return value of `Stream.get_json_schema`. + +## Static schemas +By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. + +Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. + +## Dynamic schemas +If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). + +## Dynamically modifying static schemas +Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: +``` +def get_json_schema(self): + schema = super().get_json_schema() + schema['dynamically_determined_property'] = "property" + return schema +``` + +Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/contacts.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/contacts.json new file mode 100644 index 000000000000..fd55445cc198 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/contacts.json @@ -0,0 +1,100 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "owner_id": { + "type": ["null", "number"] + }, + "is_organization": { + "type": ["null", "boolean"] + }, + "contact_id": { + "type": ["null", "number"] + }, + "parent_organization_id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "customer_status": { + "type": ["null", "string"] + }, + "prospect_status": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "industry": { + "type": ["null", "string"] + }, + "website": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "mobile": { + "type": ["null", "string"] + }, + "fax": { + "type": ["null", "string"] + }, + "twitter": { + "type": ["null", "string"] + }, + "facebook": { + "type": ["null", "string"] + }, + "linkedin": { + "type": ["null", "string"] + }, + "skype": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "billing_address": { + "type": ["null", "string"] + }, + "shipping_address": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "array"] + }, + "custom_fields": { + "type": ["null", "object"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deals.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deals.json new file mode 100644 index 000000000000..a96c3a32059d --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deals.json @@ -0,0 +1,87 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "owner_id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "number", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "hot": { + "type": ["null", "boolean"] + }, + "stage_id": { + "type": ["null", "number"] + }, + "last_stage_change_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "last_stage_change_by_id": { + "type": ["null", "number"] + }, + "last_activity_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "source_id": { + "type": ["null", "number"] + }, + "loss_reason_id": { + "type": ["null", "number"] + }, + "unqualified_reason_id": { + "type": ["null", "number"] + }, + "dropbox_email": { + "type": ["null", "string"] + }, + "contact_id": { + "type": ["null", "number"] + }, + "organization_id": { + "type": ["null", "number"] + }, + "estimated_close_date": { + "type": ["null", "string"] + }, + "customized_win_likelihood": { + "type": ["null", "number"] + }, + "tags": { + "type": ["null", "array"] + }, + "custom_fields": { + "type": ["null", "object"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "added_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/leads.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/leads.json new file mode 100644 index 000000000000..509fb4cf2cf9 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/leads.json @@ -0,0 +1,85 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "owner_id": { + "type": ["null", "number"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "organization_name": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "source_id": { + "type": ["null", "number"] + }, + "title": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "industry": { + "type": ["null", "string"] + }, + "website": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "mobile": { + "type": ["null", "string"] + }, + "fax": { + "type": ["null", "string"] + }, + "twitter": { + "type": ["null", "string"] + }, + "facebook": { + "type": ["null", "string"] + }, + "linkedin": { + "type": ["null", "string"] + }, + "skype": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "array"] + }, + "custom_fields": { + "type": ["null", "object"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/pipelines.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/pipelines.json new file mode 100644 index 000000000000..cd44ef4aed34 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/pipelines.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "disabled": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/stages.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/stages.json new file mode 100644 index 000000000000..0a826d2e8eae --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/stages.json @@ -0,0 +1,37 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "category": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "position": { + "type": ["null", "number"] + }, + "likelihood": { + "type": ["null", "number"] + }, + "pipeline_id": { + "type": ["null", "number"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py new file mode 100644 index 000000000000..4f53df8a6eb7 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py @@ -0,0 +1,209 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +import re +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from airbyte_cdk.models import SyncMode + +""" +TODO: Most comments in this class are instructive and should be deleted after the source is implemented. + +This file provides a stubbed example of how to use the Airbyte CDK to develop both a source connector which supports full refresh or and an +incremental syncs from an HTTP API. + +The various TODOs are both implementation hints and steps - fulfilling all the TODOs should be sufficient to implement one basic and one incremental +stream from a source. This pattern is the same one used by Airbyte internally to implement connectors. + +The approach here is not authoritative, and devs are free to use their own judgement. + +There are additional required TODOs in the files within the integration_tests folder and the spec.yaml file. +""" + + +# Basic full refresh stream +class ZendeskSellStream(HttpStream, ABC): + """ + TODO remove this comment + + This class represents a stream output by the connector. + This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, + parsing responses etc.. + + Each stream should extend this class (or another abstract subclass of it) to specify behavior unique to that stream. + + Typically for REST APIs each stream corresponds to a resource in the API. For example if the API + contains the endpoints + - GET v1/customers + - GET v1/employees + + then you should have three classes: + `class ZendeskSellStream(HttpStream, ABC)` which is the current class + `class Customers(ZendeskSellStream)` contains behavior to pull data for customers using v1/customers + `class Employees(ZendeskSellStream)` contains behavior to pull data for employees using v1/employees + + If some streams implement incremental sync, it is typical to create another class + `class IncrementalZendeskSellStream((ZendeskSellStream), ABC)` then have concrete stream implementations extend it. An example + is provided below. + + See the reference docs for the full list of configurable options. + """ + + url_base = "https://api.getbase.com/v2/" + primary_key = None + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. + + This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed + to most other methods in this class to help you form headers, request bodies, query params, etc.. + + For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a + 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. + The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. + + :param response: the most recent response from the API + :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. + If there are no more pages in the result, return None. + """ + regex_page='[=?/]page[_=/-]?(\d{1,3})' + meta_links = response.json()['meta']['links'] + if 'next_page' in meta_links.keys() : + {'page' : re.findall(regex_page, meta_links['next_page'])[0]} + else: + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + """ + TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. + Usually contains common params e.g. pagination size etc. + """ + return {'page': next_page_token, 'per_page' : 100} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + TODO: Override this method to define how a response is parsed. + :return an iterable containing each record in the response + """ + items = response.json()['items'] + return [item['data'] for item in items] + + +class Pipelines(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/pipelines/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "pipelines" + +class Stages(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/stages/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "stages" + + +# # Basic incremental stream +# class IncrementalZendeskSellStream(ZendeskSellStream, ABC): +# """ +# TODO fill in details of this class to implement functionality related to incremental syncs for your connector. +# if you do not need to implement incremental sync for any streams, remove this class. +# """ + +# # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. +# state_checkpoint_interval = 100 + +# @property +# def cursor_field(self) -> str: +# """ +# TODO +# Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is +# usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. + +# :return str: The name of the cursor field. +# """ +# return "updated_at" + +# def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: +# """ +# Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and +# the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. +# """ +# return {} + + +class Contacts(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/contacts/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "contacts" + +class Deals(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deals/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "deals" + +class Leads(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/leads/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "leads" + + +# Source +class SourceZendeskSell(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + """ + TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API + + See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 + for an example. + + :param config: the user-input config object conforming to the connector's spec.yaml + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + try: + authenticator = TokenAuthenticator(token = config["api_token"]) + stream = Contacts(authenticator=authenticator) + records = stream.read_records(sync_mode=SyncMode.full_refresh) + next(records) + return True, None + except Exception as e: + return False, e + + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + TODO: Replace the streams below with your own streams. + + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + # TODO remove the authenticator if not required. + auth = TokenAuthenticator(token=config["api_token"]) # Oauth2Authenticator is also available if you need oauth support + return [Contacts(authenticator=auth), Deals(authenticator=auth), Leads(authenticator=auth), Pipelines(authenticator=auth), Stages(authenticator=auth)] diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml new file mode 100644 index 000000000000..482ac82b73fe --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml @@ -0,0 +1,15 @@ +documentationUrl: https://developer.zendesk.com/api-reference/sales-crm/introduction/#get-up-to-speed-with-the-sell-api +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Source Zendesk Sell Spec + type: object + required: + - api_token + properties: + api_token: + title: API token + type: string + description: "The API token for authenticating to Zendesk Sell" + examples: + - "f23yhd630otl94y85a8bf384958473pto95847fd006da49382716or937ruw059" + airbyte_secret: true \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/__init__.py b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_incremental_streams.py new file mode 100644 index 000000000000..ad2ba1172cc1 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_incremental_streams.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_zendesk_sell.source import IncrementalZendeskSellStream + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(IncrementalZendeskSellStream, "path", "v0/example_endpoint") + mocker.patch.object(IncrementalZendeskSellStream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalZendeskSellStream, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + stream = IncrementalZendeskSellStream() + # TODO: replace this with your expected cursor field + expected_cursor_field = [] + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_base_class): + stream = IncrementalZendeskSellStream() + # TODO: replace this with your input parameters + inputs = {"current_stream_state": None, "latest_record": None} + # TODO: replace this with your expected updated stream state + expected_state = {} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_stream_slices(patch_incremental_base_class): + stream = IncrementalZendeskSellStream() + # TODO: replace this with your input parameters + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + # TODO: replace this with your expected stream slices list + expected_stream_slice = [None] + assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + mocker.patch.object(IncrementalZendeskSellStream, "cursor_field", "dummy_field") + stream = IncrementalZendeskSellStream() + assert stream.supports_incremental + + +def test_source_defined_cursor(patch_incremental_base_class): + stream = IncrementalZendeskSellStream() + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(patch_incremental_base_class): + stream = IncrementalZendeskSellStream() + # TODO: replace this with your expected checkpoint interval + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py new file mode 100644 index 000000000000..b1ee046e1399 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_zendesk_sell.source import SourceZendeskSell + + +def test_check_connection(mocker): + source = SourceZendeskSell() + logger_mock, config_mock = MagicMock(), MagicMock() + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker): + source = SourceZendeskSell() + config_mock = MagicMock() + streams = source.streams(config_mock) + # TODO: replace this with your streams number + expected_streams_number = 2 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py new file mode 100644 index 000000000000..10da8457b19c --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_zendesk_sell.source import ZendeskSellStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(ZendeskSellStream, "path", "v0/example_endpoint") + mocker.patch.object(ZendeskSellStream, "primary_key", "test_primary_key") + mocker.patch.object(ZendeskSellStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = ZendeskSellStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request parameters + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = ZendeskSellStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected next page token + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = ZendeskSellStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected parced object + expected_parsed_object = {} + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = ZendeskSellStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request headers + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = ZendeskSellStream() + # TODO: replace this with your expected http request method + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = ZendeskSellStream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = ZendeskSellStream() + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time From c7122c5d0b9528719dfa53c01036378bbc6efa9d Mon Sep 17 00:00:00 2001 From: alexchouraki Date: Tue, 11 Oct 2022 17:58:03 +0200 Subject: [PATCH 02/25] edit readme and source definitions --- .../init/src/main/resources/seed/source_definitions.yaml | 7 +++++++ .../connectors/source-zendesk-sell/README.md | 1 + 2 files changed, 8 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 6443c29c51c8..dcffe8188444 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1257,3 +1257,10 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/yandex-metrica sourceType: api releaseStage: alpha +- name: Zendesk Sell + sourceDefinitionId: c41e4903-1000-4f2c-a768-979e45301e38 + dockerRepository: airbyte/source-zendesk-sell:dev + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-sell + sourceType: api + releaseStage: alpha diff --git a/airbyte-integrations/connectors/source-zendesk-sell/README.md b/airbyte-integrations/connectors/source-zendesk-sell/README.md index ef20d50da41f..741adc1a197c 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/README.md +++ b/airbyte-integrations/connectors/source-zendesk-sell/README.md @@ -43,6 +43,7 @@ To build using Gradle, from the Airbyte repository root, run: to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zendesk_sell/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. +Note that the full process to generate api tokens is available [here](https://developer.zendesk.com/documentation/sales-crm/first-call/#1-generate-an-access-token) **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source zendesk-sell test creds` and place them into `secrets/config.json`. From 9cc80b5083741d58c7490d1b95120db3d3f976d1 Mon Sep 17 00:00:00 2001 From: alexchouraki Date: Tue, 11 Oct 2022 18:04:36 +0200 Subject: [PATCH 03/25] Update source_definitions.yaml --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index dcffe8188444..9b043b08216e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1259,7 +1259,7 @@ releaseStage: alpha - name: Zendesk Sell sourceDefinitionId: c41e4903-1000-4f2c-a768-979e45301e38 - dockerRepository: airbyte/source-zendesk-sell:dev + dockerRepository: airbyte/source-zendesk-sell dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-sell sourceType: api From f709d5cb816b2363094fd4f9813d03aad6cef5d3 Mon Sep 17 00:00:00 2001 From: alexchouraki Date: Wed, 12 Oct 2022 10:59:08 +0200 Subject: [PATCH 04/25] changing catalog and adding all available streams --- .../resources/seed/source_definitions.yaml | 7 - .../integration_tests/configured_catalog.json | 186 +++++++++++- .../schemas/call_outcomes.json | 25 ++ .../source_zendesk_sell/schemas/calls.json | 57 ++++ .../schemas/collaborations.json | 31 ++ .../schemas/custom_fields.json | 34 +++ .../schemas/deal_sources.json | 28 ++ .../schemas/deal_unqualified_reasons.json | 25 ++ .../schemas/documents.json | 40 +++ .../schemas/lead_conversions.json | 29 ++ .../schemas/lead_sources.json | 28 ++ .../schemas/lead_unqualified_reasons.json | 25 ++ .../schemas/loss_reasons.json | 25 ++ .../source_zendesk_sell/schemas/notes.json | 40 +++ .../source_zendesk_sell/schemas/orders.json | 25 ++ .../source_zendesk_sell/schemas/products.json | 46 +++ .../source_zendesk_sell/schemas/tags.json | 28 ++ .../source_zendesk_sell/schemas/tasks.json | 53 ++++ .../schemas/text_messages.json | 48 ++++ .../source_zendesk_sell/schemas/users.json | 66 +++++ .../schemas/visit_outcomes.json | 25 ++ .../source_zendesk_sell/schemas/visits.json | 45 +++ .../source_zendesk_sell/source.py | 271 ++++++++++++------ 23 files changed, 1096 insertions(+), 91 deletions(-) create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/call_outcomes.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/calls.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/collaborations.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/custom_fields.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_sources.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_unqualified_reasons.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/documents.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_conversions.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_sources.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_unqualified_reasons.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/loss_reasons.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/notes.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/orders.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/products.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tags.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tasks.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/text_messages.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/users.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visit_outcomes.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visits.json diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 9b043b08216e..6443c29c51c8 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1257,10 +1257,3 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/yandex-metrica sourceType: api releaseStage: alpha -- name: Zendesk Sell - sourceDefinitionId: c41e4903-1000-4f2c-a768-979e45301e38 - dockerRepository: airbyte/source-zendesk-sell - dockerImageTag: 0.1.0 - documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-sell - sourceType: api - releaseStage: alpha diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/configured_catalog.json index 7083a2c1ec63..20c9544fa3ca 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/configured_catalog.json @@ -24,7 +24,7 @@ "stream": { "name": "deals", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "supported_sync_modes": ["full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] @@ -36,7 +36,7 @@ "stream": { "name": "contacts", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "supported_sync_modes": ["full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] @@ -48,13 +48,193 @@ "stream": { "name": "leads", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "supported_sync_modes": ["full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append" + }, + { + "stream": { + "name": "call_outcomes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "custom_fields", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "deal_sources", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "deal_unqualified_reasons", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "documents", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "lead_conversions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "lead_sources", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "lead_unqualified_reasons", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "loss_reasons", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "notes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "orders", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "products", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "tags", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "tasks", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "text_messages", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "visit_outcomes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "visits", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/call_outcomes.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/call_outcomes.json new file mode 100644 index 000000000000..0ca39aa5f870 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/call_outcomes.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/calls.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/calls.json new file mode 100644 index 000000000000..a18616afa129 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/calls.json @@ -0,0 +1,57 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "user_id": { + "type": ["null", "number"] + }, + "summary": { + "type": ["null", "string"] + }, + "recording_url": { + "type": ["null", "string"] + }, + "outcome_id": { + "type": ["null", "number"] + }, + "duration": { + "type": ["null", "number"] + }, + "phone_number": { + "type": ["null", "number"] + }, + "incoming": { + "type": ["null", "boolean"] + }, + "missed": { + "type": ["null", "boolean"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "resource_id": { + "type": ["null", "number"] + }, + "associated_deal_ids": { + "type": ["null", "array"] + }, + "made_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/collaborations.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/collaborations.json new file mode 100644 index 000000000000..89e857aa5031 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/collaborations.json @@ -0,0 +1,31 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "resource_id": { + "type": ["null", "number"] + }, + "collaborator_id": { + "type": ["null", "number"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/custom_fields.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/custom_fields.json new file mode 100644 index 000000000000..188b7d5b0b3f --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/custom_fields.json @@ -0,0 +1,34 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "choices": { + "type": ["null", "array"] + }, + "for_company": { + "type": ["null", "boolean"] + }, + "for_contact": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_sources.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_sources.json new file mode 100644 index 000000000000..4c1d21218947 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_sources.json @@ -0,0 +1,28 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_unqualified_reasons.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_unqualified_reasons.json new file mode 100644 index 000000000000..0ca39aa5f870 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_unqualified_reasons.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/documents.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/documents.json new file mode 100644 index 000000000000..5dd4d8ecfa91 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/documents.json @@ -0,0 +1,40 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "resource_id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "content_type": { + "type": ["null", "string"] + }, + "size": { + "type": ["null", "number"] + }, + "download_url": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_conversions.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_conversions.json new file mode 100644 index 000000000000..0f973d1cd534 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_conversions.json @@ -0,0 +1,29 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "lead_id": { + "type": ["null", "number"] + }, + "individual_id": { + "type": ["null", "number"] + }, + "organization_id": { + "type": ["null", "number"] + }, + "deal_id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_sources.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_sources.json new file mode 100644 index 000000000000..4c1d21218947 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_sources.json @@ -0,0 +1,28 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_unqualified_reasons.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_unqualified_reasons.json new file mode 100644 index 000000000000..f93853b912d9 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_unqualified_reasons.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/loss_reasons.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/loss_reasons.json new file mode 100644 index 000000000000..f93853b912d9 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/loss_reasons.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/notes.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/notes.json new file mode 100644 index 000000000000..d2ea84d6f02f --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/notes.json @@ -0,0 +1,40 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "resource_id": { + "type": ["null", "number"] + }, + "content": { + "type": ["null", "string"] + }, + "is_important": { + "type": ["null", "boolean"] + }, + "tags": { + "type": ["null", "array"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "type": { + "type": ["null", "string"] + } + } + } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/orders.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/orders.json new file mode 100644 index 000000000000..d789fb749c46 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/orders.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "deal_id": { + "type": ["null", "number"] + }, + "discount": { + "type": ["null", "number"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/products.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/products.json new file mode 100644 index 000000000000..08fa23b528ca --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/products.json @@ -0,0 +1,46 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "sku": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "max_discount": { + "type": ["null", "number"] + }, + "max_markup": { + "type": ["null", "number"] + }, + "cost": { + "type": ["null", "number"] + }, + "cost_currency": { + "type": ["null", "string"] + }, + "prices": { + "type": ["null", "array"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tags.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tags.json new file mode 100644 index 000000000000..4c1d21218947 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tags.json @@ -0,0 +1,28 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tasks.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tasks.json new file mode 100644 index 000000000000..5ab1af48147e --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tasks.json @@ -0,0 +1,53 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "owner_id": { + "type": ["null", "number"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "resource_id": { + "type": ["null", "number"] + }, + "completed": { + "type": ["null", "boolean"] + }, + "completed_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "due_date": { + "type": ["null", "string"] + }, + "overdue": { + "type": ["null", "boolean"] + }, + "remind_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "content": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/text_messages.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/text_messages.json new file mode 100644 index 000000000000..fe1532cfd06c --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/text_messages.json @@ -0,0 +1,48 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "associated_deal_ids": { + "type": ["null", "array"] + }, + "content": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "incoming": { + "type": ["null", "boolean"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "resource_id": { + "type": ["null", "number"] + }, + "resource_phone_number": { + "type": ["null", "string"] + }, + "sent_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "user_id": { + "type": ["null", "number"] + }, + "user_phone_number": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/users.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/users.json new file mode 100644 index 000000000000..a3869e2839bf --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/users.json @@ -0,0 +1,66 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "invited": { + "type": ["null", "boolean"] + }, + "confirmed": { + "type": ["null", "boolean"] + }, + "phone_number": { + "type": ["null", "string"] + }, + "role": { + "type": ["null", "string"] + }, + "roles": { + "type": ["null", "array"] + }, + "team_name": { + "type": ["null", "string"] + }, + "group": { + "type": ["null", "object"] + }, + "reports_to": { + "type": ["null", "number"] + }, + "timezone": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "deleted_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "zendesk_user_id": { + "type": ["null", "string"] + }, + "identity_type": { + "type": ["null", "string"] + } + } + } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visit_outcomes.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visit_outcomes.json new file mode 100644 index 000000000000..0ca39aa5f870 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visit_outcomes.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visits.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visits.json new file mode 100644 index 000000000000..ead884e63083 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visits.json @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "outcome_id": { + "type": ["null", "number"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "resource_id": { + "type": ["null", "number"] + }, + "visited_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "resource_address": { + "type": ["null", "string"] + }, + "rep_location_verification_status": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py index 4f53df8a6eb7..0b2c43d099fb 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py @@ -13,88 +13,33 @@ from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator from airbyte_cdk.models import SyncMode -""" -TODO: Most comments in this class are instructive and should be deleted after the source is implemented. - -This file provides a stubbed example of how to use the Airbyte CDK to develop both a source connector which supports full refresh or and an -incremental syncs from an HTTP API. - -The various TODOs are both implementation hints and steps - fulfilling all the TODOs should be sufficient to implement one basic and one incremental -stream from a source. This pattern is the same one used by Airbyte internally to implement connectors. - -The approach here is not authoritative, and devs are free to use their own judgement. - -There are additional required TODOs in the files within the integration_tests folder and the spec.yaml file. -""" - - # Basic full refresh stream class ZendeskSellStream(HttpStream, ABC): """ - TODO remove this comment - This class represents a stream output by the connector. This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, parsing responses etc.. - - Each stream should extend this class (or another abstract subclass of it) to specify behavior unique to that stream. - - Typically for REST APIs each stream corresponds to a resource in the API. For example if the API - contains the endpoints - - GET v1/customers - - GET v1/employees - - then you should have three classes: - `class ZendeskSellStream(HttpStream, ABC)` which is the current class - `class Customers(ZendeskSellStream)` contains behavior to pull data for customers using v1/customers - `class Employees(ZendeskSellStream)` contains behavior to pull data for employees using v1/employees - - If some streams implement incremental sync, it is typical to create another class - `class IncrementalZendeskSellStream((ZendeskSellStream), ABC)` then have concrete stream implementations extend it. An example - is provided below. - - See the reference docs for the full list of configurable options. """ url_base = "https://api.getbase.com/v2/" primary_key = None def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. - - This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed - to most other methods in this class to help you form headers, request bodies, query params, etc.. - - For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a - 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. - The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. - - :param response: the most recent response from the API - :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. - If there are no more pages in the result, return None. - """ regex_page='[=?/]page[_=/-]?(\d{1,3})' - meta_links = response.json()['meta']['links'] - if 'next_page' in meta_links.keys() : - {'page' : re.findall(regex_page, meta_links['next_page'])[0]} - else: - return None + meta_links = response.json().get('meta', {}).get('links') + if 'next_page' in meta_links.keys(): + return {'page' : int(re.findall(regex_page, meta_links['next_page'])[0])} def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - """ - TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. - Usually contains common params e.g. pagination size etc. - """ - return {'page': next_page_token, 'per_page' : 100} + if next_page_token != None: + return {'page': next_page_token['page'], 'per_page' : 100} + else: + return {'per_page' : 100} def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - TODO: Override this method to define how a response is parsed. - :return an iterable containing each record in the response - """ + items = response.json()['items'] return [item['data'] for item in items] @@ -174,20 +119,190 @@ class Leads(ZendeskSellStream): def path(self, **kwargs) -> str: return "leads" +class CallOutcomes(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/call-outcomes/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "call_outcomes" + +class Calls(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/calls/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "calls" + +class Collaborations(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/collaborations/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "collaborations" + +class CustomFields(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/custom-fields/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return ":resource_type/custom_fields" + +class DealSources(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deal-sources/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "deal_sources" + +class DealUnqualifiedReasons(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deal-unqualified-reasons/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "deal_unqualified_reasons" + +class Documents(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/documents/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "documents" + +class LeadConversions(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/lead-conversions/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "lead_conversions" + +class LeadSources(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/lead-sources/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "lead_sources" + +class LeadUnqualifiedReasons(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/lead-unqualified-reasons/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "lead_unqualified_reasons" + +class LossReasons(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/loss-reasons/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "loss_reasons" + +class Notes(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/notes/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "notes" + +class Orders(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/orders/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "orders" + +class Products(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/products/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "products" + +class Tags(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/tags/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "tags" + +class Tasks(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/tasks/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "tasks" + +class TextMessages(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/text-messages/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "text_messages" + +class Users(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/users/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "users" + +class VisitOutcomes(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/visit-outcomes/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "visit_outcomes" + +class Visits(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/visits/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "visits" + # Source class SourceZendeskSell(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API - - See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 - for an example. - - :param config: the user-input config object conforming to the connector's spec.yaml - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ try: authenticator = TokenAuthenticator(token = config["api_token"]) stream = Contacts(authenticator=authenticator) @@ -199,11 +314,5 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - TODO: Replace the streams below with your own streams. - - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - # TODO remove the authenticator if not required. - auth = TokenAuthenticator(token=config["api_token"]) # Oauth2Authenticator is also available if you need oauth support - return [Contacts(authenticator=auth), Deals(authenticator=auth), Leads(authenticator=auth), Pipelines(authenticator=auth), Stages(authenticator=auth)] + auth = TokenAuthenticator(token=config["api_token"]) + return [Contacts(authenticator=auth), Deals(authenticator=auth), Leads(authenticator=auth), Pipelines(authenticator=auth), Stages(authenticator=auth), CallOutcomes(authenticator=auth), Calls(authenticator=auth), Collaborations(authenticator=auth), CustomFields(authenticator=auth), DealSources(authenticator=auth), DealUnqualifiedReasons(authenticator=auth), Documents(authenticator=auth), LeadConversions(authenticator=auth), LeadSources(authenticator=auth), LeadUnqualifiedReasons(authenticator=auth), LossReasons(authenticator=auth), Notes(authenticator=auth), Orders(authenticator=auth), Products(authenticator=auth), Tags(authenticator=auth), Tasks(authenticator=auth), TextMessages(authenticator=auth), Users(authenticator=auth), VisitOutcomes(authenticator=auth), Visits(authenticator=auth)] From 7d0b3045feb8f006f88e46677998b5a3d56eee28 Mon Sep 17 00:00:00 2001 From: alexchouraki Date: Wed, 12 Oct 2022 11:53:08 +0200 Subject: [PATCH 05/25] remove custom fields --- .../schemas/custom_fields.json | 34 ------------------- .../source_zendesk_sell/source.py | 11 +----- 2 files changed, 1 insertion(+), 44 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/custom_fields.json diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/custom_fields.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/custom_fields.json deleted file mode 100644 index 188b7d5b0b3f..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/custom_fields.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "number"] - }, - "name": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "choices": { - "type": ["null", "array"] - }, - "for_company": { - "type": ["null", "boolean"] - }, - "for_contact": { - "type": ["null", "boolean"] - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } - } -} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py index 0b2c43d099fb..5b1f5cb8ab76 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py @@ -146,15 +146,6 @@ class Collaborations(ZendeskSellStream): def path(self, **kwargs) -> str: return "collaborations" -class CustomFields(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/custom-fields/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return ":resource_type/custom_fields" - class DealSources(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deal-sources/ @@ -315,4 +306,4 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = TokenAuthenticator(token=config["api_token"]) - return [Contacts(authenticator=auth), Deals(authenticator=auth), Leads(authenticator=auth), Pipelines(authenticator=auth), Stages(authenticator=auth), CallOutcomes(authenticator=auth), Calls(authenticator=auth), Collaborations(authenticator=auth), CustomFields(authenticator=auth), DealSources(authenticator=auth), DealUnqualifiedReasons(authenticator=auth), Documents(authenticator=auth), LeadConversions(authenticator=auth), LeadSources(authenticator=auth), LeadUnqualifiedReasons(authenticator=auth), LossReasons(authenticator=auth), Notes(authenticator=auth), Orders(authenticator=auth), Products(authenticator=auth), Tags(authenticator=auth), Tasks(authenticator=auth), TextMessages(authenticator=auth), Users(authenticator=auth), VisitOutcomes(authenticator=auth), Visits(authenticator=auth)] + return [Contacts(authenticator=auth), Deals(authenticator=auth), Leads(authenticator=auth), Pipelines(authenticator=auth), Stages(authenticator=auth), CallOutcomes(authenticator=auth), Calls(authenticator=auth), Collaborations(authenticator=auth), DealSources(authenticator=auth), DealUnqualifiedReasons(authenticator=auth), Documents(authenticator=auth), LeadConversions(authenticator=auth), LeadSources(authenticator=auth), LeadUnqualifiedReasons(authenticator=auth), LossReasons(authenticator=auth), Notes(authenticator=auth), Orders(authenticator=auth), Products(authenticator=auth), Tags(authenticator=auth), Tasks(authenticator=auth), TextMessages(authenticator=auth), Users(authenticator=auth), VisitOutcomes(authenticator=auth), Visits(authenticator=auth)] From fc8551f3769e4cd5151d42bc74079521b5c9c8a6 Mon Sep 17 00:00:00 2001 From: alexchouraki Date: Wed, 12 Oct 2022 13:44:02 +0200 Subject: [PATCH 06/25] remove documents --- .../integration_tests/configured_catalog.json | 20 ------ .../schemas/documents.json | 40 ------------ .../source_zendesk_sell/source.py | 61 ++++++++----------- 3 files changed, 24 insertions(+), 97 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/documents.json diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/configured_catalog.json index 20c9544fa3ca..4436fdc422b3 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/configured_catalog.json @@ -66,16 +66,6 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, - { - "stream": { - "name": "custom_fields", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, { "stream": { "name": "deal_sources", @@ -96,16 +86,6 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, - { - "stream": { - "name": "documents", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, { "stream": { "name": "lead_conversions", diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/documents.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/documents.json deleted file mode 100644 index 5dd4d8ecfa91..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/documents.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "number"] - }, - "resource_type": { - "type": ["null", "string"] - }, - "resource_id": { - "type": ["null", "number"] - }, - "creator_id": { - "type": ["null", "number"] - }, - "name": { - "type": ["null", "string"] - }, - "content_type": { - "type": ["null", "string"] - }, - "size": { - "type": ["null", "number"] - }, - "download_url": { - "type": ["null", "string"] - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } - } - } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py index 5b1f5cb8ab76..14afd1f1cf9d 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py @@ -34,9 +34,9 @@ def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: if next_page_token != None: - return {'page': next_page_token['page'], 'per_page' : 100} + return {'page': next_page_token['page']} else: - return {'per_page' : 100} + return {} def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: @@ -63,33 +63,29 @@ def path(self, **kwargs) -> str: return "stages" -# # Basic incremental stream -# class IncrementalZendeskSellStream(ZendeskSellStream, ABC): -# """ -# TODO fill in details of this class to implement functionality related to incremental syncs for your connector. -# if you do not need to implement incremental sync for any streams, remove this class. -# """ - -# # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. -# state_checkpoint_interval = 100 - -# @property -# def cursor_field(self) -> str: -# """ -# TODO -# Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is -# usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. +# Basic incremental stream +class IncrementalZendeskSellStream(ZendeskSellStream, ABC): + """ + TODO fill in details of this class to implement functionality related to incremental syncs for your connector. + if you do not need to implement incremental sync for any streams, remove this class. + """ + state_checkpoint_interval = 100 -# :return str: The name of the cursor field. -# """ -# return "updated_at" + @property + def cursor_field(self) -> str: + return "updated_at" -# def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: -# """ -# Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and -# the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. -# """ -# return {} + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and + the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. + """ + if current_stream_state is not None and self.cursor_field in current_stream_state: + current_updated_at = current_stream_state[self.cursor_field] + latest_updated_at = latest_record[self.cursor_field] + return {self.cursor_field: max(current_parsed_date, latest_record_date)} + else: + return {self.cursor_field: self.cursor_field_value} class Contacts(ZendeskSellStream): @@ -164,15 +160,6 @@ class DealUnqualifiedReasons(ZendeskSellStream): def path(self, **kwargs) -> str: return "deal_unqualified_reasons" -class Documents(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/documents/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "documents" - class LeadConversions(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/lead-conversions/ @@ -306,4 +293,4 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = TokenAuthenticator(token=config["api_token"]) - return [Contacts(authenticator=auth), Deals(authenticator=auth), Leads(authenticator=auth), Pipelines(authenticator=auth), Stages(authenticator=auth), CallOutcomes(authenticator=auth), Calls(authenticator=auth), Collaborations(authenticator=auth), DealSources(authenticator=auth), DealUnqualifiedReasons(authenticator=auth), Documents(authenticator=auth), LeadConversions(authenticator=auth), LeadSources(authenticator=auth), LeadUnqualifiedReasons(authenticator=auth), LossReasons(authenticator=auth), Notes(authenticator=auth), Orders(authenticator=auth), Products(authenticator=auth), Tags(authenticator=auth), Tasks(authenticator=auth), TextMessages(authenticator=auth), Users(authenticator=auth), VisitOutcomes(authenticator=auth), Visits(authenticator=auth)] + return [Contacts(authenticator=auth), Deals(authenticator=auth), Leads(authenticator=auth), Pipelines(authenticator=auth), Stages(authenticator=auth), CallOutcomes(authenticator=auth), Calls(authenticator=auth), Collaborations(authenticator=auth), DealSources(authenticator=auth), DealUnqualifiedReasons(authenticator=auth), LeadConversions(authenticator=auth), LeadSources(authenticator=auth), LeadUnqualifiedReasons(authenticator=auth), LossReasons(authenticator=auth), Notes(authenticator=auth), Orders(authenticator=auth), Products(authenticator=auth), Tags(authenticator=auth), Tasks(authenticator=auth), TextMessages(authenticator=auth), Users(authenticator=auth), VisitOutcomes(authenticator=auth), Visits(authenticator=auth)] From b9ea2875b4cd576ea065e987e3b6456682c4d0cc Mon Sep 17 00:00:00 2001 From: alexchouraki Date: Wed, 12 Oct 2022 15:06:11 +0200 Subject: [PATCH 07/25] manage incremental syncs --- .../integration_tests/configured_catalog.json | 6 ++--- .../source_zendesk_sell/source.py | 25 +++++++++---------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/configured_catalog.json index 4436fdc422b3..3e127d892132 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/configured_catalog.json @@ -24,7 +24,7 @@ "stream": { "name": "deals", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] @@ -36,7 +36,7 @@ "stream": { "name": "contacts", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] @@ -48,7 +48,7 @@ "stream": { "name": "leads", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py index 14afd1f1cf9d..bcefe89d54fd 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py @@ -25,10 +25,13 @@ class ZendeskSellStream(HttpStream, ABC): primary_key = None def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - regex_page='[=?/]page[_=/-]?(\d{1,3})' - meta_links = response.json().get('meta', {}).get('links') - if 'next_page' in meta_links.keys(): - return {'page' : int(re.findall(regex_page, meta_links['next_page'])[0])} + try: + regex_page='[=?/]page[_=/-]?(\d{1,3})' + meta_links = response.json().get('meta', {}).get('links') + if 'next_page' in meta_links.keys(): + return {'page' : int(re.findall(regex_page, meta_links['next_page'])[0])} + except: + return None def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None @@ -65,10 +68,6 @@ def path(self, **kwargs) -> str: # Basic incremental stream class IncrementalZendeskSellStream(ZendeskSellStream, ABC): - """ - TODO fill in details of this class to implement functionality related to incremental syncs for your connector. - if you do not need to implement incremental sync for any streams, remove this class. - """ state_checkpoint_interval = 100 @property @@ -83,12 +82,12 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late if current_stream_state is not None and self.cursor_field in current_stream_state: current_updated_at = current_stream_state[self.cursor_field] latest_updated_at = latest_record[self.cursor_field] - return {self.cursor_field: max(current_parsed_date, latest_record_date)} + return {self.cursor_field: max(current_updated_at, latest_updated_at)} else: - return {self.cursor_field: self.cursor_field_value} + return {self.cursor_field: '1970-01-01T00:00:00Z'} -class Contacts(ZendeskSellStream): +class Contacts(IncrementalZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/contacts/ """ @@ -97,7 +96,7 @@ class Contacts(ZendeskSellStream): def path(self, **kwargs) -> str: return "contacts" -class Deals(ZendeskSellStream): +class Deals(IncrementalZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deals/ """ @@ -106,7 +105,7 @@ class Deals(ZendeskSellStream): def path(self, **kwargs) -> str: return "deals" -class Leads(ZendeskSellStream): +class Leads(IncrementalZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/leads/ """ From ed39a35ad9da0ee057e6a2e53cb186808f1aee2d Mon Sep 17 00:00:00 2001 From: alexchouraki Date: Thu, 13 Oct 2022 12:09:24 +0200 Subject: [PATCH 08/25] add unit tests ; add start date to specs --- .../integration_tests/sample_config.json | 3 +- .../source_zendesk_sell/spec.yaml | 11 +++++- .../unit_tests/test_incremental_streams.py | 12 +++---- .../unit_tests/test_source.py | 3 +- .../unit_tests/test_streams.py | 35 +++++++++++-------- 5 files changed, 37 insertions(+), 27 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_config.json index 9f6bd635126f..7321f9b8f455 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_config.json @@ -1,3 +1,4 @@ { - "api_token": "c56afd675afe19b87a8bf810666927baa58854fd006da22795850ee957eec854" + "api_token": "c56afd675afe19b87a8bf810666927baa58854fd006da22795850ee957eec854", + "start_date": "2020-11-12T09:05:47Z" } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml index 482ac82b73fe..2783eccaaeac 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml @@ -5,6 +5,7 @@ connectionSpecification: type: object required: - api_token + - start_date properties: api_token: title: API token @@ -12,4 +13,12 @@ connectionSpecification: description: "The API token for authenticating to Zendesk Sell" examples: - "f23yhd630otl94y85a8bf384958473pto95847fd006da49382716or937ruw059" - airbyte_secret: true \ No newline at end of file + airbyte_secret: true + start_date: + title: Start Date + type: string + description: "The date from which you'd like to replicate data for Zendesk Sell in the format YYYY-MM-DDT00:00:00Z. All data created after this date will be replicated." + examples: + - "2022-01-01T00:00:00Z" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + airbyte_secret: false \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_incremental_streams.py index ad2ba1172cc1..c0d733986ec2 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_incremental_streams.py @@ -18,17 +18,14 @@ def patch_incremental_base_class(mocker): def test_cursor_field(patch_incremental_base_class): stream = IncrementalZendeskSellStream() - # TODO: replace this with your expected cursor field - expected_cursor_field = [] + expected_cursor_field = "updated_at" assert stream.cursor_field == expected_cursor_field def test_get_updated_state(patch_incremental_base_class): stream = IncrementalZendeskSellStream() - # TODO: replace this with your input parameters - inputs = {"current_stream_state": None, "latest_record": None} - # TODO: replace this with your expected updated stream state - expected_state = {} + inputs = {"current_stream_state": {"updated_at": "2022-03-17T16:03:07Z"}, "latest_record": {"updated_at": "2022-03-18T16:03:07Z"}} + expected_state = {"updated_at": "2022-03-18T16:03:07Z"} assert stream.get_updated_state(**inputs) == expected_state @@ -54,6 +51,5 @@ def test_source_defined_cursor(patch_incremental_base_class): def test_stream_checkpoint_interval(patch_incremental_base_class): stream = IncrementalZendeskSellStream() - # TODO: replace this with your expected checkpoint interval - expected_checkpoint_interval = None + expected_checkpoint_interval = 100 assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py index b1ee046e1399..2ca3ea8825eb 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py @@ -17,6 +17,5 @@ def test_streams(mocker): source = SourceZendeskSell() config_mock = MagicMock() streams = source.streams(config_mock) - # TODO: replace this with your streams number - expected_streams_number = 2 + expected_streams_number = 23 assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py index 10da8457b19c..b2464620d1c6 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py @@ -19,43 +19,48 @@ def patch_base_class(mocker): def test_request_params(patch_base_class): stream = ZendeskSellStream() - # TODO: replace this with your input parameters inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request parameters expected_params = {} assert stream.request_params(**inputs) == expected_params -def test_next_page_token(patch_base_class): +def test_next_page_token_success(patch_base_class): stream = ZendeskSellStream() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected next page token + inputs = {"response": {'items': [{'data': {'id': 302488228, 'creator_id': 2393211, 'contact_id': 302488227, 'created_at': '2020-11-12T09:05:47Z', 'updated_at': '2022-03-23T16:53:22Z', 'title': None, 'name': 'Octavia Squidington', 'first_name': 'Octavia', 'last_name': 'Squidington'},'meta': {'version': 36, 'type': 'contact'}}], + 'meta': {'type': 'collection','count': 25,'links': {'self': 'https://api.getbase.com/v2/contacts?page=2&per_page=25', 'first_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25', 'prev_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25', 'next_page': 'https://api.getbase.com/v2/contacts?page=3&per_page=25'}}}} + expected_token = 3 + assert stream.next_page_token(**inputs) == expected_token + +def test_next_page_token_last_page(patch_base_class): + stream = ZendeskSellStream() + inputs = {"response": {'items': [{'data': {'id': 302488228, 'creator_id': 2393211, 'contact_id': 302488227, 'created_at': '2020-11-12T09:05:47Z', 'updated_at': '2022-03-23T16:53:22Z', 'title': None, 'name': 'Octavia Squidington', 'first_name': 'Octavia', 'last_name': 'Squidington'},'meta': {'version': 36, 'type': 'contact'}}], + 'meta': {'type': 'collection','count': 25,'links': {'self': 'https://api.getbase.com/v2/contacts?page=2&per_page=25', 'first_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25', 'prev_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25'}}}} expected_token = None assert stream.next_page_token(**inputs) == expected_token +def test_next_page_token_no_response(patch_base_class): + stream = ZendeskSellStream() + inputs = {None} + expected_token = None + assert stream.next_page_token(**inputs) == expected_token def test_parse_response(patch_base_class): stream = ZendeskSellStream() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected parced object - expected_parsed_object = {} + inputs = {"response": {'items': [{'data': {'id': 302488228, 'creator_id': 2393211, 'contact_id': 302488227, 'created_at': '2020-11-12T09:05:47Z', 'updated_at': '2022-03-23T16:53:22Z', 'title': None, 'name': 'Octavia Squidington', 'first_name': 'Octavia', 'last_name': 'Squidington'},'meta': {'version': 36, 'type': 'contact'}}], + 'meta': {'type': 'collection','count': 25,'links': {'self': 'https://api.getbase.com/v2/contacts?page=2&per_page=25', 'first_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25', 'prev_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25', 'next_page': 'https://api.getbase.com/v2/contacts?page=3&per_page=25'}}}} + expected_parsed_object = {'id': 302488228, 'creator_id': 2393211, 'contact_id': 302488227, 'created_at': '2020-11-12T09:05:47Z', 'updated_at': '2022-03-23T16:53:22Z', 'title': None, 'name': 'Octavia Squidington', 'first_name': 'Octavia', 'last_name': 'Squidington'} assert next(stream.parse_response(**inputs)) == expected_parsed_object def test_request_headers(patch_base_class): stream = ZendeskSellStream() - # TODO: replace this with your input parameters - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request headers - expected_headers = {} + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": 2} + expected_headers = {"page" : 2} assert stream.request_headers(**inputs) == expected_headers def test_http_method(patch_base_class): stream = ZendeskSellStream() - # TODO: replace this with your expected http request method expected_method = "GET" assert stream.http_method == expected_method From ea7adda871bb20964067acaea0b10fa5350ba766 Mon Sep 17 00:00:00 2001 From: alexchouraki Date: Thu, 13 Oct 2022 14:51:47 +0200 Subject: [PATCH 09/25] try catch for next page token method --- .../source-zendesk-sell/source_zendesk_sell/source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py index bcefe89d54fd..e0f7adb34dae 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py @@ -30,7 +30,8 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, meta_links = response.json().get('meta', {}).get('links') if 'next_page' in meta_links.keys(): return {'page' : int(re.findall(regex_page, meta_links['next_page'])[0])} - except: + except exception as e: + print(f"{e.__class__} occurred, while trying to get next page information from the following dict {meta_links}") return None def request_params( From 61a3f05e1edba733dfd78550eb3573bbcc988290 Mon Sep 17 00:00:00 2001 From: alexchouraki Date: Thu, 13 Oct 2022 14:58:35 +0200 Subject: [PATCH 10/25] Create source_to_test.py --- .../source_zendesk_sell/source_to_test.py | 305 ++++++++++++++++++ 1 file changed, 305 insertions(+) create mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source_to_test.py diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source_to_test.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source_to_test.py new file mode 100644 index 000000000000..a5b757955881 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source_to_test.py @@ -0,0 +1,305 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +import re +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams import IncrementalMixin + +# Basic full refresh stream +class ZendeskSellStream(HttpStream, ABC): + """ + This class represents a stream output by the connector. + This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, + parsing responses etc.. + """ + + url_base = "https://api.getbase.com/v2/" + primary_key = None + + def __init__(self, config: Mapping[str, Any], start_date: str, **kwargs): + super().__init__() + self.access_key = config["api_token"] + self.start_date = config["start_date"] + self._cursor_value = None + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + try: + regex_page='[=?/]page[_=/-]?(\d{1,3})' + meta_links = response.json().get('meta', {}).get('links') + if 'next_page' in meta_links.keys(): + return {'page' : int(re.findall(regex_page, meta_links['next_page'])[0])} + except exception as e: + print(f"{e.__class__} occurred, while trying to get next page information from the following dict {meta_links}") + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + if next_page_token != None: + return {'page': next_page_token['page']} + else: + return {} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + items = response.json()['items'] + return [item['data'] for item in items] + +# Basic incremental stream +class IncrementalZendeskSellStream(ZendeskSellStream, ABC): + state_checkpoint_interval = 100 + + @property + def cursor_field(self) -> str: + return "updated_at" + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and + the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. + """ + if current_stream_state is None : + return {self.cursor_field: self.start_date} + elif self.cursor_field in current_stream_state: + current_updated_at = current_stream_state[self.cursor_field] + latest_updated_at = latest_record[self.cursor_field] + return {self.cursor_field: max(current_updated_at, latest_updated_at)} + else: + return {self.cursor_field: self.start_date} + + +class Pipelines(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/pipelines/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "pipelines" + +class Stages(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/stages/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "stages" + + +class Contacts(IncrementalZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/contacts/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "contacts" + +class Deals(IncrementalZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deals/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "deals" + +class Leads(IncrementalZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/leads/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "leads" + +class CallOutcomes(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/call-outcomes/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "call_outcomes" + +class Calls(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/calls/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "calls" + +class Collaborations(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/collaborations/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "collaborations" + +class DealSources(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deal-sources/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "deal_sources" + +class DealUnqualifiedReasons(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deal-unqualified-reasons/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "deal_unqualified_reasons" + +class LeadConversions(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/lead-conversions/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "lead_conversions" + +class LeadSources(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/lead-sources/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "lead_sources" + +class LeadUnqualifiedReasons(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/lead-unqualified-reasons/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "lead_unqualified_reasons" + +class LossReasons(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/loss-reasons/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "loss_reasons" + +class Notes(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/notes/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "notes" + +class Orders(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/orders/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "orders" + +class Products(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/products/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "products" + +class Tags(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/tags/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "tags" + +class Tasks(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/tasks/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "tasks" + +class TextMessages(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/text-messages/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "text_messages" + +class Users(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/users/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "users" + +class VisitOutcomes(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/visit-outcomes/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "visit_outcomes" + +class Visits(ZendeskSellStream): + """ + Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/visits/ + """ + primary_key = "id" + + def path(self, **kwargs) -> str: + return "visits" + + +# Source +class SourceZendeskSell(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + try: + authenticator = TokenAuthenticator(token = config["api_token"]) + start_date = config['start_date'] + stream = Contacts(authenticator=authenticator, config = config, start_date = start_date) + records = stream.read_records(sync_mode=SyncMode.full_refresh) + next(records) + return True, None + except Exception as e: + return False, e + + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + auth = TokenAuthenticator(token=config["api_token"]) + start_date = config['start_date'] + return [Contacts(authenticator=auth, config = config, start_date = start_date), Deals(authenticator=auth, config = config, start_date = start_date), Leads(authenticator=auth, config = config, start_date = start_date), Pipelines(authenticator=auth, config = config, start_date = start_date), Stages(authenticator=auth, config = config, start_date = start_date), CallOutcomes(authenticator=auth, config = config, start_date = start_date), Calls(authenticator=auth, config = config, start_date = start_date), Collaborations(authenticator=auth, config = config, start_date = start_date), DealSources(authenticator=auth, config = config, start_date = start_date), DealUnqualifiedReasons(authenticator=auth, config = config, start_date = start_date), LeadConversions(authenticator=auth, config = config, start_date = start_date), LeadSources(authenticator=auth, config = config, start_date = start_date), LeadUnqualifiedReasons(authenticator=auth, config = config, start_date = start_date), LossReasons(authenticator=auth, config = config, start_date = start_date), Notes(authenticator=auth, config = config, start_date = start_date), Orders(authenticator=auth, config = config, start_date = start_date), Products(authenticator=auth, config = config, start_date = start_date), Tags(authenticator=auth, config = config, start_date = start_date), Tasks(authenticator=auth, config = config, start_date = start_date), TextMessages(authenticator=auth, config = config, start_date = start_date), Users(authenticator=auth, config = config, start_date = start_date), VisitOutcomes(authenticator=auth, config = config, start_date = start_date), Visits(authenticator=auth, config = config, start_date = start_date)] From f2b915a258a6f360da39f1b59416a430057921c6 Mon Sep 17 00:00:00 2001 From: alexchouraki Date: Thu, 13 Oct 2022 15:24:57 +0200 Subject: [PATCH 11/25] Update source.py --- .../source-zendesk-sell/source_zendesk_sell/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py index e0f7adb34dae..979f87c5c759 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py @@ -30,7 +30,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, meta_links = response.json().get('meta', {}).get('links') if 'next_page' in meta_links.keys(): return {'page' : int(re.findall(regex_page, meta_links['next_page'])[0])} - except exception as e: + except Exception as e: print(f"{e.__class__} occurred, while trying to get next page information from the following dict {meta_links}") return None From dae0abcc7fb5e9db513b7254a5821045fafeec59 Mon Sep 17 00:00:00 2001 From: alexchouraki Date: Thu, 13 Oct 2022 15:35:17 +0200 Subject: [PATCH 12/25] Update source_to_test.py --- .../source-zendesk-sell/source_zendesk_sell/source_to_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source_to_test.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source_to_test.py index a5b757955881..8786a16f3309 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source_to_test.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source_to_test.py @@ -37,7 +37,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, meta_links = response.json().get('meta', {}).get('links') if 'next_page' in meta_links.keys(): return {'page' : int(re.findall(regex_page, meta_links['next_page'])[0])} - except exception as e: + except Exception as e: print(f"{e.__class__} occurred, while trying to get next page information from the following dict {meta_links}") return None From 598f9ae617379f310437dbb54ac710b6377f7c23 Mon Sep 17 00:00:00 2001 From: alexchouraki Date: Fri, 14 Oct 2022 13:16:00 +0200 Subject: [PATCH 13/25] working source --- .../source_zendesk_sell/source.py | 51 +-- .../source_zendesk_sell/source_to_test.py | 305 ------------------ 2 files changed, 28 insertions(+), 328 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source_to_test.py diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py index 979f87c5c759..3ae0b11328d6 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py @@ -10,8 +10,9 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams import IncrementalMixin # Basic full refresh stream class ZendeskSellStream(HttpStream, ABC): @@ -24,6 +25,10 @@ class ZendeskSellStream(HttpStream, ABC): url_base = "https://api.getbase.com/v2/" primary_key = None + # def __init__(self, config: Mapping[str, Any], **kwargs): + # super().__init__() + # self.start_date = config["start_date"] + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: try: regex_page='[=?/]page[_=/-]?(\d{1,3})' @@ -43,10 +48,29 @@ def request_params( return {} def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - items = response.json()['items'] return [item['data'] for item in items] +# Basic incremental stream +class IncrementalZendeskSellStream(ZendeskSellStream, IncrementalMixin, ABC): + state_checkpoint_interval = 100 + cursor_field = "updated_at" + + # def __init__(self, **kwargs): + # super().__init__() + # self._cursor_value = None + + @property + def state(self) -> Mapping[str, Any]: + if self._cursor_value: + return {self.cursor_field: self._cursor_value} + else: + return {self.cursor_field: self.start_date} + + @state.setter + def state(self, value: Mapping[str, Any]): + self._cursor_value = value[self.cursor_field] + class Pipelines(ZendeskSellStream): """ @@ -67,27 +91,6 @@ def path(self, **kwargs) -> str: return "stages" -# Basic incremental stream -class IncrementalZendeskSellStream(ZendeskSellStream, ABC): - state_checkpoint_interval = 100 - - @property - def cursor_field(self) -> str: - return "updated_at" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and - the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. - """ - if current_stream_state is not None and self.cursor_field in current_stream_state: - current_updated_at = current_stream_state[self.cursor_field] - latest_updated_at = latest_record[self.cursor_field] - return {self.cursor_field: max(current_updated_at, latest_updated_at)} - else: - return {self.cursor_field: '1970-01-01T00:00:00Z'} - - class Contacts(IncrementalZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/contacts/ @@ -283,6 +286,7 @@ class SourceZendeskSell(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: try: authenticator = TokenAuthenticator(token = config["api_token"]) + #stream = Contacts(authenticator=authenticator, config = config) stream = Contacts(authenticator=authenticator) records = stream.read_records(sync_mode=SyncMode.full_refresh) next(records) @@ -294,3 +298,4 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = TokenAuthenticator(token=config["api_token"]) return [Contacts(authenticator=auth), Deals(authenticator=auth), Leads(authenticator=auth), Pipelines(authenticator=auth), Stages(authenticator=auth), CallOutcomes(authenticator=auth), Calls(authenticator=auth), Collaborations(authenticator=auth), DealSources(authenticator=auth), DealUnqualifiedReasons(authenticator=auth), LeadConversions(authenticator=auth), LeadSources(authenticator=auth), LeadUnqualifiedReasons(authenticator=auth), LossReasons(authenticator=auth), Notes(authenticator=auth), Orders(authenticator=auth), Products(authenticator=auth), Tags(authenticator=auth), Tasks(authenticator=auth), TextMessages(authenticator=auth), Users(authenticator=auth), VisitOutcomes(authenticator=auth), Visits(authenticator=auth)] + #return [Contacts(authenticator=auth, config = config), Deals(authenticator=auth, config = config), Leads(authenticator=auth, config = config), Pipelines(authenticator=auth, config = config), Stages(authenticator=auth, config = config), CallOutcomes(authenticator=auth, config = config), Calls(authenticator=auth, config = config), Collaborations(authenticator=auth, config = config), DealSources(authenticator=auth, config = config), DealUnqualifiedReasons(authenticator=auth, config = config), LeadConversions(authenticator=auth, config = config), LeadSources(authenticator=auth, config = config), LeadUnqualifiedReasons(authenticator=auth, config = config), LossReasons(authenticator=auth, config = config), Notes(authenticator=auth, config = config), Orders(authenticator=auth, config = config), Products(authenticator=auth, config = config), Tags(authenticator=auth, config = config), Tasks(authenticator=auth, config = config), TextMessages(authenticator=auth, config = config), Users(authenticator=auth, config = config), VisitOutcomes(authenticator=auth, config = config), Visits(authenticator=auth, config = config)] diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source_to_test.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source_to_test.py deleted file mode 100644 index 8786a16f3309..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source_to_test.py +++ /dev/null @@ -1,305 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - - -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -import re -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams import IncrementalMixin - -# Basic full refresh stream -class ZendeskSellStream(HttpStream, ABC): - """ - This class represents a stream output by the connector. - This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, - parsing responses etc.. - """ - - url_base = "https://api.getbase.com/v2/" - primary_key = None - - def __init__(self, config: Mapping[str, Any], start_date: str, **kwargs): - super().__init__() - self.access_key = config["api_token"] - self.start_date = config["start_date"] - self._cursor_value = None - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - try: - regex_page='[=?/]page[_=/-]?(\d{1,3})' - meta_links = response.json().get('meta', {}).get('links') - if 'next_page' in meta_links.keys(): - return {'page' : int(re.findall(regex_page, meta_links['next_page'])[0])} - except Exception as e: - print(f"{e.__class__} occurred, while trying to get next page information from the following dict {meta_links}") - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - if next_page_token != None: - return {'page': next_page_token['page']} - else: - return {} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - items = response.json()['items'] - return [item['data'] for item in items] - -# Basic incremental stream -class IncrementalZendeskSellStream(ZendeskSellStream, ABC): - state_checkpoint_interval = 100 - - @property - def cursor_field(self) -> str: - return "updated_at" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and - the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. - """ - if current_stream_state is None : - return {self.cursor_field: self.start_date} - elif self.cursor_field in current_stream_state: - current_updated_at = current_stream_state[self.cursor_field] - latest_updated_at = latest_record[self.cursor_field] - return {self.cursor_field: max(current_updated_at, latest_updated_at)} - else: - return {self.cursor_field: self.start_date} - - -class Pipelines(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/pipelines/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "pipelines" - -class Stages(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/stages/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "stages" - - -class Contacts(IncrementalZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/contacts/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "contacts" - -class Deals(IncrementalZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deals/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "deals" - -class Leads(IncrementalZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/leads/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "leads" - -class CallOutcomes(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/call-outcomes/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "call_outcomes" - -class Calls(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/calls/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "calls" - -class Collaborations(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/collaborations/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "collaborations" - -class DealSources(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deal-sources/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "deal_sources" - -class DealUnqualifiedReasons(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deal-unqualified-reasons/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "deal_unqualified_reasons" - -class LeadConversions(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/lead-conversions/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "lead_conversions" - -class LeadSources(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/lead-sources/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "lead_sources" - -class LeadUnqualifiedReasons(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/lead-unqualified-reasons/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "lead_unqualified_reasons" - -class LossReasons(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/loss-reasons/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "loss_reasons" - -class Notes(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/notes/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "notes" - -class Orders(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/orders/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "orders" - -class Products(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/products/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "products" - -class Tags(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/tags/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "tags" - -class Tasks(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/tasks/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "tasks" - -class TextMessages(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/text-messages/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "text_messages" - -class Users(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/users/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "users" - -class VisitOutcomes(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/visit-outcomes/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "visit_outcomes" - -class Visits(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/visits/ - """ - primary_key = "id" - - def path(self, **kwargs) -> str: - return "visits" - - -# Source -class SourceZendeskSell(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - authenticator = TokenAuthenticator(token = config["api_token"]) - start_date = config['start_date'] - stream = Contacts(authenticator=authenticator, config = config, start_date = start_date) - records = stream.read_records(sync_mode=SyncMode.full_refresh) - next(records) - return True, None - except Exception as e: - return False, e - - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = TokenAuthenticator(token=config["api_token"]) - start_date = config['start_date'] - return [Contacts(authenticator=auth, config = config, start_date = start_date), Deals(authenticator=auth, config = config, start_date = start_date), Leads(authenticator=auth, config = config, start_date = start_date), Pipelines(authenticator=auth, config = config, start_date = start_date), Stages(authenticator=auth, config = config, start_date = start_date), CallOutcomes(authenticator=auth, config = config, start_date = start_date), Calls(authenticator=auth, config = config, start_date = start_date), Collaborations(authenticator=auth, config = config, start_date = start_date), DealSources(authenticator=auth, config = config, start_date = start_date), DealUnqualifiedReasons(authenticator=auth, config = config, start_date = start_date), LeadConversions(authenticator=auth, config = config, start_date = start_date), LeadSources(authenticator=auth, config = config, start_date = start_date), LeadUnqualifiedReasons(authenticator=auth, config = config, start_date = start_date), LossReasons(authenticator=auth, config = config, start_date = start_date), Notes(authenticator=auth, config = config, start_date = start_date), Orders(authenticator=auth, config = config, start_date = start_date), Products(authenticator=auth, config = config, start_date = start_date), Tags(authenticator=auth, config = config, start_date = start_date), Tasks(authenticator=auth, config = config, start_date = start_date), TextMessages(authenticator=auth, config = config, start_date = start_date), Users(authenticator=auth, config = config, start_date = start_date), VisitOutcomes(authenticator=auth, config = config, start_date = start_date), Visits(authenticator=auth, config = config, start_date = start_date)] From 10ec349547730ee05609bdb70e708b3d94489b78 Mon Sep 17 00:00:00 2001 From: alexchouraki Date: Tue, 18 Oct 2022 19:20:32 +0200 Subject: [PATCH 14/25] Delete TODO.md --- .../source_zendesk_sell/schemas/TODO.md | 25 ------------------- 1 file changed, 25 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/TODO.md diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/TODO.md b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/TODO.md deleted file mode 100644 index cf1efadb3c9c..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/TODO.md +++ /dev/null @@ -1,25 +0,0 @@ -# TODO: Define your stream schemas -Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). - -The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. - -The schema of a stream is the return value of `Stream.get_json_schema`. - -## Static schemas -By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. - -Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. - -## Dynamic schemas -If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). - -## Dynamically modifying static schemas -Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: -``` -def get_json_schema(self): - schema = super().get_json_schema() - schema['dynamically_determined_property'] = "property" - return schema -``` - -Delete this file once you're done. Or don't. Up to you :) From a2a3ed8665200c1dc56fa4e9452056dfe2748157 Mon Sep 17 00:00:00 2001 From: alexchouraki Date: Tue, 18 Oct 2022 19:22:29 +0200 Subject: [PATCH 15/25] Update source.py --- .../source-zendesk-sell/source_zendesk_sell/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py index 3ae0b11328d6..707a21968cf7 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py @@ -36,7 +36,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, if 'next_page' in meta_links.keys(): return {'page' : int(re.findall(regex_page, meta_links['next_page'])[0])} except Exception as e: - print(f"{e.__class__} occurred, while trying to get next page information from the following dict {meta_links}") + self.logger.error(f"{e.__class__} occurred, while trying to get next page information from the following dict {meta_links}") return None def request_params( From da373f906457d5ccdda46db67a54b2f46c959b0a Mon Sep 17 00:00:00 2001 From: alexchouraki Date: Wed, 19 Oct 2022 12:09:45 +0200 Subject: [PATCH 16/25] Update spec.yaml --- .../source-zendesk-sell/source_zendesk_sell/spec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml index 2783eccaaeac..672e565c82fe 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml @@ -1,4 +1,4 @@ -documentationUrl: https://developer.zendesk.com/api-reference/sales-crm/introduction/#get-up-to-speed-with-the-sell-api +documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-sell connectionSpecification: $schema: http://json-schema.org/draft-07/schema# title: Source Zendesk Sell Spec From dffc287c3aacb3165aadba1fbcd98bd35ea8b6f3 Mon Sep 17 00:00:00 2001 From: alexchouraki Date: Wed, 19 Oct 2022 14:11:42 +0200 Subject: [PATCH 17/25] Update test_streams.py --- .../unit_tests/test_streams.py | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py index b2464620d1c6..0da2cb1a2d93 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py @@ -23,25 +23,20 @@ def test_request_params(patch_base_class): expected_params = {} assert stream.request_params(**inputs) == expected_params +@pytest.mark.parametrize( + ("inputs", "expected_token"), + [ + ({'items': [{'data': {'id': 302488228, 'creator_id': 2393211, 'contact_id': 302488227, 'created_at': '2020-11-12T09:05:47Z', 'updated_at': '2022-03-23T16:53:22Z', 'title': None, 'name': 'Octavia Squidington', 'first_name': 'Octavia', 'last_name': 'Squidington'},'meta': {'version': 36, 'type': 'contact'}}], + 'meta': {'type': 'collection','count': 25,'links': {'self': 'https://api.getbase.com/v2/contacts?page=2&per_page=25', 'first_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25', 'prev_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25', 'next_page': 'https://api.getbase.com/v2/contacts?page=3&per_page=25'}}}, 3), + ({'items': [{'data': {'id': 302488228, 'creator_id': 2393211, 'contact_id': 302488227, 'created_at': '2020-11-12T09:05:47Z', 'updated_at': '2022-03-23T16:53:22Z', 'title': None, 'name': 'Octavia Squidington', 'first_name': 'Octavia', 'last_name': 'Squidington'},'meta': {'version': 36, 'type': 'contact'}}], + 'meta': {'type': 'collection','count': 25,'links': {'self': 'https://api.getbase.com/v2/contacts?page=2&per_page=25', 'first_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25', 'prev_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25'}}}, None), + ({None}, None), + ], +) -def test_next_page_token_success(patch_base_class): - stream = ZendeskSellStream() - inputs = {"response": {'items': [{'data': {'id': 302488228, 'creator_id': 2393211, 'contact_id': 302488227, 'created_at': '2020-11-12T09:05:47Z', 'updated_at': '2022-03-23T16:53:22Z', 'title': None, 'name': 'Octavia Squidington', 'first_name': 'Octavia', 'last_name': 'Squidington'},'meta': {'version': 36, 'type': 'contact'}}], - 'meta': {'type': 'collection','count': 25,'links': {'self': 'https://api.getbase.com/v2/contacts?page=2&per_page=25', 'first_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25', 'prev_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25', 'next_page': 'https://api.getbase.com/v2/contacts?page=3&per_page=25'}}}} - expected_token = 3 - assert stream.next_page_token(**inputs) == expected_token - -def test_next_page_token_last_page(patch_base_class): - stream = ZendeskSellStream() - inputs = {"response": {'items': [{'data': {'id': 302488228, 'creator_id': 2393211, 'contact_id': 302488227, 'created_at': '2020-11-12T09:05:47Z', 'updated_at': '2022-03-23T16:53:22Z', 'title': None, 'name': 'Octavia Squidington', 'first_name': 'Octavia', 'last_name': 'Squidington'},'meta': {'version': 36, 'type': 'contact'}}], - 'meta': {'type': 'collection','count': 25,'links': {'self': 'https://api.getbase.com/v2/contacts?page=2&per_page=25', 'first_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25', 'prev_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25'}}}} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - -def test_next_page_token_no_response(patch_base_class): +def test_next_page_token(patch_base_class, inputs, expected_token): stream = ZendeskSellStream() - inputs = {None} - expected_token = None + inputs = {"response": MagicMock()} assert stream.next_page_token(**inputs) == expected_token def test_parse_response(patch_base_class): From c7cfdb6cb4bddee58e55482445b68604ea9da5de Mon Sep 17 00:00:00 2001 From: alexchouraki Date: Wed, 19 Oct 2022 18:57:47 +0200 Subject: [PATCH 18/25] give up on incremental and have something that works --- .../source_zendesk_sell/source.py | 35 ++---------- .../unit_tests/test_incremental_streams.py | 55 ------------------- 2 files changed, 4 insertions(+), 86 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_incremental_streams.py diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py index 707a21968cf7..340e0314a9f5 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py @@ -25,10 +25,6 @@ class ZendeskSellStream(HttpStream, ABC): url_base = "https://api.getbase.com/v2/" primary_key = None - # def __init__(self, config: Mapping[str, Any], **kwargs): - # super().__init__() - # self.start_date = config["start_date"] - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: try: regex_page='[=?/]page[_=/-]?(\d{1,3})' @@ -51,27 +47,6 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp items = response.json()['items'] return [item['data'] for item in items] -# Basic incremental stream -class IncrementalZendeskSellStream(ZendeskSellStream, IncrementalMixin, ABC): - state_checkpoint_interval = 100 - cursor_field = "updated_at" - - # def __init__(self, **kwargs): - # super().__init__() - # self._cursor_value = None - - @property - def state(self) -> Mapping[str, Any]: - if self._cursor_value: - return {self.cursor_field: self._cursor_value} - else: - return {self.cursor_field: self.start_date} - - @state.setter - def state(self, value: Mapping[str, Any]): - self._cursor_value = value[self.cursor_field] - - class Pipelines(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/pipelines/ @@ -91,7 +66,7 @@ def path(self, **kwargs) -> str: return "stages" -class Contacts(IncrementalZendeskSellStream): +class Contacts(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/contacts/ """ @@ -100,7 +75,7 @@ class Contacts(IncrementalZendeskSellStream): def path(self, **kwargs) -> str: return "contacts" -class Deals(IncrementalZendeskSellStream): +class Deals(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deals/ """ @@ -109,7 +84,7 @@ class Deals(IncrementalZendeskSellStream): def path(self, **kwargs) -> str: return "deals" -class Leads(IncrementalZendeskSellStream): +class Leads(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/leads/ """ @@ -286,7 +261,6 @@ class SourceZendeskSell(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: try: authenticator = TokenAuthenticator(token = config["api_token"]) - #stream = Contacts(authenticator=authenticator, config = config) stream = Contacts(authenticator=authenticator) records = stream.read_records(sync_mode=SyncMode.full_refresh) next(records) @@ -297,5 +271,4 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = TokenAuthenticator(token=config["api_token"]) - return [Contacts(authenticator=auth), Deals(authenticator=auth), Leads(authenticator=auth), Pipelines(authenticator=auth), Stages(authenticator=auth), CallOutcomes(authenticator=auth), Calls(authenticator=auth), Collaborations(authenticator=auth), DealSources(authenticator=auth), DealUnqualifiedReasons(authenticator=auth), LeadConversions(authenticator=auth), LeadSources(authenticator=auth), LeadUnqualifiedReasons(authenticator=auth), LossReasons(authenticator=auth), Notes(authenticator=auth), Orders(authenticator=auth), Products(authenticator=auth), Tags(authenticator=auth), Tasks(authenticator=auth), TextMessages(authenticator=auth), Users(authenticator=auth), VisitOutcomes(authenticator=auth), Visits(authenticator=auth)] - #return [Contacts(authenticator=auth, config = config), Deals(authenticator=auth, config = config), Leads(authenticator=auth, config = config), Pipelines(authenticator=auth, config = config), Stages(authenticator=auth, config = config), CallOutcomes(authenticator=auth, config = config), Calls(authenticator=auth, config = config), Collaborations(authenticator=auth, config = config), DealSources(authenticator=auth, config = config), DealUnqualifiedReasons(authenticator=auth, config = config), LeadConversions(authenticator=auth, config = config), LeadSources(authenticator=auth, config = config), LeadUnqualifiedReasons(authenticator=auth, config = config), LossReasons(authenticator=auth, config = config), Notes(authenticator=auth, config = config), Orders(authenticator=auth, config = config), Products(authenticator=auth, config = config), Tags(authenticator=auth, config = config), Tasks(authenticator=auth, config = config), TextMessages(authenticator=auth, config = config), Users(authenticator=auth, config = config), VisitOutcomes(authenticator=auth, config = config), Visits(authenticator=auth, config = config)] + return [Contacts(authenticator=auth), Deals(authenticator=auth), Leads(authenticator=auth), Pipelines(authenticator=auth), Stages(authenticator=auth), CallOutcomes(authenticator=auth), Calls(authenticator=auth), Collaborations(authenticator=auth), DealSources(authenticator=auth), DealUnqualifiedReasons(authenticator=auth), LeadConversions(authenticator=auth), LeadSources(authenticator=auth), LeadUnqualifiedReasons(authenticator=auth), LossReasons(authenticator=auth), Notes(authenticator=auth), Orders(authenticator=auth), Products(authenticator=auth), Tags(authenticator=auth), Tasks(authenticator=auth), TextMessages(authenticator=auth), Users(authenticator=auth), VisitOutcomes(authenticator=auth), Visits(authenticator=auth)] \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_incremental_streams.py deleted file mode 100644 index c0d733986ec2..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_incremental_streams.py +++ /dev/null @@ -1,55 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - - -from airbyte_cdk.models import SyncMode -from pytest import fixture -from source_zendesk_sell.source import IncrementalZendeskSellStream - - -@fixture -def patch_incremental_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(IncrementalZendeskSellStream, "path", "v0/example_endpoint") - mocker.patch.object(IncrementalZendeskSellStream, "primary_key", "test_primary_key") - mocker.patch.object(IncrementalZendeskSellStream, "__abstractmethods__", set()) - - -def test_cursor_field(patch_incremental_base_class): - stream = IncrementalZendeskSellStream() - expected_cursor_field = "updated_at" - assert stream.cursor_field == expected_cursor_field - - -def test_get_updated_state(patch_incremental_base_class): - stream = IncrementalZendeskSellStream() - inputs = {"current_stream_state": {"updated_at": "2022-03-17T16:03:07Z"}, "latest_record": {"updated_at": "2022-03-18T16:03:07Z"}} - expected_state = {"updated_at": "2022-03-18T16:03:07Z"} - assert stream.get_updated_state(**inputs) == expected_state - - -def test_stream_slices(patch_incremental_base_class): - stream = IncrementalZendeskSellStream() - # TODO: replace this with your input parameters - inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} - # TODO: replace this with your expected stream slices list - expected_stream_slice = [None] - assert stream.stream_slices(**inputs) == expected_stream_slice - - -def test_supports_incremental(patch_incremental_base_class, mocker): - mocker.patch.object(IncrementalZendeskSellStream, "cursor_field", "dummy_field") - stream = IncrementalZendeskSellStream() - assert stream.supports_incremental - - -def test_source_defined_cursor(patch_incremental_base_class): - stream = IncrementalZendeskSellStream() - assert stream.source_defined_cursor - - -def test_stream_checkpoint_interval(patch_incremental_base_class): - stream = IncrementalZendeskSellStream() - expected_checkpoint_interval = 100 - assert stream.state_checkpoint_interval == expected_checkpoint_interval From 3604d630bf8a46ab8f07a4bdf13c0741a3b04901 Mon Sep 17 00:00:00 2001 From: alexchouraki Date: Thu, 20 Oct 2022 16:05:03 +0200 Subject: [PATCH 19/25] removing start date necessity in spec and sample configs --- .../integration_tests/invalid_config.json | 2 +- .../integration_tests/sample_config.json | 3 +-- .../source-zendesk-sell/source_zendesk_sell/spec.yaml | 11 +---------- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/invalid_config.json index f65c5e3c6e87..cb2463e43ef3 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/invalid_config.json @@ -1,3 +1,3 @@ { - "api_token": "f23yhd630otl94y85a8bf384958473pto95847fd006da49382716or937ruw059" + "api_token": "" } diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_config.json index 7321f9b8f455..9f6bd635126f 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_config.json @@ -1,4 +1,3 @@ { - "api_token": "c56afd675afe19b87a8bf810666927baa58854fd006da22795850ee957eec854", - "start_date": "2020-11-12T09:05:47Z" + "api_token": "c56afd675afe19b87a8bf810666927baa58854fd006da22795850ee957eec854" } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml index 672e565c82fe..75f6198267c4 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml @@ -5,7 +5,6 @@ connectionSpecification: type: object required: - api_token - - start_date properties: api_token: title: API token @@ -13,12 +12,4 @@ connectionSpecification: description: "The API token for authenticating to Zendesk Sell" examples: - "f23yhd630otl94y85a8bf384958473pto95847fd006da49382716or937ruw059" - airbyte_secret: true - start_date: - title: Start Date - type: string - description: "The date from which you'd like to replicate data for Zendesk Sell in the format YYYY-MM-DDT00:00:00Z. All data created after this date will be replicated." - examples: - - "2022-01-01T00:00:00Z" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - airbyte_secret: false \ No newline at end of file + airbyte_secret: true \ No newline at end of file From f1367469be9527a92b9fa6fc0b7de1197aff2cbe Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 27 Oct 2022 08:09:23 -0300 Subject: [PATCH 20/25] add connector to source def --- .../init/src/main/resources/seed/source_definitions.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 4f8608cba5f6..efa4563f8e3a 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1173,6 +1173,14 @@ icon: zendesk.svg sourceType: api releaseStage: generally_available +- name: Zendesk Sell + sourceDefinitionId: 982eaa4c-bba1-4cce-a971-06a41f700b8c + dockerRepository: airbyte/source-zendesk-sell + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-sell + icon: zendesk.svg + sourceType: api + releaseStage: alpha - name: Zendesk Sunshine sourceDefinitionId: 325e0640-e7b3-4e24-b823-3361008f603f dockerRepository: airbyte/source-zendesk-sunshine From 389a97521a42bdfd6612258f4568dfa400202941 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 27 Oct 2022 08:29:56 -0300 Subject: [PATCH 21/25] run format --- .../integration_tests/acceptance.py | 2 +- .../integration_tests/sample_config.json | 2 +- .../source_zendesk_sell/schemas/calls.json | 110 +++++----- .../source_zendesk_sell/schemas/contacts.json | 196 +++++++++--------- .../source_zendesk_sell/schemas/deals.json | 170 +++++++-------- .../source_zendesk_sell/schemas/leads.json | 166 +++++++-------- .../source_zendesk_sell/schemas/notes.json | 76 +++---- .../schemas/pipelines.json | 2 +- .../source_zendesk_sell/schemas/products.json | 88 ++++---- .../source_zendesk_sell/schemas/tasks.json | 102 ++++----- .../schemas/text_messages.json | 92 ++++---- .../source_zendesk_sell/schemas/users.json | 128 ++++++------ .../source_zendesk_sell/schemas/visits.json | 86 ++++---- .../source_zendesk_sell/source.py | 97 +++++++-- .../source_zendesk_sell/spec.yaml | 4 +- .../unit_tests/test_streams.py | 114 +++++++++- docs/integrations/sources/zendesk-sell.md | 79 +++++++ 17 files changed, 879 insertions(+), 635 deletions(-) create mode 100644 docs/integrations/sources/zendesk-sell.md diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py index cd2dc4aaffb2..950b53b59d41 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py @@ -11,4 +11,4 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" - yield \ No newline at end of file + yield diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_config.json index 9f6bd635126f..84230b311b70 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/sample_config.json @@ -1,3 +1,3 @@ { "api_token": "c56afd675afe19b87a8bf810666927baa58854fd006da22795850ee957eec854" -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/calls.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/calls.json index a18616afa129..94612ce6a249 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/calls.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/calls.json @@ -1,57 +1,57 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "number"] - }, - "user_id": { - "type": ["null", "number"] - }, - "summary": { - "type": ["null", "string"] - }, - "recording_url": { - "type": ["null", "string"] - }, - "outcome_id": { - "type": ["null", "number"] - }, - "duration": { - "type": ["null", "number"] - }, - "phone_number": { - "type": ["null", "number"] - }, - "incoming": { - "type": ["null", "boolean"] - }, - "missed": { - "type": ["null", "boolean"] - }, - "resource_type": { - "type": ["null", "string"] - }, - "resource_id": { - "type": ["null", "number"] - }, - "associated_deal_ids": { - "type": ["null", "array"] - }, - "made_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "user_id": { + "type": ["null", "number"] + }, + "summary": { + "type": ["null", "string"] + }, + "recording_url": { + "type": ["null", "string"] + }, + "outcome_id": { + "type": ["null", "number"] + }, + "duration": { + "type": ["null", "number"] + }, + "phone_number": { + "type": ["null", "number"] + }, + "incoming": { + "type": ["null", "boolean"] + }, + "missed": { + "type": ["null", "boolean"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "resource_id": { + "type": ["null", "number"] + }, + "associated_deal_ids": { + "type": ["null", "array"] + }, + "made_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" } - } \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/contacts.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/contacts.json index fd55445cc198..022461fce783 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/contacts.json @@ -1,100 +1,100 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "number"] - }, - "creator_id": { - "type": ["null", "number"] - }, - "owner_id": { - "type": ["null", "number"] - }, - "is_organization": { - "type": ["null", "boolean"] - }, - "contact_id": { - "type": ["null", "number"] - }, - "parent_organization_id": { - "type": ["null", "number"] - }, - "name": { - "type": ["null", "string"] - }, - "first_name": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "customer_status": { - "type": ["null", "string"] - }, - "prospect_status": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "industry": { - "type": ["null", "string"] - }, - "website": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "mobile": { - "type": ["null", "string"] - }, - "fax": { - "type": ["null", "string"] - }, - "twitter": { - "type": ["null", "string"] - }, - "facebook": { - "type": ["null", "string"] - }, - "linkedin": { - "type": ["null", "string"] - }, - "skype": { - "type": ["null", "string"] - }, - "address": { - "type": ["null", "string"] - }, - "billing_address": { - "type": ["null", "string"] - }, - "shipping_address": { - "type": ["null", "string"] - }, - "tags": { - "type": ["null", "array"] - }, - "custom_fields": { - "type": ["null", "object"] - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "owner_id": { + "type": ["null", "number"] + }, + "is_organization": { + "type": ["null", "boolean"] + }, + "contact_id": { + "type": ["null", "number"] + }, + "parent_organization_id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "customer_status": { + "type": ["null", "string"] + }, + "prospect_status": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "industry": { + "type": ["null", "string"] + }, + "website": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "mobile": { + "type": ["null", "string"] + }, + "fax": { + "type": ["null", "string"] + }, + "twitter": { + "type": ["null", "string"] + }, + "facebook": { + "type": ["null", "string"] + }, + "linkedin": { + "type": ["null", "string"] + }, + "skype": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "billing_address": { + "type": ["null", "string"] + }, + "shipping_address": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "array"] + }, + "custom_fields": { + "type": ["null", "object"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" } - } \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deals.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deals.json index a96c3a32059d..c0f3104de75a 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deals.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deals.json @@ -1,87 +1,87 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "number"] - }, - "creator_id": { - "type": ["null", "number"] - }, - "owner_id": { - "type": ["null", "number"] - }, - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "number", "string"] - }, - "currency": { - "type": ["null", "string"] - }, - "hot": { - "type": ["null", "boolean"] - }, - "stage_id": { - "type": ["null", "number"] - }, - "last_stage_change_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "last_stage_change_by_id": { - "type": ["null", "number"] - }, - "last_activity_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "source_id": { - "type": ["null", "number"] - }, - "loss_reason_id": { - "type": ["null", "number"] - }, - "unqualified_reason_id": { - "type": ["null", "number"] - }, - "dropbox_email": { - "type": ["null", "string"] - }, - "contact_id": { - "type": ["null", "number"] - }, - "organization_id": { - "type": ["null", "number"] - }, - "estimated_close_date": { - "type": ["null", "string"] - }, - "customized_win_likelihood": { - "type": ["null", "number"] - }, - "tags": { - "type": ["null", "array"] - }, - "custom_fields": { - "type": ["null", "object"] - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "added_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "owner_id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "number", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "hot": { + "type": ["null", "boolean"] + }, + "stage_id": { + "type": ["null", "number"] + }, + "last_stage_change_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "last_stage_change_by_id": { + "type": ["null", "number"] + }, + "last_activity_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "source_id": { + "type": ["null", "number"] + }, + "loss_reason_id": { + "type": ["null", "number"] + }, + "unqualified_reason_id": { + "type": ["null", "number"] + }, + "dropbox_email": { + "type": ["null", "string"] + }, + "contact_id": { + "type": ["null", "number"] + }, + "organization_id": { + "type": ["null", "number"] + }, + "estimated_close_date": { + "type": ["null", "string"] + }, + "customized_win_likelihood": { + "type": ["null", "number"] + }, + "tags": { + "type": ["null", "array"] + }, + "custom_fields": { + "type": ["null", "object"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "added_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" } - } \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/leads.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/leads.json index 509fb4cf2cf9..d31d0c53d6a0 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/leads.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/leads.json @@ -1,85 +1,85 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "number"] - }, - "creator_id": { - "type": ["null", "number"] - }, - "owner_id": { - "type": ["null", "number"] - }, - "first_name": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "organization_name": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "source_id": { - "type": ["null", "number"] - }, - "title": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "industry": { - "type": ["null", "string"] - }, - "website": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "mobile": { - "type": ["null", "string"] - }, - "fax": { - "type": ["null", "string"] - }, - "twitter": { - "type": ["null", "string"] - }, - "facebook": { - "type": ["null", "string"] - }, - "linkedin": { - "type": ["null", "string"] - }, - "skype": { - "type": ["null", "string"] - }, - "address": { - "type": ["null", "string"] - }, - "tags": { - "type": ["null", "array"] - }, - "custom_fields": { - "type": ["null", "object"] - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "owner_id": { + "type": ["null", "number"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "organization_name": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "source_id": { + "type": ["null", "number"] + }, + "title": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "industry": { + "type": ["null", "string"] + }, + "website": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "mobile": { + "type": ["null", "string"] + }, + "fax": { + "type": ["null", "string"] + }, + "twitter": { + "type": ["null", "string"] + }, + "facebook": { + "type": ["null", "string"] + }, + "linkedin": { + "type": ["null", "string"] + }, + "skype": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "array"] + }, + "custom_fields": { + "type": ["null", "object"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" } - } \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/notes.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/notes.json index d2ea84d6f02f..3ac4eb3fe957 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/notes.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/notes.json @@ -1,40 +1,40 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "number"] - }, - "creator_id": { - "type": ["null", "number"] - }, - "resource_type": { - "type": ["null", "string"] - }, - "resource_id": { - "type": ["null", "number"] - }, - "content": { - "type": ["null", "string"] - }, - "is_important": { - "type": ["null", "boolean"] - }, - "tags": { - "type": ["null", "array"] - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "type": { - "type": ["null", "string"] - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "resource_id": { + "type": ["null", "number"] + }, + "content": { + "type": ["null", "string"] + }, + "is_important": { + "type": ["null", "boolean"] + }, + "tags": { + "type": ["null", "array"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "type": { + "type": ["null", "string"] } - } \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/pipelines.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/pipelines.json index cd44ef4aed34..6d25e0550eef 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/pipelines.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/pipelines.json @@ -17,7 +17,7 @@ "type": ["null", "string"], "format": "date-time", "airbyte_type": "timestamp_with_timezone" - }, + }, "disabled": { "type": ["null", "boolean"] } diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/products.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/products.json index 08fa23b528ca..47f7873f7ec0 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/products.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/products.json @@ -1,46 +1,46 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "number"] - }, - "name": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "sku": { - "type": ["null", "string"] - }, - "active": { - "type": ["null", "boolean"] - }, - "max_discount": { - "type": ["null", "number"] - }, - "max_markup": { - "type": ["null", "number"] - }, - "cost": { - "type": ["null", "number"] - }, - "cost_currency": { - "type": ["null", "string"] - }, - "prices": { - "type": ["null", "array"] - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "sku": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "max_discount": { + "type": ["null", "number"] + }, + "max_markup": { + "type": ["null", "number"] + }, + "cost": { + "type": ["null", "number"] + }, + "cost_currency": { + "type": ["null", "string"] + }, + "prices": { + "type": ["null", "array"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" } - } \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tasks.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tasks.json index 5ab1af48147e..66a456a01b3e 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tasks.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tasks.json @@ -1,53 +1,53 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "number"] - }, - "creator_id": { - "type": ["null", "number"] - }, - "owner_id": { - "type": ["null", "number"] - }, - "resource_type": { - "type": ["null", "string"] - }, - "resource_id": { - "type": ["null", "number"] - }, - "completed": { - "type": ["null", "boolean"] - }, - "completed_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "due_date": { - "type": ["null", "string"] - }, - "overdue": { - "type": ["null", "boolean"] - }, - "remind_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "content": { - "type": ["null", "string"] - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "owner_id": { + "type": ["null", "number"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "resource_id": { + "type": ["null", "number"] + }, + "completed": { + "type": ["null", "boolean"] + }, + "completed_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "due_date": { + "type": ["null", "string"] + }, + "overdue": { + "type": ["null", "boolean"] + }, + "remind_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "content": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" } - } \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/text_messages.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/text_messages.json index fe1532cfd06c..2fbbc8db5c0e 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/text_messages.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/text_messages.json @@ -1,48 +1,48 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "number"] - }, - "associated_deal_ids": { - "type": ["null", "array"] - }, - "content": { - "type": ["null", "string"] - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "incoming": { - "type": ["null", "boolean"] - }, - "resource_type": { - "type": ["null", "string"] - }, - "resource_id": { - "type": ["null", "number"] - }, - "resource_phone_number": { - "type": ["null", "string"] - }, - "sent_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "user_id": { - "type": ["null", "number"] - }, - "user_phone_number": { - "type": ["null", "string"] - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "associated_deal_ids": { + "type": ["null", "array"] + }, + "content": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "incoming": { + "type": ["null", "boolean"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "resource_id": { + "type": ["null", "number"] + }, + "resource_phone_number": { + "type": ["null", "string"] + }, + "sent_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "user_id": { + "type": ["null", "number"] + }, + "user_phone_number": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" } - } \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/users.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/users.json index a3869e2839bf..1c8c43003290 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/users.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/users.json @@ -1,66 +1,66 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "number"] - }, - "name": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "invited": { - "type": ["null", "boolean"] - }, - "confirmed": { - "type": ["null", "boolean"] - }, - "phone_number": { - "type": ["null", "string"] - }, - "role": { - "type": ["null", "string"] - }, - "roles": { - "type": ["null", "array"] - }, - "team_name": { - "type": ["null", "string"] - }, - "group": { - "type": ["null", "object"] - }, - "reports_to": { - "type": ["null", "number"] - }, - "timezone": { - "type": ["null", "string"] - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "deleted_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "zendesk_user_id": { - "type": ["null", "string"] - }, - "identity_type": { - "type": ["null", "string"] - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "invited": { + "type": ["null", "boolean"] + }, + "confirmed": { + "type": ["null", "boolean"] + }, + "phone_number": { + "type": ["null", "string"] + }, + "role": { + "type": ["null", "string"] + }, + "roles": { + "type": ["null", "array"] + }, + "team_name": { + "type": ["null", "string"] + }, + "group": { + "type": ["null", "object"] + }, + "reports_to": { + "type": ["null", "number"] + }, + "timezone": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "deleted_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "zendesk_user_id": { + "type": ["null", "string"] + }, + "identity_type": { + "type": ["null", "string"] } - } \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visits.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visits.json index ead884e63083..5118a43d1929 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visits.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visits.json @@ -1,45 +1,45 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "number"] - }, - "creator_id": { - "type": ["null", "number"] - }, - "outcome_id": { - "type": ["null", "number"] - }, - "resource_type": { - "type": ["null", "string"] - }, - "resource_id": { - "type": ["null", "number"] - }, - "visited_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "resource_address": { - "type": ["null", "string"] - }, - "rep_location_verification_status": { - "type": ["null", "string"] - }, - "summary": { - "type": ["null", "string"] - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "creator_id": { + "type": ["null", "number"] + }, + "outcome_id": { + "type": ["null", "number"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "resource_id": { + "type": ["null", "number"] + }, + "visited_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "resource_address": { + "type": ["null", "string"] + }, + "rep_location_verification_status": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" } - } \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py index 340e0314a9f5..b3c87ce17e57 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py @@ -3,16 +3,17 @@ # +import re from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -import re + import requests +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams import IncrementalMixin + # Basic full refresh stream class ZendeskSellStream(HttpStream, ABC): @@ -27,10 +28,10 @@ class ZendeskSellStream(HttpStream, ABC): def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: try: - regex_page='[=?/]page[_=/-]?(\d{1,3})' - meta_links = response.json().get('meta', {}).get('links') - if 'next_page' in meta_links.keys(): - return {'page' : int(re.findall(regex_page, meta_links['next_page'])[0])} + regex_page = r"[=?/]page[_=/-]?(\d{1,3})" + meta_links = response.json().get("meta", {}).get("links") + if "next_page" in meta_links.keys(): + return {"page": int(re.findall(regex_page, meta_links["next_page"])[0])} except Exception as e: self.logger.error(f"{e.__class__} occurred, while trying to get next page information from the following dict {meta_links}") return None @@ -38,28 +39,32 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - if next_page_token != None: - return {'page': next_page_token['page']} + if next_page_token: + return {"page": next_page_token["page"]} else: return {} def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - items = response.json()['items'] - return [item['data'] for item in items] + items = response.json()["items"] + return [item["data"] for item in items] + class Pipelines(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/pipelines/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "pipelines" + class Stages(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/stages/ """ + primary_key = "id" def path(self, **kwargs) -> str: @@ -70,186 +75,227 @@ class Contacts(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/contacts/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "contacts" + class Deals(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deals/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "deals" + class Leads(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/leads/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "leads" + class CallOutcomes(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/call-outcomes/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "call_outcomes" + class Calls(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/calls/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "calls" + class Collaborations(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/collaborations/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "collaborations" + class DealSources(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deal-sources/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "deal_sources" + class DealUnqualifiedReasons(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deal-unqualified-reasons/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "deal_unqualified_reasons" + class LeadConversions(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/lead-conversions/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "lead_conversions" + class LeadSources(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/lead-sources/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "lead_sources" + class LeadUnqualifiedReasons(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/lead-unqualified-reasons/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "lead_unqualified_reasons" + class LossReasons(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/loss-reasons/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "loss_reasons" + class Notes(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/notes/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "notes" + class Orders(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/orders/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "orders" + class Products(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/products/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "products" + class Tags(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/tags/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "tags" + class Tasks(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/tasks/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "tasks" + class TextMessages(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/text-messages/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "text_messages" + class Users(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/users/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "users" + class VisitOutcomes(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/visit-outcomes/ """ + primary_key = "id" def path(self, **kwargs) -> str: return "visit_outcomes" + class Visits(ZendeskSellStream): """ Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/visits/ """ + primary_key = "id" def path(self, **kwargs) -> str: @@ -260,7 +306,7 @@ def path(self, **kwargs) -> str: class SourceZendeskSell(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: try: - authenticator = TokenAuthenticator(token = config["api_token"]) + authenticator = TokenAuthenticator(token=config["api_token"]) stream = Contacts(authenticator=authenticator) records = stream.read_records(sync_mode=SyncMode.full_refresh) next(records) @@ -268,7 +314,30 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: except Exception as e: return False, e - def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = TokenAuthenticator(token=config["api_token"]) - return [Contacts(authenticator=auth), Deals(authenticator=auth), Leads(authenticator=auth), Pipelines(authenticator=auth), Stages(authenticator=auth), CallOutcomes(authenticator=auth), Calls(authenticator=auth), Collaborations(authenticator=auth), DealSources(authenticator=auth), DealUnqualifiedReasons(authenticator=auth), LeadConversions(authenticator=auth), LeadSources(authenticator=auth), LeadUnqualifiedReasons(authenticator=auth), LossReasons(authenticator=auth), Notes(authenticator=auth), Orders(authenticator=auth), Products(authenticator=auth), Tags(authenticator=auth), Tasks(authenticator=auth), TextMessages(authenticator=auth), Users(authenticator=auth), VisitOutcomes(authenticator=auth), Visits(authenticator=auth)] \ No newline at end of file + return [ + Contacts(authenticator=auth), + Deals(authenticator=auth), + Leads(authenticator=auth), + Pipelines(authenticator=auth), + Stages(authenticator=auth), + CallOutcomes(authenticator=auth), + Calls(authenticator=auth), + Collaborations(authenticator=auth), + DealSources(authenticator=auth), + DealUnqualifiedReasons(authenticator=auth), + LeadConversions(authenticator=auth), + LeadSources(authenticator=auth), + LeadUnqualifiedReasons(authenticator=auth), + LossReasons(authenticator=auth), + Notes(authenticator=auth), + Orders(authenticator=auth), + Products(authenticator=auth), + Tags(authenticator=auth), + Tasks(authenticator=auth), + TextMessages(authenticator=auth), + Users(authenticator=auth), + VisitOutcomes(authenticator=auth), + Visits(authenticator=auth), + ] diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml index 75f6198267c4..eb92f5a7083e 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml @@ -10,6 +10,6 @@ connectionSpecification: title: API token type: string description: "The API token for authenticating to Zendesk Sell" - examples: + examples: - "f23yhd630otl94y85a8bf384958473pto95847fd006da49382716or937ruw059" - airbyte_secret: true \ No newline at end of file + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py index 0da2cb1a2d93..a23072811cf4 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py @@ -23,34 +23,130 @@ def test_request_params(patch_base_class): expected_params = {} assert stream.request_params(**inputs) == expected_params + @pytest.mark.parametrize( ("inputs", "expected_token"), [ - ({'items': [{'data': {'id': 302488228, 'creator_id': 2393211, 'contact_id': 302488227, 'created_at': '2020-11-12T09:05:47Z', 'updated_at': '2022-03-23T16:53:22Z', 'title': None, 'name': 'Octavia Squidington', 'first_name': 'Octavia', 'last_name': 'Squidington'},'meta': {'version': 36, 'type': 'contact'}}], - 'meta': {'type': 'collection','count': 25,'links': {'self': 'https://api.getbase.com/v2/contacts?page=2&per_page=25', 'first_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25', 'prev_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25', 'next_page': 'https://api.getbase.com/v2/contacts?page=3&per_page=25'}}}, 3), - ({'items': [{'data': {'id': 302488228, 'creator_id': 2393211, 'contact_id': 302488227, 'created_at': '2020-11-12T09:05:47Z', 'updated_at': '2022-03-23T16:53:22Z', 'title': None, 'name': 'Octavia Squidington', 'first_name': 'Octavia', 'last_name': 'Squidington'},'meta': {'version': 36, 'type': 'contact'}}], - 'meta': {'type': 'collection','count': 25,'links': {'self': 'https://api.getbase.com/v2/contacts?page=2&per_page=25', 'first_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25', 'prev_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25'}}}, None), + ( + { + "items": [ + { + "data": { + "id": 302488228, + "creator_id": 2393211, + "contact_id": 302488227, + "created_at": "2020-11-12T09:05:47Z", + "updated_at": "2022-03-23T16:53:22Z", + "title": None, + "name": "Octavia Squidington", + "first_name": "Octavia", + "last_name": "Squidington", + }, + "meta": {"version": 36, "type": "contact"}, + } + ], + "meta": { + "type": "collection", + "count": 25, + "links": { + "self": "https://api.getbase.com/v2/contacts?page=2&per_page=25", + "first_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", + "prev_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", + "next_page": "https://api.getbase.com/v2/contacts?page=3&per_page=25", + }, + }, + }, + 3, + ), + ( + { + "items": [ + { + "data": { + "id": 302488228, + "creator_id": 2393211, + "contact_id": 302488227, + "created_at": "2020-11-12T09:05:47Z", + "updated_at": "2022-03-23T16:53:22Z", + "title": None, + "name": "Octavia Squidington", + "first_name": "Octavia", + "last_name": "Squidington", + }, + "meta": {"version": 36, "type": "contact"}, + } + ], + "meta": { + "type": "collection", + "count": 25, + "links": { + "self": "https://api.getbase.com/v2/contacts?page=2&per_page=25", + "first_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", + "prev_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", + }, + }, + }, + None, + ), ({None}, None), ], ) - def test_next_page_token(patch_base_class, inputs, expected_token): stream = ZendeskSellStream() inputs = {"response": MagicMock()} assert stream.next_page_token(**inputs) == expected_token + def test_parse_response(patch_base_class): stream = ZendeskSellStream() - inputs = {"response": {'items': [{'data': {'id': 302488228, 'creator_id': 2393211, 'contact_id': 302488227, 'created_at': '2020-11-12T09:05:47Z', 'updated_at': '2022-03-23T16:53:22Z', 'title': None, 'name': 'Octavia Squidington', 'first_name': 'Octavia', 'last_name': 'Squidington'},'meta': {'version': 36, 'type': 'contact'}}], - 'meta': {'type': 'collection','count': 25,'links': {'self': 'https://api.getbase.com/v2/contacts?page=2&per_page=25', 'first_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25', 'prev_page': 'https://api.getbase.com/v2/contacts?page=1&per_page=25', 'next_page': 'https://api.getbase.com/v2/contacts?page=3&per_page=25'}}}} - expected_parsed_object = {'id': 302488228, 'creator_id': 2393211, 'contact_id': 302488227, 'created_at': '2020-11-12T09:05:47Z', 'updated_at': '2022-03-23T16:53:22Z', 'title': None, 'name': 'Octavia Squidington', 'first_name': 'Octavia', 'last_name': 'Squidington'} + inputs = { + "response": { + "items": [ + { + "data": { + "id": 302488228, + "creator_id": 2393211, + "contact_id": 302488227, + "created_at": "2020-11-12T09:05:47Z", + "updated_at": "2022-03-23T16:53:22Z", + "title": None, + "name": "Octavia Squidington", + "first_name": "Octavia", + "last_name": "Squidington", + }, + "meta": {"version": 36, "type": "contact"}, + } + ], + "meta": { + "type": "collection", + "count": 25, + "links": { + "self": "https://api.getbase.com/v2/contacts?page=2&per_page=25", + "first_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", + "prev_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", + "next_page": "https://api.getbase.com/v2/contacts?page=3&per_page=25", + }, + }, + } + } + expected_parsed_object = { + "id": 302488228, + "creator_id": 2393211, + "contact_id": 302488227, + "created_at": "2020-11-12T09:05:47Z", + "updated_at": "2022-03-23T16:53:22Z", + "title": None, + "name": "Octavia Squidington", + "first_name": "Octavia", + "last_name": "Squidington", + } assert next(stream.parse_response(**inputs)) == expected_parsed_object def test_request_headers(patch_base_class): stream = ZendeskSellStream() inputs = {"stream_slice": None, "stream_state": None, "next_page_token": 2} - expected_headers = {"page" : 2} + expected_headers = {"page": 2} assert stream.request_headers(**inputs) == expected_headers diff --git a/docs/integrations/sources/zendesk-sell.md b/docs/integrations/sources/zendesk-sell.md new file mode 100644 index 000000000000..0706b6bc563b --- /dev/null +++ b/docs/integrations/sources/zendesk-sell.md @@ -0,0 +1,79 @@ +# Zendesk Sunshine + +## Sync overview + +The Zendesk Sell source supports Full Refresh. + +This source can sync data for the [Zendesk Sell API](https://developer.zendesk.com/api-reference/sales-crm/introduction/). + +### Output schema + +This Source is capable of syncing the following core Streams: + +* Call Outcomes +* Calls +* Collaborations +* Contacts +* Deal Sources +* Deal Unqualified Reason +* Deals +* Lead Conversions +* Lead Sources +* Lead Unqualified Reason +* Leads +* Loss Reasons +* Notes +* Orders +* Pipelines +* Products +* Stages +* Tags +* Tasks +* Text Messages +* Users +* Visit Outcomes +* Visits + +### Data type mapping + +| Integration Type | Airbyte Type | Notes | +| :--- | :--- | :--- | +| `string` | `string` | | +| `number` | `number` | | +| `array` | `array` | | +| `object` | `object` | | + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | Yes | | +| Incremental Sync | Yes | | + +### Performance considerations + +The connector is restricted by normal Zendesk [requests limitation](https://developer.zendesk.com/api-reference/ticketing/account-configuration/usage_limits/) + +The Zendesk connector should not run into Zendesk API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. + +## Getting started + +### Requirements + +* Zendesk Sell API Token + + +### Setup guide + +Please follow this [guide](https://developer.zendesk.com/documentation/custom-data/custom-objects/getting-started-with-custom-objects/#enabling-custom-objects) + +Generate an API Token or oauth2.0 Access token as described in [here](https://developer.zendesk.com/api-reference/ticketing/introduction/#security-and-authentication) + +We recommend creating a restricted, read-only key specifically for Airbyte access. This will allow you to control which resources Airbyte should be able to access. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :--- | :--- | :--- | :--- | +| 0.1.0 | 2022-10-27 | [17888](https://github.com/airbytehq/airbyte/pull/17888) | Initial Release | + From a06daab7dc2f1c2634e540fa00effb4308fa4799 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 27 Oct 2022 08:43:59 -0300 Subject: [PATCH 22/25] update airbyte cdk version --- airbyte-integrations/connectors/source-zendesk-sell/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/setup.py b/airbyte-integrations/connectors/source-zendesk-sell/setup.py index 0900235594ab..dfcfbfe0bf76 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1.56", + "airbyte-cdk~=0.2", ] TEST_REQUIREMENTS = [ From f472653d2c29f14c74d49ab01fcbe45913c75dbc Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Fri, 28 Oct 2022 18:33:44 -0300 Subject: [PATCH 23/25] correct unit test --- .../connectors/source-zendesk-sell/setup.py | 2 +- .../source_zendesk_sell/source.py | 9 ++- .../unit_tests/test_source.py | 16 ++++- .../unit_tests/test_streams.py | 59 +++++-------------- 4 files changed, 38 insertions(+), 48 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/setup.py b/airbyte-integrations/connectors/source-zendesk-sell/setup.py index dfcfbfe0bf76..ab5314798e39 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", + "airbyte-cdk~=0.4", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py index b3c87ce17e57..d6868c021c71 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py @@ -28,13 +28,16 @@ class ZendeskSellStream(HttpStream, ABC): def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: try: + meta_links = {} regex_page = r"[=?/]page[_=/-]?(\d{1,3})" - meta_links = response.json().get("meta", {}).get("links") + data = response.json() + if data: + meta_links = data.get("meta", {}).get("links") if "next_page" in meta_links.keys(): return {"page": int(re.findall(regex_page, meta_links["next_page"])[0])} + return None except Exception as e: self.logger.error(f"{e.__class__} occurred, while trying to get next page information from the following dict {meta_links}") - return None def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None @@ -46,7 +49,7 @@ def request_params( def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: items = response.json()["items"] - return [item["data"] for item in items] + yield from [item["data"] for item in items] class Pipelines(ZendeskSellStream): diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py index 2ca3ea8825eb..be17104d4566 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py @@ -4,12 +4,26 @@ from unittest.mock import MagicMock +from pytest import fixture from source_zendesk_sell.source import SourceZendeskSell +@fixture +def config(): + return {"config": {"user_auth_key": "", "start_date": "2021-01-01T00:00:00Z", "outcome_names": ""}} -def test_check_connection(mocker): + +def test_check_connection(mocker, requests_mock, config): source = SourceZendeskSell() logger_mock, config_mock = MagicMock(), MagicMock() + requests_mock.get( + "https://api.getbase.com/v2/contacts", + json={ + "items": [ + {"data": {}} + ] + } + ) + assert source.check_connection(logger_mock, config_mock) == (True, None) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py index a23072811cf4..57f1f751c84e 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py @@ -4,6 +4,7 @@ from http import HTTPStatus from unittest.mock import MagicMock +from unittest import mock import pytest from source_zendesk_sell.source import ZendeskSellStream @@ -29,22 +30,7 @@ def test_request_params(patch_base_class): [ ( { - "items": [ - { - "data": { - "id": 302488228, - "creator_id": 2393211, - "contact_id": 302488227, - "created_at": "2020-11-12T09:05:47Z", - "updated_at": "2022-03-23T16:53:22Z", - "title": None, - "name": "Octavia Squidington", - "first_name": "Octavia", - "last_name": "Squidington", - }, - "meta": {"version": 36, "type": "contact"}, - } - ], + "items": [], "meta": { "type": "collection", "count": 25, @@ -56,26 +42,11 @@ def test_request_params(patch_base_class): }, }, }, - 3, + {"page": 3}, ), ( { - "items": [ - { - "data": { - "id": 302488228, - "creator_id": 2393211, - "contact_id": 302488227, - "created_at": "2020-11-12T09:05:47Z", - "updated_at": "2022-03-23T16:53:22Z", - "title": None, - "name": "Octavia Squidington", - "first_name": "Octavia", - "last_name": "Squidington", - }, - "meta": {"version": 36, "type": "contact"}, - } - ], + "items": [], "meta": { "type": "collection", "count": 25, @@ -91,16 +62,17 @@ def test_request_params(patch_base_class): ({None}, None), ], ) -def test_next_page_token(patch_base_class, inputs, expected_token): +def test_next_page_token(mocker, requests_mock, patch_base_class, inputs, expected_token): stream = ZendeskSellStream() - inputs = {"response": MagicMock()} - assert stream.next_page_token(**inputs) == expected_token + response = mocker.MagicMock() + response.json.return_value = inputs + assert stream.next_page_token(response) == expected_token -def test_parse_response(patch_base_class): +def test_parse_response(patch_base_class, mocker): stream = ZendeskSellStream() - inputs = { - "response": { + response = mocker.MagicMock() + response.json.return_value = { "items": [ { "data": { @@ -127,7 +99,6 @@ def test_parse_response(patch_base_class): "next_page": "https://api.getbase.com/v2/contacts?page=3&per_page=25", }, }, - } } expected_parsed_object = { "id": 302488228, @@ -140,14 +111,16 @@ def test_parse_response(patch_base_class): "first_name": "Octavia", "last_name": "Squidington", } - assert next(stream.parse_response(**inputs)) == expected_parsed_object + assert next(stream.parse_response(response)) == expected_parsed_object def test_request_headers(patch_base_class): stream = ZendeskSellStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": 2} + stream_slice = None + stream_state = None + next_page_token = {"page": 2} expected_headers = {"page": 2} - assert stream.request_headers(**inputs) == expected_headers + assert stream.request_params(stream_slice, stream_state, next_page_token) == expected_headers def test_http_method(patch_base_class): From 1b8fd1cc309e86ee65d81ac7d8d6ecc237e0bbf6 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Fri, 28 Oct 2022 19:13:09 -0300 Subject: [PATCH 24/25] run format --- .../unit_tests/test_source.py | 10 +--- .../unit_tests/test_streams.py | 49 +++++++++---------- 2 files changed, 26 insertions(+), 33 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py index be17104d4566..063308dd0189 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py @@ -7,6 +7,7 @@ from pytest import fixture from source_zendesk_sell.source import SourceZendeskSell + @fixture def config(): return {"config": {"user_auth_key": "", "start_date": "2021-01-01T00:00:00Z", "outcome_names": ""}} @@ -15,14 +16,7 @@ def config(): def test_check_connection(mocker, requests_mock, config): source = SourceZendeskSell() logger_mock, config_mock = MagicMock(), MagicMock() - requests_mock.get( - "https://api.getbase.com/v2/contacts", - json={ - "items": [ - {"data": {}} - ] - } - ) + requests_mock.get("https://api.getbase.com/v2/contacts", json={"items": [{"data": {}}]}) assert source.check_connection(logger_mock, config_mock) == (True, None) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py index 57f1f751c84e..aac02fc6c0e2 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py @@ -4,7 +4,6 @@ from http import HTTPStatus from unittest.mock import MagicMock -from unittest import mock import pytest from source_zendesk_sell.source import ZendeskSellStream @@ -73,32 +72,32 @@ def test_parse_response(patch_base_class, mocker): stream = ZendeskSellStream() response = mocker.MagicMock() response.json.return_value = { - "items": [ - { - "data": { - "id": 302488228, - "creator_id": 2393211, - "contact_id": 302488227, - "created_at": "2020-11-12T09:05:47Z", - "updated_at": "2022-03-23T16:53:22Z", - "title": None, - "name": "Octavia Squidington", - "first_name": "Octavia", - "last_name": "Squidington", - }, - "meta": {"version": 36, "type": "contact"}, - } - ], - "meta": { - "type": "collection", - "count": 25, - "links": { - "self": "https://api.getbase.com/v2/contacts?page=2&per_page=25", - "first_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", - "prev_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", - "next_page": "https://api.getbase.com/v2/contacts?page=3&per_page=25", + "items": [ + { + "data": { + "id": 302488228, + "creator_id": 2393211, + "contact_id": 302488227, + "created_at": "2020-11-12T09:05:47Z", + "updated_at": "2022-03-23T16:53:22Z", + "title": None, + "name": "Octavia Squidington", + "first_name": "Octavia", + "last_name": "Squidington", }, + "meta": {"version": 36, "type": "contact"}, + } + ], + "meta": { + "type": "collection", + "count": 25, + "links": { + "self": "https://api.getbase.com/v2/contacts?page=2&per_page=25", + "first_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", + "prev_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", + "next_page": "https://api.getbase.com/v2/contacts?page=3&per_page=25", }, + }, } expected_parsed_object = { "id": 302488228, From 254d48ff55e7a186ee9803d3195348f70b72521d Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 28 Oct 2022 22:26:15 +0000 Subject: [PATCH 25/25] auto-bump connector version --- .../src/main/resources/seed/source_specs.yaml | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index ea9316dbe684..b1f00f3b467b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -11974,6 +11974,26 @@ path_in_connector_config: - "credentials" - "client_secret" +- dockerImage: "airbyte/source-zendesk-sell:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/zendesk-sell" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Source Zendesk Sell Spec" + type: "object" + required: + - "api_token" + properties: + api_token: + title: "API token" + type: "string" + description: "The API token for authenticating to Zendesk Sell" + examples: + - "f23yhd630otl94y85a8bf384958473pto95847fd006da49382716or937ruw059" + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-zendesk-sunshine:0.1.1" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/zendesk_sunshine"