From 12c80acf92557f87eb9e8643f8dfa50b5892726c Mon Sep 17 00:00:00 2001 From: Alcadeus Date: Sun, 8 Oct 2023 16:32:06 +0530 Subject: [PATCH 1/9] migrated insightly --- .../connectors/source-insightly/Dockerfile | 4 +- .../connectors/source-insightly/README.md | 64 +-- .../{unit_tests => }/__init__.py | 0 .../acceptance-test-config.yml | 2 +- .../acceptance-test-docker.sh | 1 + .../connectors/source-insightly/bootstrap.md | 13 - .../connectors/source-insightly/build.gradle | 8 + .../connectors/source-insightly/icon.svg | 1 - .../integration_tests/abnormal_state.json | 2 +- .../integration_tests/configured_catalog.json | 2 +- .../integration_tests/invalid_config.json | 2 +- .../integration_tests/sample_config.json | 2 +- .../integration_tests/sample_state.json | 2 +- .../connectors/source-insightly/metadata.yaml | 19 +- .../connectors/source-insightly/setup.py | 6 +- .../source_insightly/manifest.yaml | 390 +++++++++++++++++ .../source_insightly/source.py | 412 +----------------- .../source_insightly/spec.json | 25 -- .../unit_tests/test_incremental_streams.py | 81 ---- .../unit_tests/test_source.py | 56 --- .../unit_tests/test_streams.py | 126 ------ 21 files changed, 438 insertions(+), 780 deletions(-) rename airbyte-integrations/connectors/source-insightly/{unit_tests => }/__init__.py (100%) mode change 100644 => 100755 airbyte-integrations/connectors/source-insightly/acceptance-test-docker.sh delete mode 100644 airbyte-integrations/connectors/source-insightly/bootstrap.md create mode 100644 airbyte-integrations/connectors/source-insightly/build.gradle delete mode 100644 airbyte-integrations/connectors/source-insightly/icon.svg create mode 100644 airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml delete mode 100644 airbyte-integrations/connectors/source-insightly/source_insightly/spec.json delete mode 100644 airbyte-integrations/connectors/source-insightly/unit_tests/test_incremental_streams.py delete mode 100644 airbyte-integrations/connectors/source-insightly/unit_tests/test_source.py delete mode 100644 airbyte-integrations/connectors/source-insightly/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-insightly/Dockerfile b/airbyte-integrations/connectors/source-insightly/Dockerfile index c4ac2c8fb73a..2eb27731e7a7 100644 --- a/airbyte-integrations/connectors/source-insightly/Dockerfile +++ b/airbyte-integrations/connectors/source-insightly/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.13-alpine3.15 as base +FROM python:3.9.11-alpine3.15 as base # build and load all requirements FROM base as builder @@ -34,5 +34,5 @@ COPY source_insightly ./source_insightly ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.0 LABEL io.airbyte.name=airbyte/source-insightly diff --git a/airbyte-integrations/connectors/source-insightly/README.md b/airbyte-integrations/connectors/source-insightly/README.md index 61fbc856b46f..989117856b81 100644 --- a/airbyte-integrations/connectors/source-insightly/README.md +++ b/airbyte-integrations/connectors/source-insightly/README.md @@ -1,35 +1,10 @@ # Insightly Source -This is the repository for the Insightly source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/insightly). +This is the repository for the Insightly configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/insightly). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -39,7 +14,7 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/insightly) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/insightly) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_insightly/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -47,14 +22,6 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source insightly test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -79,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-insightly:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-insightly:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-insightly/unit_tests/__init__.py b/airbyte-integrations/connectors/source-insightly/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-insightly/unit_tests/__init__.py rename to airbyte-integrations/connectors/source-insightly/__init__.py diff --git a/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml b/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml index a7de796e79a5..abea18882106 100644 --- a/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml @@ -27,4 +27,4 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state: - future_state_path: integration_tests/abnormal_state.json + future_state_path: integration_tests/abnormal_state.json \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-insightly/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-insightly/acceptance-test-docker.sh old mode 100644 new mode 100755 index 5797d20fe9a7..b6d65deeccb4 --- a/airbyte-integrations/connectors/source-insightly/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-insightly/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-insightly/bootstrap.md b/airbyte-integrations/connectors/source-insightly/bootstrap.md deleted file mode 100644 index d52b29577dea..000000000000 --- a/airbyte-integrations/connectors/source-insightly/bootstrap.md +++ /dev/null @@ -1,13 +0,0 @@ -# Insightly -OpenWeather is an online service offering an API to retrieve historical, current and forecasted weather data over the globe. - -### Auth -API calls are authenticated through an API key. An API key can be retrieved from Insightly User Settings page in the API section. - -### Rate limits -The API has different rate limits for different account types. Keep that in mind when syncing large amounts of data: -* Free/Gratis - 1,000 requests/day/instance -* Legacy plans - 20,000 requests/day/instance -* Plus - 40,000 requests/day/instance -* Professional - 60,000 requests/day/instance -* Enterprise - 100,000 requests/day/instance diff --git a/airbyte-integrations/connectors/source-insightly/build.gradle b/airbyte-integrations/connectors/source-insightly/build.gradle new file mode 100644 index 000000000000..c23c48b88275 --- /dev/null +++ b/airbyte-integrations/connectors/source-insightly/build.gradle @@ -0,0 +1,8 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' +} + +airbytePython { + moduleDirectory 'source_insightly' +} diff --git a/airbyte-integrations/connectors/source-insightly/icon.svg b/airbyte-integrations/connectors/source-insightly/icon.svg deleted file mode 100644 index d0741d947f6b..000000000000 --- a/airbyte-integrations/connectors/source-insightly/icon.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-insightly/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-insightly/integration_tests/abnormal_state.json index c06e9d0a75c0..60d25c39160d 100644 --- a/airbyte-integrations/connectors/source-insightly/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-insightly/integration_tests/abnormal_state.json @@ -10,4 +10,4 @@ } } } -] +] \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-insightly/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-insightly/integration_tests/configured_catalog.json index 0873c8cb593f..b42c7123143b 100644 --- a/airbyte-integrations/connectors/source-insightly/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-insightly/integration_tests/configured_catalog.json @@ -23,4 +23,4 @@ "destination_sync_mode": "append" } ] -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-insightly/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-insightly/integration_tests/invalid_config.json index ac0271de4fbd..f9c6dff6b868 100644 --- a/airbyte-integrations/connectors/source-insightly/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-insightly/integration_tests/invalid_config.json @@ -1,4 +1,4 @@ { "token": "bad-token", "start_date": "2019-01-01T00:00:00Z" -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-insightly/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-insightly/integration_tests/sample_config.json index 2dd9a2f7b016..6da5a9d217f6 100644 --- a/airbyte-integrations/connectors/source-insightly/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-insightly/integration_tests/sample_config.json @@ -1,4 +1,4 @@ { "token": "my-insightly-api-token", "start_date": "2022-10-01T00:00:00Z" -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-insightly/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-insightly/integration_tests/sample_state.json index 3127937e0f0e..48974dd13c99 100644 --- a/airbyte-integrations/connectors/source-insightly/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-insightly/integration_tests/sample_state.json @@ -10,4 +10,4 @@ } } } -] +] \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-insightly/metadata.yaml b/airbyte-integrations/connectors/source-insightly/metadata.yaml index 06202981ce95..321d751ffa7a 100644 --- a/airbyte-integrations/connectors/source-insightly/metadata.yaml +++ b/airbyte-integrations/connectors/source-insightly/metadata.yaml @@ -1,24 +1,27 @@ data: + allowedHosts: + hosts: + - TODO # Please change to the hostname of the source. + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 38f84314-fe6a-4257-97be-a8dcd942d693 - dockerImageTag: 0.1.3 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-insightly githubIssueLabel: source-insightly icon: insightly.svg license: MIT name: Insightly - registries: - cloud: - enabled: true - oss: - enabled: true releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/insightly tags: - - language:python + - language:lowcode ab_internal: sl: 100 ql: 100 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-insightly/setup.py b/airbyte-integrations/connectors/source-insightly/setup.py index a8c4637c1342..afac830b077e 100644 --- a/airbyte-integrations/connectors/source-insightly/setup.py +++ b/airbyte-integrations/connectors/source-insightly/setup.py @@ -6,14 +6,14 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", - "pendulum==2.1.2", + "airbyte-cdk~=0.1", ] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", + "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml b/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml new file mode 100644 index 000000000000..37ae545edde1 --- /dev/null +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml @@ -0,0 +1,390 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + requester: + type: HttpRequester + url_base: "https://api.na1.insightly.com/v3.1/" + http_method: "GET" + authenticator: + type: BasicHttpAuthenticator + username: "{{ config['token'] }}" + request_options_provider: + request_parameters: + count_total: True, + date_incremental_sync: + type: DatetimeBasedCursor + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + datetime: "{{ now_utc() }}" + datetime_format: "%Y-%m-%d %H:%M:%S.%f+00:00" + datetime_format: "%Y-%m-%dT%H:%M:%S" + cursor_granularity: PT1S + step: P1D + cursor_field: "DATE_UPDATED_UTC" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: DefaultPaginator + pagination_strategy: + type: "OffsetIncrement" + page_size: 500 + page_token_option: + type: RequestOption + inject_into: "request_parameter" + field_name: "skip" + page_size_option: + inject_into: "request_parameter" + field_name: "top" + requester: + $ref: "#/definitions/requester" + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + base_incremental_stream: + incremental_sync: + $ref: "#/definitions/date_incremental_sync" + retriever: + $ref: "#/definitions/retriever" + activity_sets_stream: + $ref: "#/definitions/base_stream" + name: "activity_sets" + primary_key: "ACTIVITYSET_ID" + $parameters: + path: "/ActivitySets" + contacts_stream: + $ref: "#/definitions/base_incremental_stream" + name: "contacts" + primary_key: "CONTACT_ID" + $parameters: + path: "/Contacts/Search" + countries_stream: + $ref: "#/definitions/base_stream" + name: "countries" + primary_key: "COUNTRY_NAME" + $parameters: + path: "/Countries" + currencies_stream: + $ref: "#/definitions/base_stream" + name: "currencies" + primary_key: "CURRENCY_CODE" + $parameters: + path: "/Currencies" + emails_stream: + $ref: "#/definitions/base_stream" + name: "emails" + primary_key: "EMAIL_ID" + $parameters: + path: "/Emails" + events_stream: + $ref: "#/definitions/base_incremental_stream" + name: "events" + primary_key: "EVENT_ID" + $parameters: + path: "/Events/Search" + knowledge_article_categories_stream: + $ref: "#/definitions/base_incremental_stream" + name: "knowledge_article_categories" + primary_key: "CATEGORY_ID" + $parameters: + path: "/KnowledgeArticleCategory/Search" + knowledge_article_folders_stream: + $ref: "#/definitions/base_incremental_stream" + name: "knowledge_article_folders" + primary_key: "FOLDER_ID" + $parameters: + path: "/KnowledgeArticleFolder/Search" + knowledge_articles_stream: + $ref: "#/definitions/base_incremental_stream" + name: "knowledge_articles" + primary_key: "ARTICLE_ID" + $parameters: + path: "/KnowledgeArticle/Search" + leads_stream: + $ref: "#/definitions/base_incremental_stream" + name: "leads" + primary_key: "LEAD_ID" + $parameters: + path: "/Leads/Search" + lead_sources_stream: + $ref: "#/definitions/base_stream" + name: "lead_sources" + primary_key: "LEAD_SOURCE_ID" + $parameters: + path: "/LeadSources" + lead_statuses_stream: + $ref: "#/definitions/base_stream" + name: "lead_statuses" + primary_key: "LEAD_STATUS_ID" + $parameters: + path: "/LeadStatuses" + milestones_stream: + $ref: "#/definitions/base_incremental_stream" + name: "milestones" + primary_key: "MILESTONE_ID" + $parameters: + path: "/Milestones/Search" + notes_stream: + $ref: "#/definitions/base_incremental_stream" + name: "notes" + primary_key: "NOTE_ID" + $parameters: + path: "/Notes/Search" + opportunities_stream: + $ref: "#/definitions/base_incremental_stream" + name: "opportunities" + primary_key: "OPPORTUNITY_ID" + $parameters: + path: "/Opportunities/Search" + opportunity_categories_stream: + $ref: "#/definitions/base_stream" + name: "opportunity_categories" + primary_key: "CATEGORY_ID" + $parameters: + path: "/OpportunityCategories" + opportunity_products_stream: + $ref: "#/definitions/base_incremental_stream" + name: "opportunity_products" + primary_key: "OPPORTUNITY_ITEM_ID" + $parameters: + path: "/OpportunityLineItem/Search" + opportunity_state_reasons_stream: + $ref: "#/definitions/base_stream" + name: "opportunity_state_reasons" + primary_key: "STATE_REASON_ID" + $parameters: + path: "/OpportunityStateReasons" + organizations_stream: + $ref: "#/definitions/base_incremental_stream" + name: "organizations" + primary_key: "ORGANIZATION_ID" + $parameters: + path: "/Organizations/Search" + pipelines_stream: + $ref: "#/definitions/base_stream" + name: "pipelines" + primary_key: "PIPELINE_ID" + $parameters: + path: "/Pipelines" + pipeline_stages_stream: + $ref: "#/definitions/base_stream" + name: "pipeline_stages" + primary_key: "STAGE_ID" + $parameters: + path: "/PipelineStages" + pricebook_entries_stream: + $ref: "#/definitions/base_incremental_stream" + name: "price_book_entries" + primary_key: "PRICEBOOK_ENTRY_ID" + $parameters: + path: "/PricebookEntry/Search" + pricebooks_stream: + $ref: "#/definitions/base_incremental_stream" + name: "price_books" + primary_key: "PRICEBOOK_ID" + $parameters: + path: "/Pricebook/Search" + products_stream: + $ref: "#/definitions/base_incremental_stream" + name: "products" + primary_key: "PRODUCT_ID" + $parameters: + path: "/Product/Search" + project_categories_stream: + $ref: "#/definitions/base_stream" + name: "project_categories" + primary_key: "CATEGORY_ID" + $parameters: + path: "/ProjectCategories" + projects_stream: + $ref: "#/definitions/base_incremental_stream" + name: "projects" + primary_key: "PROJECT_ID" + $parameters: + path: "/Projects/Search" + prospects_stream: + $ref: "#/definitions/base_incremental_stream" + name: "prospects" + primary_key: "PROSPECT_ID" + $parameters: + path: "/Prospect/Search" + quote_products_stream: + $ref: "#/definitions/base_incremental_stream" + name: "quote_products" + primary_key: "QUOTATION_ITEM_ID" + $parameters: + path: "/QuotationLineItem/Search" + quotes_stream: + $ref: "#/definitions/base_incremental_stream" + name: "quotes" + primary_key: "QUOTE_ID" + $parameters: + path: "/Quotation/Search" + relationships_stream: + $ref: "#/definitions/base_stream" + name: "relationships" + primary_key: "RELATIONSHIP_ID" + $parameters: + path: "/Relationships" + tags_stream: + $ref: "#/definitions/base_stream" + name: "tags" + primary_key: "TAG_NAME" + $parameters: + path: "/Tags" + task_categories_stream: + $ref: "#/definitions/base_stream" + name: "task_categories" + primary_key: "CATEGORY_ID" + $parameters: + path: "/TaskCategories" + tasks_stream: + $ref: "#/definitions/base_incremental_stream" + name: "tasks" + primary_key: "TASK_ID" + $parameters: + path: "/Tasks/Search" + team_members_stream: + $ref: "#/definitions/base_stream" + name: "team_members" + primary_key: "MEMBER_USER_ID" + $parameters: + path: "/TeamMembers" + teams_stream: + $ref: "#/definitions/base_stream" + name: "teams" + primary_key: "TEAM_ID" + $parameters: + path: "/Teams" + tickets_stream: + $ref: "#/definitions/base_incremental_stream" + name: "tickets" + primary_key: "TICKET_ID" + $parameters: + path: "/Tickets/Search" + users_stream: + $ref: "#/definitions/base_incremental_stream" + name: "users" + primary_key: "USER_ID" + $parameters: + path: "/Users/Search" + +streams: + - "#/definitions/activity_sets_stream" + - "#/definitions/contacts_stream" + - "#/definitions/countries_stream" + - "#/definitions/currencies_stream" + - "#/definitions/emails_stream" + - "#/definitions/events_stream" + - "#/definitions/knowledge_article_categories_stream" + - "#/definitions/knowledge_article_folders_stream" + - "#/definitions/knowledge_articles_stream" + - "#/definitions/leads_stream" + - "#/definitions/lead_sources_stream" + - "#/definitions/lead_statuses_stream" + - "#/definitions/milestones_stream" + - "#/definitions/notes_stream" + - "#/definitions/opportunities_stream" + - "#/definitions/opportunity_categories_stream" + - "#/definitions/opportunity_products_stream" + - "#/definitions/opportunity_state_reasons_stream" + - "#/definitions/organizations_stream" + - "#/definitions/pipelines_stream" + - "#/definitions/pipeline_stages_stream" + - "#/definitions/pricebook_entries_stream" + - "#/definitions/pricebooks_stream" + - "#/definitions/products_stream" + - "#/definitions/project_categories_stream" + - "#/definitions/projects_stream" + - "#/definitions/prospects_stream" + - "#/definitions/quote_products_stream" + - "#/definitions/quotes_stream" + - "#/definitions/relationships_stream" + - "#/definitions/tags_stream" + - "#/definitions/task_categories_stream" + - "#/definitions/tasks_stream" + - "#/definitions/team_members_stream" + - "#/definitions/teams_stream" + - "#/definitions/tickets_stream" + - "#/definitions/users_stream" + +check: + type: CheckStream + stream_names: + - "activity_sets" + - "contacts" + - "countries" + - "currencies" + - "emails" + - "events" + - "knowledge_article_categories" + - "knowledge_article_folders" + - "knowledge_articles" + - "leads" + - "lead_sources" + - "lead_statuses" + - "milestones" + - "notes" + - "opportunities" + - "opportunity_categories" + - "opportunity_products" + - "opportunity_state_reasons" + - "organizations" + - "pipelines" + - "pipeline_stages" + - "price_book_entries" + - "price_books" + - "products" + - "project_categories" + - "projects" + - "prospects" + - "quote_products" + - "quotes" + - "relationships" + - "tags" + - "task_categories" + - "tasks" + - "team_members" + - "teams" + - "tickets" + - "users" + +spec: + type: Spec + documentationUrl: https://docs.airbyte.com/integrations/sources/insightly + connection_specification: + "$schema": http://json-schema.org/draft-07/schema# + title: Insightly Spec + type: object + required: + - token + - start_date + additionalProperties: true + properties: + token: + type: + - string + - 'null' + title: API Token + description: Your Insightly API token. + airbyte_secret: true + start_date: + type: + - string + - 'null' + title: Start Date + description: The date from which you'd like to replicate data for Insightly + in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will + be replicated. Note that it will be used only for incremental streams. + examples: + - '2021-03-01T00:00:00Z' + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/source.py b/airbyte-integrations/connectors/source-insightly/source_insightly/source.py index 68133dfb1df3..29fa855efc5b 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/source.py +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/source.py @@ -2,409 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from datetime import datetime -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -from urllib.parse import parse_qs, urlparse +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import pendulum -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import BasicHttpAuthenticator -from requests.auth import AuthBase +WARNING: Do not modify this file. +""" -PAGE_SIZE = 500 -BASE_URL = "https://api.insightly.com/v3.1/" - -# Basic full refresh stream -class InsightlyStream(HttpStream, ABC): - total_count: int = 0 - page_size: Optional[int] = PAGE_SIZE - - url_base = BASE_URL - - def __init__(self, authenticator: AuthBase, start_date: str = None, **kwargs): - self.start_date = start_date - super().__init__(authenticator=authenticator) - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - parsed = urlparse(response.request.url) - previous_skip = parse_qs(parsed.query)["skip"][0] - new_skip = int(previous_skip) + self.page_size - return new_skip if new_skip <= self.total_count else None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return { - "count_total": True, - "top": self.page_size, - "skip": next_page_token or 0, - } - - def request_headers(self, **kwargs) -> Mapping[str, Any]: - return {"Accept": "application/json"} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - self.total_count = int(response.headers.get("X-Total-Count", 0)) - results = response.json() - yield from results - - -class ActivitySets(InsightlyStream): - primary_key = "ACTIVITYSET_ID" - - def path(self, **kwargs) -> str: - return "ActivitySets" - - -class Countries(InsightlyStream): - primary_key = "COUNTRY_NAME" - - def path(self, **kwargs) -> str: - return "Countries" - - -class Currencies(InsightlyStream): - primary_key = "CURRENCY_CODE" - - def path(self, **kwargs) -> str: - return "Currencies" - - -class Emails(InsightlyStream): - primary_key = "EMAIL_ID" - - def path(self, **kwargs) -> str: - return "Emails" - - -class LeadSources(InsightlyStream): - primary_key = "LEAD_SOURCE_ID" - - def path(self, **kwargs) -> str: - return "LeadSources" - - -class LeadStatuses(InsightlyStream): - primary_key = "LEAD_STATUS_ID" - - def path(self, **kwargs) -> str: - return "LeadStatuses" - - -class OpportunityCategories(InsightlyStream): - primary_key = "CATEGORY_ID" - - def path(self, **kwargs) -> str: - return "OpportunityCategories" - - -class OpportunityStateReasons(InsightlyStream): - primary_key = "STATE_REASON_ID" - - def path(self, **kwargs) -> str: - return "OpportunityStateReasons" - - -class Pipelines(InsightlyStream): - primary_key = "PIPELINE_ID" - - def path(self, **kwargs) -> str: - return "Pipelines" - - -class PipelineStages(InsightlyStream): - primary_key = "STAGE_ID" - - def path(self, **kwargs) -> str: - return "PipelineStages" - - -class ProjectCategories(InsightlyStream): - primary_key = "CATEGORY_ID" - - def path(self, **kwargs) -> str: - return "ProjectCategories" - - -class Relationships(InsightlyStream): - primary_key = "RELATIONSHIP_ID" - - def path(self, **kwargs) -> str: - return "Relationships" - - -class Tags(InsightlyStream): - primary_key = "TAG_NAME" - - def path(self, **kwargs) -> str: - return "Tags" - - -class TaskCategories(InsightlyStream): - primary_key = "CATEGORY_ID" - - def path(self, **kwargs) -> str: - return "TaskCategories" - - -class TeamMembers(InsightlyStream): - primary_key = "MEMBER_USER_ID" - - def path(self, **kwargs) -> str: - return "TeamMembers" - - -class Teams(InsightlyStream): - primary_key = "TEAM_ID" - - def path(self, **kwargs) -> str: - return "Teams" - - -class IncrementalInsightlyStream(InsightlyStream, ABC): - """Insighlty incremental stream using `updated_after_utc` filter""" - - cursor_field = "DATE_UPDATED_UTC" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - - start_datetime = pendulum.parse(self.start_date) - cursor_datetime = stream_state.get(self.cursor_field) - if cursor_datetime: - if isinstance(cursor_datetime, datetime): - start_datetime = cursor_datetime - else: - start_datetime = pendulum.parse(cursor_datetime) - - # subtract 1 second to make the incremental request inclusive - start_datetime = start_datetime.subtract(seconds=1) - - params.update({"updated_after_utc": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")}) - return params - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - record_time = pendulum.parse(latest_record[self.cursor_field]) - current_state = current_stream_state.get(self.cursor_field) - if current_state: - current_state = current_state if isinstance(current_state, datetime) else pendulum.parse(current_state) - - current_stream_state[self.cursor_field] = max(record_time, current_state) if current_state else record_time - return current_stream_state - - -class Contacts(IncrementalInsightlyStream): - primary_key = "CONTACT_ID" - - def path(self, **kwargs) -> str: - return "Contacts/Search" - - -class Events(IncrementalInsightlyStream): - primary_key = "EVENT_ID" - - def path(self, **kwargs) -> str: - return "Events/Search" - - -class KnowledgeArticleCategories(IncrementalInsightlyStream): - primary_key = "CATEGORY_ID" - - def path(self, **kwargs) -> str: - return "KnowledgeArticleCategory/Search" - - -class KnowledgeArticleFolders(IncrementalInsightlyStream): - primary_key = "FOLDER_ID" - - def path(self, **kwargs) -> str: - return "KnowledgeArticleFolder/Search" - - -class KnowledgeArticles(IncrementalInsightlyStream): - primary_key = "ARTICLE_ID" - - def path(self, **kwargs) -> str: - return "KnowledgeArticle/Search" - - -class Leads(IncrementalInsightlyStream): - primary_key = "LEAD_ID" - - def path(self, **kwargs) -> str: - return "Leads/Search" - - -class Milestones(IncrementalInsightlyStream): - primary_key = "MILESTONE_ID" - - def path(self, **kwargs) -> str: - return "Milestones/Search" - - -class Notes(IncrementalInsightlyStream): - primary_key = "NOTE_ID" - - def path(self, **kwargs) -> str: - return "Notes/Search" - - -class Opportunities(IncrementalInsightlyStream): - primary_key = "OPPORTUNITY_ID" - - def path(self, **kwargs) -> str: - return "Opportunities/Search" - - -class OpportunityProducts(IncrementalInsightlyStream): - primary_key = "OPPORTUNITY_ITEM_ID" - - def path(self, **kwargs) -> str: - return "OpportunityLineItem/Search" - - -class Organisations(IncrementalInsightlyStream): - primary_key = "ORGANISATION_ID" - - def path(self, **kwargs) -> str: - return "Organisations/Search" - - -class PricebookEntries(IncrementalInsightlyStream): - primary_key = "PRICEBOOK_ENTRY_ID" - - def path(self, **kwargs) -> str: - return "PricebookEntry/Search" - - -class Pricebooks(IncrementalInsightlyStream): - primary_key = "PRICEBOOK_ID" - - def path(self, **kwargs) -> str: - return "Pricebook/Search" - - -class Products(IncrementalInsightlyStream): - primary_key = "PRODUCT_ID" - - def path(self, **kwargs) -> str: - return "Product/Search" - - -class Projects(IncrementalInsightlyStream): - primary_key = "PROJECT_ID" - - def path(self, **kwargs) -> str: - return "Projects/Search" - - -class Prospects(IncrementalInsightlyStream): - primary_key = "PROSPECT_ID" - - def path(self, **kwargs) -> str: - return "Prospect/Search" - - -class QuoteProducts(IncrementalInsightlyStream): - primary_key = "QUOTATION_ITEM_ID" - - def path(self, **kwargs) -> str: - return "QuotationLineItem/Search" - - -class Quotes(IncrementalInsightlyStream): - primary_key = "QUOTE_ID" - - def path(self, **kwargs) -> str: - return "Quotation/Search" - - -class Tasks(IncrementalInsightlyStream): - primary_key = "TASK_ID" - - def path(self, **kwargs) -> str: - return "Tasks/Search" - - -class Tickets(IncrementalInsightlyStream): - primary_key = "TICKET_ID" - - def path(self, **kwargs) -> str: - return "Ticket/Search" - - -class Users(IncrementalInsightlyStream): - primary_key = "USER_ID" - - def path(self, **kwargs) -> str: - return "Users/Search" - - -# Source -class SourceInsightly(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - token = config.get("token") - response = requests.get(f"{BASE_URL}Instance", auth=(token, "")) - response.raise_for_status() - - result = response.json() - logger.info(result) - - return True, None - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - - auth = BasicHttpAuthenticator(username=config.get("token"), password="") - return [ - ActivitySets(authenticator=auth, **config), - Contacts(authenticator=auth, **config), - Countries(authenticator=auth, **config), - Currencies(authenticator=auth, **config), - Emails(authenticator=auth, **config), - Events(authenticator=auth, **config), - KnowledgeArticleCategories(authenticator=auth, **config), - KnowledgeArticleFolders(authenticator=auth, **config), - KnowledgeArticles(authenticator=auth, **config), - LeadSources(authenticator=auth, **config), - LeadStatuses(authenticator=auth, **config), - Leads(authenticator=auth, **config), - Milestones(authenticator=auth, **config), - Notes(authenticator=auth, **config), - Opportunities(authenticator=auth, **config), - OpportunityCategories(authenticator=auth, **config), - OpportunityProducts(authenticator=auth, **config), - OpportunityStateReasons(authenticator=auth, **config), - Organisations(authenticator=auth, **config), - PipelineStages(authenticator=auth, **config), - Pipelines(authenticator=auth, **config), - PricebookEntries(authenticator=auth, **config), - Pricebooks(authenticator=auth, **config), - Products(authenticator=auth, **config), - ProjectCategories(authenticator=auth, **config), - Projects(authenticator=auth, **config), - Prospects(authenticator=auth, **config), - QuoteProducts(authenticator=auth, **config), - Quotes(authenticator=auth, **config), - Relationships(authenticator=auth, **config), - Tags(authenticator=auth, **config), - TaskCategories(authenticator=auth, **config), - Tasks(authenticator=auth, **config), - TeamMembers(authenticator=auth, **config), - Teams(authenticator=auth, **config), - Tickets(authenticator=auth, **config), - Users(authenticator=auth, **config), - ] +# Declarative Source +class SourceInsightly(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/spec.json b/airbyte-integrations/connectors/source-insightly/source_insightly/spec.json deleted file mode 100644 index a21504f3e553..000000000000 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/spec.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/insightly", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Insightly Spec", - "type": "object", - "required": ["token", "start_date"], - "additionalProperties": true, - "properties": { - "token": { - "type": ["string", "null"], - "title": "API Token", - "description": "Your Insightly API token.", - "airbyte_secret": true - }, - "start_date": { - "type": ["string", "null"], - "title": "Start Date", - "description": "The date from which you'd like to replicate data for Insightly in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated. Note that it will be used only for incremental streams.", - "examples": ["2021-03-01T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - } - } - } -} diff --git a/airbyte-integrations/connectors/source-insightly/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-insightly/unit_tests/test_incremental_streams.py deleted file mode 100644 index eb854c3bae84..000000000000 --- a/airbyte-integrations/connectors/source-insightly/unit_tests/test_incremental_streams.py +++ /dev/null @@ -1,81 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import pendulum -from airbyte_cdk.sources.streams.http.auth import BasicHttpAuthenticator -from pytest import fixture -from source_insightly.source import IncrementalInsightlyStream - -start_date = "2021-01-01T00:00:00Z" -authenticator = BasicHttpAuthenticator(username="test", password="") - - -@fixture -def patch_incremental_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(IncrementalInsightlyStream, "path", "v0/example_endpoint") - mocker.patch.object(IncrementalInsightlyStream, "primary_key", "test_primary_key") - mocker.patch.object(IncrementalInsightlyStream, "__abstractmethods__", set()) - - -def test_cursor_field(patch_incremental_base_class): - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - expected_cursor_field = "DATE_UPDATED_UTC" - assert stream.cursor_field == expected_cursor_field - - -def test_incremental_params(patch_incremental_base_class): - """ - After talking to the insightly team we learned that the DATE_UPDATED_UTC - cursor is exclusive. Subtracting 1 second from the previous state makes it inclusive. - """ - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - inputs = { - "stream_slice": None, - "stream_state": {"DATE_UPDATED_UTC": pendulum.datetime(2023, 5, 15, 18, 12, 44, tz="UTC")}, - "next_page_token": None, - } - expected_params = { - "count_total": True, - "skip": 0, - "top": 500, - "updated_after_utc": "2023-05-15T18:12:43Z", # 1 second subtracted from stream_state - } - assert stream.request_params(**inputs) == expected_params - - -def test_get_updated_state(patch_incremental_base_class): - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - inputs = { - "current_stream_state": {"DATE_UPDATED_UTC": "2021-01-01T00:00:00Z"}, - "latest_record": {"DATE_UPDATED_UTC": "2021-02-01T00:00:00Z"}, - } - expected_state = {"DATE_UPDATED_UTC": pendulum.datetime(2021, 2, 1, 0, 0, 0, tz="UTC")} - assert stream.get_updated_state(**inputs) == expected_state - - -def test_get_updated_state_no_current_state(patch_incremental_base_class): - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - inputs = {"current_stream_state": {}, "latest_record": {"DATE_UPDATED_UTC": "2021-01-01T00:00:00Z"}} - expected_state = {"DATE_UPDATED_UTC": pendulum.datetime(2021, 1, 1, 0, 0, 0, tz="UTC")} - assert stream.get_updated_state(**inputs) == expected_state - - -def test_supports_incremental(patch_incremental_base_class, mocker): - mocker.patch.object(IncrementalInsightlyStream, "cursor_field", "dummy_field") - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - assert stream.supports_incremental - - -def test_source_defined_cursor(patch_incremental_base_class): - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - assert stream.source_defined_cursor - - -def test_stream_checkpoint_interval(patch_incremental_base_class): - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - # TODO: replace this with your expected checkpoint interval - expected_checkpoint_interval = None - assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-insightly/unit_tests/test_source.py b/airbyte-integrations/connectors/source-insightly/unit_tests/test_source.py deleted file mode 100644 index 4e9f2c408756..000000000000 --- a/airbyte-integrations/connectors/source-insightly/unit_tests/test_source.py +++ /dev/null @@ -1,56 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock, patch - -from source_insightly.source import SourceInsightly - - -class MockResponse: - def __init__(self, json_data, status_code): - self.json_data = json_data - self.status_code = status_code - - def json(self): - return self.json_data - - def raise_for_status(self): - if self.status_code != 200: - raise Exception("Bad things happened") - - -def mocked_requests_get(fail=False): - def wrapper(*args, **kwargs): - if fail: - return MockResponse(None, 404) - - return MockResponse( - {"INSTANCE_NAME": "bossco", "INSTANCE_SUBDOMAIN": None, "PLAN_NAME": "Gratis", "NEW_USER_EXPERIENCE_ENABLED": True}, 200 - ) - - return wrapper - - -@patch("requests.get", side_effect=mocked_requests_get()) -def test_check_connection(mocker): - source = SourceInsightly() - logger_mock, config_mock = MagicMock(), MagicMock() - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -@patch("requests.get", side_effect=mocked_requests_get(fail=True)) -def test_check_connection_fail(mocker): - source = SourceInsightly() - logger_mock, config_mock = MagicMock(), MagicMock() - assert source.check_connection(logger_mock, config_mock)[0] is False - assert source.check_connection(logger_mock, config_mock)[1] is not None - - -def test_streams(mocker): - source = SourceInsightly() - config_mock = MagicMock() - streams = source.streams(config_mock) - - expected_streams_number = 37 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-insightly/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-insightly/unit_tests/test_streams.py deleted file mode 100644 index e44895f5db26..000000000000 --- a/airbyte-integrations/connectors/source-insightly/unit_tests/test_streams.py +++ /dev/null @@ -1,126 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from airbyte_cdk.sources.streams.http.auth import BasicHttpAuthenticator -from source_insightly.source import InsightlyStream - -authenticator = BasicHttpAuthenticator(username="test", password="") - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(InsightlyStream, "path", "v0/example_endpoint") - mocker.patch.object(InsightlyStream, "primary_key", "test_primary_key") - mocker.patch.object(InsightlyStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {"count_total": True, "skip": 0, "top": 500} - assert stream.request_params(**inputs) == expected_params - - -def test_request_param_with_next_page_token(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": 1000} - expected_params = {"count_total": True, "skip": 1000, "top": 500} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - stream.total_count = 10000 - - request = MagicMock() - request.url = "https://api.insight.ly/v0/example_endpoint?count_total=True&skip=0&top=500" - response = MagicMock() - response.status_code = HTTPStatus.OK - response.request = request - - inputs = {"response": response} - expected_token = 500 - assert stream.next_page_token(**inputs) == expected_token - - -def test_next_page_token_last_records(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - stream.total_count = 2100 - - request = MagicMock() - request.url = "https://api.insight.ly/v0/example_endpoint?count_total=True&skip=1500&top=500" - response = MagicMock() - response.status_code = HTTPStatus.OK - response.request = request - - inputs = {"response": response} - expected_token = 2000 - assert stream.next_page_token(**inputs) == expected_token - - -def test_next_page_token_no_more_records(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - stream.total_count = 1000 - - request = MagicMock() - request.url = "https://api.insight.ly/v0/example_endpoint?count_total=True&skip=1000&top=500" - response = MagicMock() - response.status_code = HTTPStatus.OK - response.request = request - - inputs = {"response": response} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_parse_response(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - - response = MagicMock() - response.json = MagicMock(return_value=[{"data_field": [{"keys": ["keys"]}]}]) - - inputs = {"stream_state": "test_stream_state", "response": response} - expected_parsed_object = response.json()[0] - assert next(stream.parse_response(**inputs)) == expected_parsed_object - - -def test_request_headers(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {"Accept": "application/json"} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = InsightlyStream(authenticator=authenticator) - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = InsightlyStream(authenticator=authenticator) - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time From d9c22b36500903b565f4b0170ab31f84d503c841 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Mon, 23 Oct 2023 20:45:11 -0300 Subject: [PATCH 2/9] made quick fixes --- .../connectors/source-insightly/Dockerfile | 2 +- .../connectors/source-insightly/build.gradle | 8 -------- airbyte-integrations/connectors/source-insightly/setup.py | 5 +---- docs/integrations/sources/insightly.md | 1 + 4 files changed, 3 insertions(+), 13 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-insightly/build.gradle diff --git a/airbyte-integrations/connectors/source-insightly/Dockerfile b/airbyte-integrations/connectors/source-insightly/Dockerfile index 2eb27731e7a7..6a744a678910 100644 --- a/airbyte-integrations/connectors/source-insightly/Dockerfile +++ b/airbyte-integrations/connectors/source-insightly/Dockerfile @@ -34,5 +34,5 @@ COPY source_insightly ./source_insightly ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-insightly diff --git a/airbyte-integrations/connectors/source-insightly/build.gradle b/airbyte-integrations/connectors/source-insightly/build.gradle deleted file mode 100644 index c23c48b88275..000000000000 --- a/airbyte-integrations/connectors/source-insightly/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'source_insightly' -} diff --git a/airbyte-integrations/connectors/source-insightly/setup.py b/airbyte-integrations/connectors/source-insightly/setup.py index afac830b077e..a3c070098791 100644 --- a/airbyte-integrations/connectors/source-insightly/setup.py +++ b/airbyte-integrations/connectors/source-insightly/setup.py @@ -5,15 +5,12 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", -] +MAIN_REQUIREMENTS = ["airbyte-cdk"] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/docs/integrations/sources/insightly.md b/docs/integrations/sources/insightly.md index b51287925972..24c5aa71dba9 100644 --- a/docs/integrations/sources/insightly.md +++ b/docs/integrations/sources/insightly.md @@ -71,6 +71,7 @@ The connector is restricted by Insightly [requests limitation](https://api.na1.i | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------- | +| 0.2.0 | 2023-10-23 |[31162](https://github.com/airbytehq/airbyte/pull/31162) | Migrate to low-code framework | | 0.1.3 | 2023-05-15 |[26079](https://github.com/airbytehq/airbyte/pull/26079) | Make incremental syncs timestamp inclusive | | 0.1.2 | 2023-03-23 |[24422](https://github.com/airbytehq/airbyte/pull/24422) | Fix incremental timedelta causing missing records | | 0.1.1 | 2022-11-11 |[19356](https://github.com/airbytehq/airbyte/pull/19356) | Fix state date parse bug | From 59e54d9177575a59eb99a3bb8d7002aa1b0a6c74 Mon Sep 17 00:00:00 2001 From: sajarin Date: Mon, 13 Nov 2023 12:30:33 -0500 Subject: [PATCH 3/9] fix: update manifest, configured_catalog and get tests passing --- .../acceptance-test-config.yml | 35 +- .../connectors/source-insightly/icon.svg | 1 + .../integration_tests/abnormal_state.json | 136 +++++- .../integration_tests/configured_catalog.json | 420 +++++++++++++++++- .../integration_tests/sample_state.json | 4 +- .../source_insightly/manifest.yaml | 170 ++++--- .../source_insightly/schemas/contacts.json | 7 + .../schemas/opportunities.json | 11 +- .../schemas/organisations.json | 7 + .../source_insightly/schemas/projects.json | 7 + .../source_insightly/schemas/tasks.json | 4 + .../source_insightly/schemas/teams.json | 3 +- 12 files changed, 714 insertions(+), 91 deletions(-) create mode 100644 airbyte-integrations/connectors/source-insightly/icon.svg diff --git a/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml b/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml index abea18882106..f66cb596b587 100644 --- a/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml @@ -18,13 +18,46 @@ acceptance_tests: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: emails + bypass_reason: "no data for this stream in our sandbox account" + - name: events + bypass_reason: "no data for this stream in our sandbox account" + - name: milestones + bypass_reason: "no data for this stream in our sandbox account" + - name: notes + bypass_reason: "no data for this stream in our sandbox account" + - name: opportunity_categories + bypass_reason: "no data for this stream in our sandbox account" + - name: project_categories + bypass_reason: "no data for this stream in our sandbox account" + - name: knowledge_article_categories + bypass_reason: "current sandbox account does not have permissions to access this stream (403 error)" + - name: knowledge_article_folders + bypass_reason: "current sandbox account does not have permissions to access this stream (403 error)" + - name: knowledge_articles + bypass_reason: "current sandbox account does not have permissions to access this stream (403 error)" + - name: lead_sources + bypass_reason: "current sandbox account does not have permissions to access this stream (403 error)" + - name: lead_statuses + bypass_reason: "current sandbox account does not have permissions to access this stream (403 error)" + - name: prospects + bypass_reason: "current sandbox account does not have permissions to access this stream (403 error)" + full_refresh: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + ignored_fields: + contacts: + - name: "IMAGE_URL" + bypass_reason: "image url is a dynamic s3 url with a changing expiration date in the query parameter" + organisations: + - name: "IMAGE_URL" + bypass_reason: "image url is a dynamic s3 url with a changing expiration date in the query parameter" incremental: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state: - future_state_path: integration_tests/abnormal_state.json \ No newline at end of file + future_state_path: integration_tests/abnormal_state.json diff --git a/airbyte-integrations/connectors/source-insightly/icon.svg b/airbyte-integrations/connectors/source-insightly/icon.svg new file mode 100644 index 000000000000..d0741d947f6b --- /dev/null +++ b/airbyte-integrations/connectors/source-insightly/icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-insightly/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-insightly/integration_tests/abnormal_state.json index 60d25c39160d..f49326b9fd77 100644 --- a/airbyte-integrations/connectors/source-insightly/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-insightly/integration_tests/abnormal_state.json @@ -1,4 +1,136 @@ [ + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "contacts" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "events" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "knowledge_article_categories" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "knowledge_article_folders" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "knowledge_articles" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "milestones" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "notes" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "opportunities" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "organisations" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "projects" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "prospects" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "tasks" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, { "type": "STREAM", "stream": { @@ -6,8 +138,8 @@ "name": "users" }, "stream_state": { - "DATE_UPDATED_UTC": "2122-10-17T19:10:14+00:00" + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" } } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-insightly/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-insightly/integration_tests/configured_catalog.json index b42c7123143b..60107bacf103 100644 --- a/airbyte-integrations/connectors/source-insightly/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-insightly/integration_tests/configured_catalog.json @@ -1,26 +1,424 @@ { "streams": [ { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "team_members", + "default_cursor_field": null, "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["MEMBER_USER_ID"]] + "name": "activity_sets", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["ACTIVITYSET_ID"]], + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "users", + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "contacts", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["CONTACT_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "countries", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["COUNTRY_NAME"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "currencies", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["CURRENCY_CODE"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "emails", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["EMAIL_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "events", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["EVENT_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "knowledge_article_categories", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["CATEGORY_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "knowledge_article_folders", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["FOLDER_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "knowledge_articles", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["ARTICLE_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "lead_sources", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["LEAD_SOURCE_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "lead_statuses", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["LEAD_STATUS_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "milestones", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["MILESTONE_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "notes", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["NOTE_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "opportunities", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["OPPORTUNITY_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "opportunity_categories", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["CATEGORY_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "opportunity_state_reasons", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["STATE_REASON_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "organisations", + "namespace": null, "source_defined_cursor": true, + "source_defined_primary_key": [["ORGANISATION_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "pipelines", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["PIPELINE_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "pipeline_stages", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["STAGE_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "project_categories", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["CATEGORY_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { "default_cursor_field": ["DATE_UPDATED_UTC"], - "source_defined_primary_key": [["USER_ID"]] + "json_schema": {}, + "name": "projects", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["PROJECT_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "prospects", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["PROSPECT_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "relationships", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["RELATIONSHIP_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "task_categories", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["CATEGORY_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "tasks", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["TASK_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "team_members", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["MEMBER_USER_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "teams", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["TEAM_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "users", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["USER_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "incremental", - "destination_sync_mode": "append" + "sync_mode": "full_refresh" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-insightly/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-insightly/integration_tests/sample_state.json index 48974dd13c99..672363e87f6e 100644 --- a/airbyte-integrations/connectors/source-insightly/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-insightly/integration_tests/sample_state.json @@ -6,8 +6,8 @@ "name": "users" }, "stream_state": { - "DATE_UPDATED_UTC": "2022-10-17T19:10:14+00:00" + "DATE_UPDATED_UTC": "2022-10-17 19:10:14" } } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml b/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml index 37ae545edde1..f4e8af347d3c 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml @@ -6,6 +6,7 @@ definitions: extractor: type: DpathExtractor field_path: [] + requester: type: HttpRequester url_base: "https://api.na1.insightly.com/v3.1/" @@ -13,9 +14,14 @@ definitions: authenticator: type: BasicHttpAuthenticator username: "{{ config['token'] }}" - request_options_provider: - request_parameters: - count_total: True, + request_parameters: + count_total: "True" + updated_after_utc: "{{ stream_state['DATE_UPDATED_UTC'] }}" + error_handler: + response_filters: + - http_codes: [403] + action: IGNORE #ignore 403 errors for knowledge_article streams, lead streams and prospects + date_incremental_sync: type: DatetimeBasedCursor start_datetime: @@ -24,10 +30,11 @@ definitions: end_datetime: datetime: "{{ now_utc() }}" datetime_format: "%Y-%m-%d %H:%M:%S.%f+00:00" - datetime_format: "%Y-%m-%dT%H:%M:%S" + datetime_format: "%Y-%m-%d %H:%M:%S" cursor_granularity: PT1S - step: P1D + step: P30D # Step should reflect the actual time granularity you need cursor_field: "DATE_UPDATED_UTC" + retriever: type: SimpleRetriever record_selector: @@ -46,231 +53,270 @@ definitions: field_name: "top" requester: $ref: "#/definitions/requester" + base_stream: type: DeclarativeStream retriever: $ref: "#/definitions/retriever" + base_incremental_stream: incremental_sync: $ref: "#/definitions/date_incremental_sync" retriever: $ref: "#/definitions/retriever" + activity_sets_stream: $ref: "#/definitions/base_stream" name: "activity_sets" primary_key: "ACTIVITYSET_ID" $parameters: path: "/ActivitySets" + contacts_stream: $ref: "#/definitions/base_incremental_stream" name: "contacts" primary_key: "CONTACT_ID" $parameters: path: "/Contacts/Search" + countries_stream: $ref: "#/definitions/base_stream" name: "countries" primary_key: "COUNTRY_NAME" $parameters: path: "/Countries" + currencies_stream: $ref: "#/definitions/base_stream" name: "currencies" primary_key: "CURRENCY_CODE" $parameters: path: "/Currencies" + emails_stream: $ref: "#/definitions/base_stream" name: "emails" primary_key: "EMAIL_ID" $parameters: path: "/Emails" + events_stream: $ref: "#/definitions/base_incremental_stream" name: "events" primary_key: "EVENT_ID" $parameters: path: "/Events/Search" + knowledge_article_categories_stream: $ref: "#/definitions/base_incremental_stream" name: "knowledge_article_categories" primary_key: "CATEGORY_ID" $parameters: path: "/KnowledgeArticleCategory/Search" + knowledge_article_folders_stream: $ref: "#/definitions/base_incremental_stream" name: "knowledge_article_folders" primary_key: "FOLDER_ID" $parameters: path: "/KnowledgeArticleFolder/Search" + knowledge_articles_stream: $ref: "#/definitions/base_incremental_stream" name: "knowledge_articles" primary_key: "ARTICLE_ID" $parameters: path: "/KnowledgeArticle/Search" + leads_stream: $ref: "#/definitions/base_incremental_stream" name: "leads" primary_key: "LEAD_ID" $parameters: - path: "/Leads/Search" + path: "/Leads" + lead_sources_stream: $ref: "#/definitions/base_stream" name: "lead_sources" primary_key: "LEAD_SOURCE_ID" $parameters: path: "/LeadSources" + lead_statuses_stream: $ref: "#/definitions/base_stream" name: "lead_statuses" primary_key: "LEAD_STATUS_ID" $parameters: path: "/LeadStatuses" + milestones_stream: $ref: "#/definitions/base_incremental_stream" name: "milestones" primary_key: "MILESTONE_ID" $parameters: path: "/Milestones/Search" + notes_stream: $ref: "#/definitions/base_incremental_stream" name: "notes" primary_key: "NOTE_ID" $parameters: path: "/Notes/Search" + opportunities_stream: $ref: "#/definitions/base_incremental_stream" name: "opportunities" primary_key: "OPPORTUNITY_ID" $parameters: path: "/Opportunities/Search" + opportunity_categories_stream: $ref: "#/definitions/base_stream" name: "opportunity_categories" primary_key: "CATEGORY_ID" $parameters: path: "/OpportunityCategories" + opportunity_products_stream: $ref: "#/definitions/base_incremental_stream" name: "opportunity_products" primary_key: "OPPORTUNITY_ITEM_ID" $parameters: - path: "/OpportunityLineItem/Search" + path: "/OpportunityLineItem" + opportunity_state_reasons_stream: $ref: "#/definitions/base_stream" name: "opportunity_state_reasons" primary_key: "STATE_REASON_ID" $parameters: path: "/OpportunityStateReasons" - organizations_stream: + + organisations_stream: $ref: "#/definitions/base_incremental_stream" - name: "organizations" - primary_key: "ORGANIZATION_ID" + name: "organisations" + primary_key: "ORGANISATION_ID" $parameters: - path: "/Organizations/Search" + path: "/Organisations" + pipelines_stream: $ref: "#/definitions/base_stream" name: "pipelines" primary_key: "PIPELINE_ID" $parameters: path: "/Pipelines" + pipeline_stages_stream: $ref: "#/definitions/base_stream" name: "pipeline_stages" primary_key: "STAGE_ID" $parameters: path: "/PipelineStages" + pricebook_entries_stream: $ref: "#/definitions/base_incremental_stream" name: "price_book_entries" primary_key: "PRICEBOOK_ENTRY_ID" $parameters: - path: "/PricebookEntry/Search" + path: "/PricebookEntry" + pricebooks_stream: $ref: "#/definitions/base_incremental_stream" name: "price_books" primary_key: "PRICEBOOK_ID" $parameters: - path: "/Pricebook/Search" + path: "/Pricebook" + products_stream: $ref: "#/definitions/base_incremental_stream" name: "products" primary_key: "PRODUCT_ID" $parameters: - path: "/Product/Search" + path: "/Product" + project_categories_stream: $ref: "#/definitions/base_stream" name: "project_categories" primary_key: "CATEGORY_ID" $parameters: path: "/ProjectCategories" + projects_stream: $ref: "#/definitions/base_incremental_stream" name: "projects" primary_key: "PROJECT_ID" $parameters: path: "/Projects/Search" + prospects_stream: $ref: "#/definitions/base_incremental_stream" name: "prospects" primary_key: "PROSPECT_ID" $parameters: path: "/Prospect/Search" + quote_products_stream: $ref: "#/definitions/base_incremental_stream" name: "quote_products" primary_key: "QUOTATION_ITEM_ID" $parameters: - path: "/QuotationLineItem/Search" + path: "/QuotationLineItem" + quotes_stream: $ref: "#/definitions/base_incremental_stream" name: "quotes" primary_key: "QUOTE_ID" $parameters: - path: "/Quotation/Search" + path: "/Quotation" + relationships_stream: $ref: "#/definitions/base_stream" name: "relationships" primary_key: "RELATIONSHIP_ID" $parameters: path: "/Relationships" + tags_stream: $ref: "#/definitions/base_stream" name: "tags" primary_key: "TAG_NAME" $parameters: path: "/Tags" + task_categories_stream: $ref: "#/definitions/base_stream" name: "task_categories" primary_key: "CATEGORY_ID" $parameters: path: "/TaskCategories" + tasks_stream: $ref: "#/definitions/base_incremental_stream" name: "tasks" primary_key: "TASK_ID" $parameters: path: "/Tasks/Search" + team_members_stream: $ref: "#/definitions/base_stream" name: "team_members" primary_key: "MEMBER_USER_ID" $parameters: path: "/TeamMembers" + teams_stream: $ref: "#/definitions/base_stream" name: "teams" primary_key: "TEAM_ID" $parameters: path: "/Teams" + tickets_stream: $ref: "#/definitions/base_incremental_stream" name: "tickets" primary_key: "TICKET_ID" $parameters: - path: "/Tickets/Search" + path: "/Tickets" + users_stream: $ref: "#/definitions/base_incremental_stream" name: "users" @@ -288,75 +334,54 @@ streams: - "#/definitions/knowledge_article_categories_stream" - "#/definitions/knowledge_article_folders_stream" - "#/definitions/knowledge_articles_stream" - - "#/definitions/leads_stream" + # - "#/definitions/leads_stream" - "#/definitions/lead_sources_stream" - "#/definitions/lead_statuses_stream" - "#/definitions/milestones_stream" - "#/definitions/notes_stream" - "#/definitions/opportunities_stream" - "#/definitions/opportunity_categories_stream" - - "#/definitions/opportunity_products_stream" + # - "#/definitions/opportunity_products_stream" - "#/definitions/opportunity_state_reasons_stream" - - "#/definitions/organizations_stream" + - "#/definitions/organisations_stream" - "#/definitions/pipelines_stream" - "#/definitions/pipeline_stages_stream" - - "#/definitions/pricebook_entries_stream" - - "#/definitions/pricebooks_stream" - - "#/definitions/products_stream" + # - "#/definitions/pricebook_entries_stream" + # - "#/definitions/pricebooks_stream" + # - "#/definitions/products_stream" - "#/definitions/project_categories_stream" - "#/definitions/projects_stream" - "#/definitions/prospects_stream" - - "#/definitions/quote_products_stream" - - "#/definitions/quotes_stream" + # - "#/definitions/quote_products_stream" + # - "#/definitions/quotes_stream" - "#/definitions/relationships_stream" - - "#/definitions/tags_stream" + # - "#/definitions/tags_stream" - "#/definitions/task_categories_stream" - "#/definitions/tasks_stream" - "#/definitions/team_members_stream" - "#/definitions/teams_stream" - - "#/definitions/tickets_stream" + # - "#/definitions/tickets_stream" - "#/definitions/users_stream" check: type: CheckStream stream_names: - "activity_sets" - - "contacts" - - "countries" - - "currencies" - - "emails" - - "events" - - "knowledge_article_categories" - - "knowledge_article_folders" - - "knowledge_articles" - - "leads" - - "lead_sources" - - "lead_statuses" - - "milestones" - - "notes" - - "opportunities" - - "opportunity_categories" - - "opportunity_products" - - "opportunity_state_reasons" - - "organizations" - - "pipelines" - - "pipeline_stages" - - "price_book_entries" - - "price_books" - - "products" - - "project_categories" - - "projects" - - "prospects" - - "quote_products" - - "quotes" - - "relationships" - - "tags" - - "task_categories" - - "tasks" - - "team_members" - - "teams" - - "tickets" - - "users" + # - "contacts" + # - "countries" + # - "currencies" + # - "opportunities" + # - "opportunity_state_reasons" + # - "organizations" + # - "pipelines" + # - "pipeline_stages" + # - "projects" + # - "relationships" + # - "task_categories" + # - "tasks" + # - "team_members" + # - "teams" + # - "users" spec: type: Spec @@ -366,25 +391,26 @@ spec: title: Insightly Spec type: object required: - - token - - start_date + - token + - start_date additionalProperties: true properties: token: type: - - string - - 'null' + - string + - "null" title: API Token description: Your Insightly API token. airbyte_secret: true start_date: type: - - string - - 'null' + - string + - "null" title: Start Date - description: The date from which you'd like to replicate data for Insightly + description: + The date from which you'd like to replicate data for Insightly in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated. Note that it will be used only for incremental streams. examples: - - '2021-03-01T00:00:00Z' - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" \ No newline at end of file + - "2021-03-01T00:00:00Z" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/contacts.json b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/contacts.json index dd0b5039fe01..6b6fd0672c44 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/contacts.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, "type": "object", "properties": { "CONTACT_ID": { @@ -114,6 +115,12 @@ "TITLE": { "type": ["string", "null"] }, + "VISIBLE_TEAM_ID": { + "type": ["integer", "null"] + }, + "VISIBLE_TO": { + "type": ["string", "null"] + }, "EMAIL_OPTED_OUT": { "type": ["boolean", "null"] }, diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/opportunities.json b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/opportunities.json index e154e26090d2..ca20e33d36b8 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/opportunities.json +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/opportunities.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, "type": "object", "properties": { "OPPORTUNITY_ID": { @@ -27,7 +28,7 @@ "type": ["string", "null"] }, "BID_AMOUNT": { - "type": ["integer", "null"] + "type": ["number", "null"] }, "BID_TYPE": { "type": ["string", "null"] @@ -48,7 +49,7 @@ "format": "date-time" }, "OPPORTUNITY_VALUE": { - "type": ["integer", "null"] + "type": ["number", "null"] }, "PROBABILITY": { "type": ["integer", "null"] @@ -83,6 +84,12 @@ "PRICEBOOK_ID": { "type": ["integer", "null"] }, + "VISIBLE_TEAM_ID": { + "type": ["integer", "null"] + }, + "VISIBLE_TO": { + "type": ["string", "null"] + }, "CUSTOMFIELDS": { "type": "array", "items": { diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/organisations.json b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/organisations.json index b2dd05169110..9f7c844de6ad 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/organisations.json +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/organisations.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, "type": "object", "properties": { "ORGANISATION_ID": { @@ -84,6 +85,12 @@ "SOCIAL_TWITTER": { "type": ["string", "null"] }, + "VISIBLE_TEAM_ID": { + "type": ["integer", "null"] + }, + "VISIBLE_TO": { + "type": ["string", "null"] + }, "CUSTOMFIELDS": { "type": "array", "items": { diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/projects.json b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/projects.json index fa91ca651ef4..d11b04949b52 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/projects.json +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/projects.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, "type": "object", "properties": { "PROJECT_ID": { @@ -60,6 +61,12 @@ "RESPONSIBLE_USER_ID": { "type": ["integer", "null"] }, + "VISIBLE_TEAM_ID": { + "type": ["integer", "null"] + }, + "VISIBLE_TO": { + "type": ["string", "null"] + }, "CUSTOMFIELDS": { "type": "array", "items": { diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/tasks.json b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/tasks.json index 6b41220e9e10..752a64002f8f 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/tasks.json +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/tasks.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, "type": "object", "properties": { "TASK_ID": { @@ -34,6 +35,9 @@ "PERCENT_COMPLETE": { "type": ["integer", "null"] }, + "PUBLICLY_VISIBLE": { + "type": ["null", "boolean"] + }, "START_DATE": { "type": ["string", "null"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/teams.json b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/teams.json index b8454b8b393e..367acda41e53 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/teams.json +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/teams.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, "type": "object", "properties": { "TEAM_ID": { @@ -20,7 +21,7 @@ "format": "date-time" }, "TEAMMEMBERS": { - "type": "object" + "type": "array" } } } From 5a8c7248884836b5ee24978b18ba94eb0e59b0b3 Mon Sep 17 00:00:00 2001 From: sajarin Date: Mon, 13 Nov 2023 17:44:46 +0000 Subject: [PATCH 4/9] Automated Commit - Formatting Changes --- .../source-insightly/integration_tests/invalid_config.json | 2 +- .../source-insightly/integration_tests/sample_config.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-insightly/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-insightly/integration_tests/invalid_config.json index f9c6dff6b868..ac0271de4fbd 100644 --- a/airbyte-integrations/connectors/source-insightly/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-insightly/integration_tests/invalid_config.json @@ -1,4 +1,4 @@ { "token": "bad-token", "start_date": "2019-01-01T00:00:00Z" -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-insightly/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-insightly/integration_tests/sample_config.json index 6da5a9d217f6..2dd9a2f7b016 100644 --- a/airbyte-integrations/connectors/source-insightly/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-insightly/integration_tests/sample_config.json @@ -1,4 +1,4 @@ { "token": "my-insightly-api-token", "start_date": "2022-10-01T00:00:00Z" -} \ No newline at end of file +} From 636a27dd2016fd3694551dc253ae212bb96f5a0c Mon Sep 17 00:00:00 2001 From: sajarin Date: Mon, 13 Nov 2023 12:46:44 -0500 Subject: [PATCH 5/9] fix: update url links with proper search endpoints --- .../source_insightly/manifest.yaml | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml b/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml index f4e8af347d3c..44d2377339e0 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml @@ -98,7 +98,7 @@ definitions: name: "emails" primary_key: "EMAIL_ID" $parameters: - path: "/Emails" + path: "/Emails/Search" events_stream: $ref: "#/definitions/base_incremental_stream" @@ -133,7 +133,7 @@ definitions: name: "leads" primary_key: "LEAD_ID" $parameters: - path: "/Leads" + path: "/Leads/Search" lead_sources_stream: $ref: "#/definitions/base_stream" @@ -182,7 +182,7 @@ definitions: name: "opportunity_products" primary_key: "OPPORTUNITY_ITEM_ID" $parameters: - path: "/OpportunityLineItem" + path: "/OpportunityLineItem/Search" opportunity_state_reasons_stream: $ref: "#/definitions/base_stream" @@ -196,7 +196,7 @@ definitions: name: "organisations" primary_key: "ORGANISATION_ID" $parameters: - path: "/Organisations" + path: "/Organisations/Search" pipelines_stream: $ref: "#/definitions/base_stream" @@ -217,21 +217,21 @@ definitions: name: "price_book_entries" primary_key: "PRICEBOOK_ENTRY_ID" $parameters: - path: "/PricebookEntry" + path: "/PricebookEntry/Search" pricebooks_stream: $ref: "#/definitions/base_incremental_stream" name: "price_books" primary_key: "PRICEBOOK_ID" $parameters: - path: "/Pricebook" + path: "/Pricebook/Search" products_stream: $ref: "#/definitions/base_incremental_stream" name: "products" primary_key: "PRODUCT_ID" $parameters: - path: "/Product" + path: "/Product/Search" project_categories_stream: $ref: "#/definitions/base_stream" @@ -259,14 +259,14 @@ definitions: name: "quote_products" primary_key: "QUOTATION_ITEM_ID" $parameters: - path: "/QuotationLineItem" + path: "/QuotationLineItem/Search" quotes_stream: $ref: "#/definitions/base_incremental_stream" name: "quotes" primary_key: "QUOTE_ID" $parameters: - path: "/Quotation" + path: "/Quotation/Search" relationships_stream: $ref: "#/definitions/base_stream" @@ -315,7 +315,7 @@ definitions: name: "tickets" primary_key: "TICKET_ID" $parameters: - path: "/Tickets" + path: "/Ticket/Search" users_stream: $ref: "#/definitions/base_incremental_stream" @@ -343,7 +343,7 @@ streams: - "#/definitions/opportunity_categories_stream" # - "#/definitions/opportunity_products_stream" - "#/definitions/opportunity_state_reasons_stream" - - "#/definitions/organisations_stream" + - "#/definitions/organizations_stream" - "#/definitions/pipelines_stream" - "#/definitions/pipeline_stages_stream" # - "#/definitions/pricebook_entries_stream" From c9f0d92f71f41edaaed14105e3c0a81b659936bb Mon Sep 17 00:00:00 2001 From: sajarin Date: Mon, 13 Nov 2023 13:04:32 -0500 Subject: [PATCH 6/9] fix: correct organisations stream definition --- .../connectors/source-insightly/source_insightly/manifest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml b/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml index 44d2377339e0..d32620efb6a3 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml @@ -343,7 +343,7 @@ streams: - "#/definitions/opportunity_categories_stream" # - "#/definitions/opportunity_products_stream" - "#/definitions/opportunity_state_reasons_stream" - - "#/definitions/organizations_stream" + - "#/definitions/organisations_stream" - "#/definitions/pipelines_stream" - "#/definitions/pipeline_stages_stream" # - "#/definitions/pricebook_entries_stream" From f153d6e2200db70d9fb050b533b07c60ba320c93 Mon Sep 17 00:00:00 2001 From: sajarin Date: Mon, 13 Nov 2023 15:31:49 -0500 Subject: [PATCH 7/9] fix: add 429 error handling --- .../source_insightly/manifest.yaml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml b/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml index d32620efb6a3..d5392b0dcae4 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml @@ -18,9 +18,17 @@ definitions: count_total: "True" updated_after_utc: "{{ stream_state['DATE_UPDATED_UTC'] }}" error_handler: - response_filters: - - http_codes: [403] - action: IGNORE #ignore 403 errors for knowledge_article streams, lead streams and prospects + type: "CompositeErrorHandler" + error_handlers: + - response_filters: + - http_codes: [403] + action: IGNORE #ignore 403 errors for knowledge_article streams, lead streams and prospects + - response_filters: + - http_codes: [429] + action: RETRY + backoff_strategies: + - type: "ConstantBackoffStrategy" + backoff_time_in_seconds: 6.0 date_incremental_sync: type: DatetimeBasedCursor From 1c3dbfd643761bfeaa9ceca82a10b2359d89acf4 Mon Sep 17 00:00:00 2001 From: sajarin Date: Tue, 14 Nov 2023 10:23:29 -0500 Subject: [PATCH 8/9] fix: update check streams --- .../connectors/source-insightly/source_insightly/manifest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml b/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml index d5392b0dcae4..5f28969be084 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml @@ -375,7 +375,7 @@ check: type: CheckStream stream_names: - "activity_sets" - # - "contacts" + - "contacts" # - "countries" # - "currencies" # - "opportunities" From 79b98b406894201956b9ad2cb1fb13dc59501881 Mon Sep 17 00:00:00 2001 From: sajarin Date: Tue, 14 Nov 2023 10:47:17 -0500 Subject: [PATCH 9/9] fix: add ignored column for users stream --- .../connectors/source-insightly/acceptance-test-config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml b/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml index f66cb596b587..b0f10cdef373 100644 --- a/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml @@ -55,6 +55,9 @@ acceptance_tests: organisations: - name: "IMAGE_URL" bypass_reason: "image url is a dynamic s3 url with a changing expiration date in the query parameter" + users: + - name: "USER_CURRENCY" + bypass_reason: "this can change between sequential reads" incremental: tests: - config_path: "secrets/config.json"