From f00079a42d34aa0d30ab80de2b9d02fc73da9713 Mon Sep 17 00:00:00 2001 From: Aviraj Gour Date: Sun, 6 Aug 2023 16:25:53 +0530 Subject: [PATCH 1/6] migrated to low code --- .../connectors/source-freshservice/Dockerfile | 36 +- .../connectors/source-freshservice/README.md | 66 +- .../source-freshservice/__init__.py | 3 + .../acceptance-test-config.yml | 54 +- .../acceptance-test-docker.sh | 1 + .../integration_tests/__init__.py | 2 +- .../integration_tests/acceptance.py | 2 + .../integration_tests/invalid_config.json | 6 +- .../integration_tests/sample_config.json | 6 +- .../source-freshservice/metadata.yaml | 23 +- .../source-freshservice/requirements.txt | 2 +- .../sample_files/configured_catalog.json | 1046 ----------------- .../connectors/source-freshservice/setup.py | 9 +- .../source_freshservice/__init__.py | 2 +- .../source_freshservice/manifest.yaml | 190 +++ .../source_freshservice/schemas/agents.json | 8 +- .../source_freshservice/schemas/assets.json | 77 +- .../source_freshservice/schemas/changes.json | 129 +- .../schemas/locations.json | 67 +- .../source_freshservice/schemas/problems.json | 115 +- .../schemas/purchase_orders.json | 91 +- .../schemas/requesters.json | 107 +- .../source_freshservice/schemas/software.json | 66 +- .../source_freshservice/schemas/vendors.json | 66 +- .../source_freshservice/source.py | 76 +- .../source_freshservice/spec.json | 31 - .../source_freshservice/spec.yaml | 31 + .../source_freshservice/streams.py | 197 ---- .../unit_tests/__init__.py | 3 - .../unit_tests/conftest.py | 14 - .../unit_tests/test_source.py | 43 - .../unit_tests/test_stream.py | 57 - 32 files changed, 899 insertions(+), 1727 deletions(-) create mode 100644 airbyte-integrations/connectors/source-freshservice/__init__.py mode change 100644 => 100755 airbyte-integrations/connectors/source-freshservice/acceptance-test-docker.sh delete mode 100644 airbyte-integrations/connectors/source-freshservice/sample_files/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-freshservice/source_freshservice/manifest.yaml delete mode 100644 airbyte-integrations/connectors/source-freshservice/source_freshservice/spec.json create mode 100644 airbyte-integrations/connectors/source-freshservice/source_freshservice/spec.yaml delete mode 100644 airbyte-integrations/connectors/source-freshservice/source_freshservice/streams.py delete mode 100644 airbyte-integrations/connectors/source-freshservice/unit_tests/__init__.py delete mode 100644 airbyte-integrations/connectors/source-freshservice/unit_tests/conftest.py delete mode 100644 airbyte-integrations/connectors/source-freshservice/unit_tests/test_source.py delete mode 100644 airbyte-integrations/connectors/source-freshservice/unit_tests/test_stream.py diff --git a/airbyte-integrations/connectors/source-freshservice/Dockerfile b/airbyte-integrations/connectors/source-freshservice/Dockerfile index 68de82bef15a..965cbf8d720a 100644 --- a/airbyte-integrations/connectors/source-freshservice/Dockerfile +++ b/airbyte-integrations/connectors/source-freshservice/Dockerfile @@ -1,16 +1,38 @@ -FROM python:3.9-slim +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base WORKDIR /airbyte/integration_code -COPY source_freshservice ./source_freshservice + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only COPY main.py ./ -COPY setup.py ./ -RUN pip install . +COPY source_freshservice ./source_freshservice ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.1.0 +LABEL io.airbyte.version=2.0.0 LABEL io.airbyte.name=airbyte/source-freshservice diff --git a/airbyte-integrations/connectors/source-freshservice/README.md b/airbyte-integrations/connectors/source-freshservice/README.md index 8f50e4975edc..bc19d433edc6 100644 --- a/airbyte-integrations/connectors/source-freshservice/README.md +++ b/airbyte-integrations/connectors/source-freshservice/README.md @@ -1,35 +1,10 @@ # Freshservice Source -This is the repository for the Freshservice source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/freshservice). +This is the repository for the Freshservice configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/freshservice). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -39,22 +14,14 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/freshservice) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_freshservice/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/freshservice) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_freshservice/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source freshservice test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -79,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-freshservice:dev disco docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-freshservice:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-freshservice/__init__.py b/airbyte-integrations/connectors/source-freshservice/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshservice/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml b/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml index ea3b89747601..339dbaaf6d69 100644 --- a/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml @@ -1,37 +1,33 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-freshservice:dev -tests: +acceptance_tests: spec: - - spec_path: "source_freshservice/spec.json" + tests: + - spec_path: "source_freshservice/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.1" + tests: + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "1.1.0" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: - [ - "assets", - "releases", - "problems", - "software", - "products", - "changes", - "vendors", - "purchase_orders", - "satisfaction_survey_responses", - ] - incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: ["name":"satisfaction_survey_responses","name":"releases"] + incremental: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-freshservice/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-freshservice/acceptance-test-docker.sh old mode 100644 new mode 100755 index 5797d20fe9a7..b6d65deeccb4 --- a/airbyte-integrations/connectors/source-freshservice/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-freshservice/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-freshservice/integration_tests/__init__.py b/airbyte-integrations/connectors/source-freshservice/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-freshservice/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-freshservice/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-freshservice/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-freshservice/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-freshservice/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-freshservice/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-freshservice/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-freshservice/integration_tests/invalid_config.json index 294b2030787c..fc8aed54b155 100644 --- a/airbyte-integrations/connectors/source-freshservice/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-freshservice/integration_tests/invalid_config.json @@ -1,5 +1,5 @@ { - "domain_name": "mydomain.freshservice.com", - "api_key": "", - "start_date": "2123-09-01T00:00:00Z" + "domain_name":"domain.freshservice.com", + "api_key": "invalid_api_key", + "start_date":"2020-10-01T00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-freshservice/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-freshservice/integration_tests/sample_config.json index 1f60922419fd..cfa25ee511e1 100644 --- a/airbyte-integrations/connectors/source-freshservice/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-freshservice/integration_tests/sample_config.json @@ -1,5 +1,5 @@ { - "domain_name": "mydomain.freshservice.com", - "api_key": "12345", - "start_date": "2021-09-01T00:00:00Z" + "domain_name":"domain.freshservice.com", + "api_key": "api_key", + "start_date":"2020-10-01T00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-freshservice/metadata.yaml b/airbyte-integrations/connectors/source-freshservice/metadata.yaml index 9dda68aedd0b..6af4b29ccb30 100644 --- a/airbyte-integrations/connectors/source-freshservice/metadata.yaml +++ b/airbyte-integrations/connectors/source-freshservice/metadata.yaml @@ -1,24 +1,25 @@ data: + allowedHosts: + hosts: + - TODO # Please change to the hostname of the source. + registries: + oss: + enabled: false + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: 9bb85338-ea95-4c93-b267-6be89125b267 - dockerImageTag: 1.1.0 + dockerImageTag: 2.0.0 dockerRepository: airbyte/source-freshservice githubIssueLabel: source-freshservice icon: freshservice.svg license: MIT name: Freshservice - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: TODO releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/freshservice tags: - - language:python - ab_internal: - sl: 100 - ql: 200 - supportLevel: community + - language:lowcode metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-freshservice/requirements.txt b/airbyte-integrations/connectors/source-freshservice/requirements.txt index 91de78ac4144..cc57334ef619 100644 --- a/airbyte-integrations/connectors/source-freshservice/requirements.txt +++ b/airbyte-integrations/connectors/source-freshservice/requirements.txt @@ -1,2 +1,2 @@ -e ../../bases/connector-acceptance-test --e . \ No newline at end of file +-e . diff --git a/airbyte-integrations/connectors/source-freshservice/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-freshservice/sample_files/configured_catalog.json deleted file mode 100644 index 141f41b723cb..000000000000 --- a/airbyte-integrations/connectors/source-freshservice/sample_files/configured_catalog.json +++ /dev/null @@ -1,1046 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "tickets", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "attachments": { - "type": "array" - }, - "cc_emails": { - "type": "array" - }, - "department_id": { - "type": "integer" - }, - "custom_fields": { - "type": "object" - }, - "deleted": { - "type": "boolean" - }, - "description": { - "type": "string" - }, - "description_text": { - "type": "string" - }, - "due_by": { - "type": "string" - }, - "email": { - "type": "string" - }, - "email_config_id": { - "type": "integer" - }, - "fr_due_by": { - "type": "string" - }, - "fr_escalated": { - "type": "boolean" - }, - "fwd_emails": { - "type": "array" - }, - "group_id": { - "type": "integer" - }, - "id": { - "type": "integer" - }, - "is_escalated": { - "type": "boolean" - }, - "name": { - "type": "string" - }, - "phone": { - "type": "string" - }, - "priority": { - "type": "integer" - }, - "category": { - "type": "string" - }, - "sub_category": { - "type": "string" - }, - "item_category": { - "type": "string" - }, - "reply_cc_emails": { - "type": "array" - }, - "requester_id": { - "type": "integer" - }, - "responder_id": { - "type": "integer" - }, - "source": { - "type": "integer" - }, - "spam": { - "type": "boolean" - }, - "status": { - "type": "integer" - }, - "subject": { - "type": "string" - }, - "tags": { - "type": "array" - }, - "to_emails": { - "type": "array" - }, - "type": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - }, - "urgency": { - "type": "integer" - }, - "impact": { - "type": "integer" - } - } - }, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "problems", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "agent_id": { - "type": "integer" - }, - "requester_id": { - "type": "integer" - }, - "group_id": { - "type": "integer" - }, - "description": { - "type": "string" - }, - "description_text": { - "type": "string" - }, - "priority": { - "type": "integer" - }, - "status": { - "type": "integer" - }, - "impact": { - "type": "integer" - }, - "known_error": { - "type": "boolean" - }, - "subject": { - "type": "string" - }, - "due_by": { - "type": "string" - }, - "department_id": { - "type": "integer" - }, - "category": { - "type": "string" - }, - "sub_category": { - "type": "string" - }, - "item_category": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - }, - "associated_change": { - "type": "integer" - }, - "custom_fields": { - "type": "object" - }, - "analysis_fields": { - "type": "object" - } - } - }, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "changes", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "agent_id": { - "type": "integer" - }, - "description": { - "type": "string" - }, - "description_text": { - "type": "string" - }, - "requester_id": { - "type": "integer" - }, - "group_id": { - "type": "integer" - }, - "priority": { - "type": "integer" - }, - "impact": { - "type": "integer" - }, - "status": { - "type": "integer" - }, - "risk": { - "type": "integer" - }, - "change_type": { - "type": "integer" - }, - "approval_status": { - "type": "integer" - }, - "planned_start_date": { - "type": "integer" - }, - "planned_end_date": { - "type": "integer" - }, - "subject": { - "type": "string" - }, - "department_id": { - "type": "integer" - }, - "category": { - "type": "string" - }, - "sub_category": { - "type": "string" - }, - "item_category": { - "type": "string" - }, - "custom_fields": { - "type": "object" - }, - "maintenance_window": { - "type": "object" - }, - "blackout_window": { - "type": "object" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } - } - }, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "releases", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "agent_id": { - "type": "integer" - }, - "group_id": { - "type": "integer" - }, - "priority": { - "type": "integer" - }, - "status": { - "type": "integer" - }, - "release_type": { - "type": "integer" - }, - "subject": { - "type": "string" - }, - "description": { - "type": "string" - }, - "planned_start_date": { - "type": "string" - }, - "planned_end_date": { - "type": "string" - }, - "work_start_date": { - "type": "string" - }, - "work_end_date": { - "type": "string" - }, - "department_id": { - "type": "integer" - }, - "category": { - "type": "string" - }, - "sub_category": { - "type": "string" - }, - "item_category": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - }, - "associated_assets": { - "type": "array" - }, - "associated_changes": { - "type": "array" - }, - "custom_fields": { - "type": "object" - }, - "planning_fields": { - "type": "object" - } - } - }, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "requesters", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "first_name": { - "type": "string" - }, - "last_name": { - "type": "string" - }, - "job_title": { - "type": "string" - }, - "primary_email": { - "type": "string" - }, - "secondary_emails": { - "type": "array" - }, - "work_phone_number": { - "type": "string" - }, - "mobile_phone_number": { - "type": "string" - }, - "department_ids": { - "type": "array" - }, - "can_see_all_tickets_from_associated_departments": { - "type": "boolean" - }, - "reporting_manager_id": { - "type": "integer" - }, - "address": { - "type": "string" - }, - "time_zone": { - "type": "string" - }, - "time_format": { - "type": "string" - }, - "language": { - "type": "string" - }, - "location_id": { - "type": "integer" - }, - "background_information": { - "type": "string" - }, - "custom_fields": { - "type": "object" - }, - "active": { - "type": "boolean" - }, - "has_logged_in": { - "type": "boolean" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - }, - "is_agent": { - "type": "boolean" - } - } - }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "agents", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "first_name": { - "type": "string" - }, - "last_name": { - "type": "string" - }, - "occasional": { - "type": "boolean" - }, - "job_title": { - "type": "string" - }, - "email": { - "type": "string" - }, - "work_phone_number": { - "type": "string" - }, - "mobile_phone_number": { - "type": "string" - }, - "department_ids": { - "type": "array" - }, - "can_see_all_tickets_from_associated_departments": { - "type": "boolean" - }, - "reporting_manager_id": { - "type": "integer" - }, - "address": { - "type": "string" - }, - "time_zone": { - "type": "string" - }, - "time_format": { - "type": "string" - }, - "language": { - "type": "string" - }, - "location_id": { - "type": "integer" - }, - "background_information": { - "type": "string" - }, - "scoreboard_level_id": { - "type": "integer" - }, - "ticket_scope": { - "type": "string" - }, - "problem_scope": { - "type": "string" - }, - "change_scope": { - "type": "string" - }, - "release_scope": { - "type": "string" - }, - "group_ids": { - "type": "array" - }, - "member_of": { - "type": "array" - }, - "observer_of": { - "type": "array" - }, - "role_ids": { - "type": "array" - }, - "roles": { - "type": "array" - }, - "last_login_at": { - "type": "string" - }, - "last_active_at": { - "type": "string" - }, - "custom_fields": { - "type": "object" - }, - "has_logged_in": { - "type": "boolean" - }, - "active": { - "type": "boolean" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } - } - }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "locations", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "parent_location_id": { - "type": "integer" - }, - "primary_contact_id": { - "type": "integer" - }, - "line1": { - "type": "string" - }, - "line2": { - "type": "string" - }, - "city": { - "type": "string" - }, - "state": { - "type": "string" - }, - "country": { - "type": "string" - }, - "zipcode": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } - } - }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "products", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "asset_type_id": { - "type": "integer" - }, - "manufacturer": { - "type": "string" - }, - "status": { - "type": "string" - }, - "mode_of_procurement": { - "type": "string" - }, - "depreciation_type_id": { - "type": "integer" - }, - "description": { - "type": "string" - }, - "description_text": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } - } - }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "vendors", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "primary_contact_id": { - "type": "integer" - }, - "line1": { - "type": "string" - }, - "city": { - "type": "string" - }, - "state": { - "type": "string" - }, - "country": { - "type": "string" - }, - "zipcode": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } - } - }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "assets", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "display_id": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "asset_type_id": { - "type": "integer" - }, - "asset_tag": { - "type": "string" - }, - "impact": { - "type": "string" - }, - "author_type": { - "type": "string" - }, - "usage_type": { - "type": "string" - }, - "user_id": { - "type": "integer" - }, - "location_id": { - "type": "integer" - }, - "department_id": { - "type": "integer" - }, - "agent_id": { - "type": "integer" - }, - "group_id": { - "type": "integer" - }, - "assigned_on": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } - } - }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "purchase_orders", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "vendor_id": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "po_number": { - "type": "string" - }, - "vendor_details": { - "type": "string" - }, - "expected_delivery_date": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - }, - "created_by": { - "type": "string" - }, - "status": { - "type": "integer" - }, - "shipping_address": { - "type": "string" - }, - "billing_same_as_shipping": { - "type": "boolean" - }, - "billing_address": { - "type": "string" - }, - "currency_code": { - "type": "string" - }, - "conversion_rate": { - "type": "number" - }, - "department_id": { - "type": "integer" - }, - "discount_percentage": { - "type": "integer" - }, - "tax_percentage": { - "type": "integer" - }, - "shipping_cost": { - "type": "integer" - }, - "custom_fields": { - "type": "object" - }, - "purchase_items": { - "type": "array" - } - } - }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "software", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "application_type": { - "type": "string" - }, - "status": { - "type": "string" - }, - "publisher_id": { - "type": "integer" - }, - "managed_by_id": { - "type": "integer" - }, - "notes": { - "type": "string" - }, - "category": { - "type": "string" - }, - "sources": { - "type": "string" - }, - "user_count": { - "type": "integer" - }, - "installation_count": { - "type": "integer" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } - } - }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshservice/setup.py b/airbyte-integrations/connectors/source-freshservice/setup.py index b6f51d335627..8ccedf94055a 100644 --- a/airbyte-integrations/connectors/source-freshservice/setup.py +++ b/airbyte-integrations/connectors/source-freshservice/setup.py @@ -10,20 +10,19 @@ ] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", - "responses~=0.13.3", "connector-acceptance-test", ] setup( name="source_freshservice", description="Source implementation for Freshservice.", - author="Tuan Nguyen", - author_email="anhtuan.nguyen@me.com", + author="Airbyte", + author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/__init__.py b/airbyte-integrations/connectors/source-freshservice/source_freshservice/__init__.py index 32d0e324d0dc..f18cc5950ea0 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/__init__.py +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/manifest.yaml b/airbyte-integrations/connectors/source-freshservice/source_freshservice/manifest.yaml new file mode 100644 index 000000000000..c5f74cc42ca5 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/manifest.yaml @@ -0,0 +1,190 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["{{ parameters.path_extractor }}"] + requester: + type: HttpRequester + url_base: "https://{{config['domain_name']}}/api/v2/" + http_method: "GET" + authenticator: + type: "BasicHttpAuthenticator" + username: "{{config['api_key']}}" + password: "" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "PageIncrement" + page_size: 30 + start_from_page: 1 + page_token_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "page" + requester: + $ref: "#/definitions/requester" + + incremental_base: + type: DatetimeBasedCursor + cursor_field: "updated_at" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + field_name: "updated_since" + inject_into: "request_parameter" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + tickets_stream: + $ref: "#/definitions/base_stream" + name: "tickets" + primary_key: "id" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "tickets" + path: "/tickets" + + satisfaction_survey_responses_stream: + name: "satisfaction_survey_responses" + primary_key: "id" + $parameters: + path_extractor: "csat_response" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "tickets/{{ stream_slice.parent_id }}/csat_response" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/tickets_stream" + parent_key: "id" + partition_field: "parent_id" + + problems_stream: + $ref: "#/definitions/base_stream" + name: "problems" + primary_key: "id" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "problems" + path: "/problems" + + changes_stream: + $ref: "#/definitions/base_stream" + name: "changes" + primary_key: "id" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "changes" + path: "/changes" + + releases_stream: + $ref: "#/definitions/base_stream" + name: "releases" + primary_key: "id" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "releases" + path: "/changes" + + requesters_stream: + $ref: "#/definitions/base_stream" + name: "requesters" + primary_key: "id" + $parameters: + path_extractor: "requesters" + path: "/requesters" + + agents_stream: + $ref: "#/definitions/base_stream" + name: "agents" + primary_key: "id" + $parameters: + path_extractor: "agents" + path: "/agents" + + locations_stream: + $ref: "#/definitions/base_stream" + name: "locations" + primary_key: "id" + $parameters: + path_extractor: "locations" + path: "/locations" + + products_stream: + $ref: "#/definitions/base_stream" + name: "products" + primary_key: "id" + $parameters: + path_extractor: "products" + path: "/products" + + vendors_stream: + $ref: "#/definitions/base_stream" + name: "vendors" + primary_key: "id" + $parameters: + path_extractor: "vendors" + path: "/vendors" + + assets_stream: + $ref: "#/definitions/base_stream" + name: "assets" + primary_key: "id" + $parameters: + path_extractor: "assets" + path: "/assets" + + software_stream: + $ref: "#/definitions/base_stream" + name: "software" + primary_key: "id" + $parameters: + path_extractor: "applications" + path: "/applications" + + purchase_orders_stream: + $ref: "#/definitions/base_stream" + name: "purchase_orders" + primary_key: "id" + $parameters: + path_extractor: "purchase_orders" + path: "/purchase_orders" + + +streams: + - "#/definitions/tickets_stream" + - "#/definitions/satisfaction_survey_responses_stream" + - "#/definitions/problems_stream" + - "#/definitions/changes_stream" + - "#/definitions/releases_stream" + - "#/definitions/requesters_stream" + - "#/definitions/agents_stream" + - "#/definitions/locations_stream" + - "#/definitions/products_stream" + - "#/definitions/vendors_stream" + - "#/definitions/assets_stream" + - "#/definitions/purchase_orders_stream" + - "#/definitions/software_stream" + +check: + type: CheckStream + stream_names: + - "tickets" diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/agents.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/agents.json index ed438d09de03..4d69983591cd 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/agents.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/agents.json @@ -139,7 +139,13 @@ "type": ["null", "array"] }, "workspace_info": { - "type": ["null", "string"] + "type": ["null","array"], + "items": { + "additionalProperties": true + } + }, + "work_schedule_id": { + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/assets.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/assets.json index 0f259a323c3f..8e39dcb9526e 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/assets.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/assets.json @@ -1,23 +1,66 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { - "id": { "type": "integer" }, - "display_id": { "type": ["null", "integer"] }, - "name": { "type": ["null", "string"] }, - "description": { "type": ["null", "string"] }, - "asset_type_id": { "type": ["null", "integer"] }, - "asset_tag": { "type": ["null", "string"] }, - "impact": { "type": ["null", "string"] }, - "author_type": { "type": ["null", "string"] }, - "usage_type": { "type": ["null", "string"] }, - "user_id": { "type": ["null", "integer"] }, - "location_id": { "type": ["null", "integer"] }, - "department_id": { "type": ["null", "integer"] }, - "agent_id": { "type": ["null", "integer"] }, - "group_id": { "type": ["null", "integer"] }, - "assigned_on": { "type": ["null", "string"] }, - "created_at": { "type": ["null", "string"] }, - "updated_at": { "type": ["null", "string"] } + "id": { + "type": "integer" + }, + "display_id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "asset_type_id": { + "type": ["null", "integer"] + }, + "asset_tag": { + "type": ["null", "string"] + }, + "impact": { + "type": ["null", "string"] + }, + "author_type": { + "type": ["null", "string"] + }, + "usage_type": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "location_id": { + "type": ["null", "integer"] + }, + "department_id": { + "type": ["null", "integer"] + }, + "agent_id": { + "type": ["null", "integer"] + }, + "group_id": { + "type": ["null", "integer"] + }, + "assigned_on": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "discovery_enabled": { + "type": ["null", "boolean"] + }, + "end_of_life": { + "type": ["null", "string"], + "format":"%Y-%m-%d" + } } } + diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/changes.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/changes.json index cb9e3aff1c6d..10e92ecacad0 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/changes.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/changes.json @@ -1,30 +1,111 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { - "id": { "type": "integer" }, - "agent_id": { "type": ["null", "integer"] }, - "description": { "type": ["null", "string"] }, - "description_text": { "type": ["null", "string"] }, - "requester_id": { "type": ["null", "integer"] }, - "group_id": { "type": ["null", "integer"] }, - "priority": { "type": ["null", "integer"] }, - "impact": { "type": ["null", "integer"] }, - "status": { "type": ["null", "integer"] }, - "risk": { "type": ["null", "integer"] }, - "change_type": { "type": ["null", "integer"] }, - "approval_status": { "type": ["null", "integer"] }, - "planned_start_date": { "type": ["null", "string"] }, - "planned_end_date": { "type": ["null", "string"] }, - "subject": { "type": ["null", "string"] }, - "department_id": { "type": ["null", "integer"] }, - "category": { "type": ["null", "string"] }, - "sub_category": { "type": ["null", "string"] }, - "item_category": { "type": ["null", "string"] }, - "custom_fields": { "type": ["null", "object"] }, - "maintenance_window": { "type": ["null", "object"] }, - "blackout_window": { "type": ["null", "object"] }, - "created_at": { "type": ["null", "string"] }, - "updated_at": { "type": ["null", "string"] } + "id": { + "type": "integer" + }, + "agent_id": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "description_text": { + "type": ["null", "string"] + }, + "requester_id": { + "type": ["null", "integer"] + }, + "group_id": { + "type": ["null", "integer"] + }, + "priority": { + "type": ["null", "integer"] + }, + "impact": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "integer"] + }, + "risk": { + "type": ["null", "integer"] + }, + "change_type": { + "type": ["null", "integer"] + }, + "approval_status": { + "type": ["null", "integer"] + }, + "planned_start_date": { + "type": ["null", "string"] + }, + "planned_end_date": { + "type": ["null", "string"] + }, + "subject": { + "type": ["null", "string"] + }, + "department_id": { + "type": ["null", "integer"] + }, + "category": { + "type": ["null", "string"] + }, + "sub_category": { + "type": ["null", "string"] + }, + "item_category": { + "type": ["null", "string"] + }, + "custom_fields": { + "type": ["null", "object"], + "additionalProperties": true + }, + "maintenance_window": { + "type": ["null", "object"], + "additionalProperties": true + }, + "blackout_window": { + "type": ["null", "object"], + "additionalProperties": true + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "planned_effort": { + "type": ["string", "null"] + }, + "attachments": { + "type": ["null","array"], + "items": { + "additionalProperties": true + } + }, + "impacted_services": { + "type": ["null","array"], + "items": { + "additionalProperties": true + } + }, + "workspace_id": { + "type": ["null","integer"] + }, + "change_window_id": { + "type": ["null","integer"] + }, + "assets": { + "type": ["null","array"], + "items": { + "additionalProperties": true + } + } } } diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/locations.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/locations.json index 154048ff14e6..748b273350af 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/locations.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/locations.json @@ -3,21 +3,56 @@ "type": "object", "additionalProperties": true, "properties": { - "id": { "type": "integer" }, - "name": { "type": ["null", "string"] }, - "parent_location_id": { "type": ["null", "integer"] }, - "primary_contact_id": { "type": ["null", "integer"] }, - "line1": { "type": ["null", "string"] }, - "line2": { "type": ["null", "string"] }, - "city": { "type": ["null", "string"] }, - "state": { "type": ["null", "string"] }, - "country": { "type": ["null", "string"] }, - "zipcode": { "type": ["null", "string"] }, - "created_at": { "type": ["null", "string"] }, - "updated_at": { "type": ["null", "string"] }, - "email": {"type": ["null", "string"] }, - "phone": {"type": ["null", "string"] }, - "address": {"type": ["null", "string"] }, - "contact_name": {"type": ["null", "string"] } + "id": { + "type": "integer" + }, + "name": { + "type": ["null", "string"] + }, + "parent_location_id": { + "type": ["null", "integer"] + }, + "primary_contact_id": { + "type": ["null", "integer"] + }, + "address": { + "type": "object", + "additionalProperties": true, + "properties": { + "line1": { + "type": ["null", "string"] + }, + "line2": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "zipcode": { + "type": ["null", "string"] + } + } + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "contact_name": { + "type": ["null", "string"] + } } } diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/problems.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/problems.json index c6bcd36e8291..dd3221a21173 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/problems.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/problems.json @@ -1,27 +1,100 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { - "id": { "type": "integer" }, - "agent_id": { "type": ["null", "integer"] }, - "requester_id": { "type": ["null", "integer"] }, - "group_id": { "type": ["null", "integer"] }, - "description": { "type": ["null", "string"] }, - "description_text": { "type": ["null", "string"] }, - "priority": { "type": ["null", "integer"] }, - "status": { "type": ["null", "integer"] }, - "impact": { "type": ["null", "integer"] }, - "known_error": { "type": ["null", "boolean"] }, - "subject": { "type": ["null", "string"] }, - "due_by": { "type": ["null", "string"] }, - "department_id": { "type": ["null", "integer"] }, - "category": { "type": ["null", "string"] }, - "sub_category": { "type": ["null", "string"] }, - "item_category": { "type": ["null", "string"] }, - "created_at": { "type": ["null", "string"] }, - "updated_at": { "type": ["null", "string"] }, - "associated_change": { "type": ["null", "integer"] }, - "custom_fields": { "type": ["null", "object"] }, - "analysis_fields": { "type": "object" } + "id": { + "type": "integer" + }, + "agent_id": { + "type": ["null", "integer"] + }, + "requester_id": { + "type": ["null", "integer"] + }, + "group_id": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "description_text": { + "type": ["null", "string"] + }, + "priority": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "integer"] + }, + "impact": { + "type": ["null", "integer"] + }, + "known_error": { + "type": ["null", "boolean"] + }, + "subject": { + "type": ["null", "string"] + }, + "due_by": { + "type": ["null", "string"] + }, + "department_id": { + "type": ["null", "integer"] + }, + "category": { + "type": ["null", "string"] + }, + "sub_category": { + "type": ["null", "string"] + }, + "item_category": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "associated_change": { + "type": ["null", "integer"] + }, + "custom_fields": { + "type": ["null", "object"], + "additionalProperties": true + }, + "analysis_fields": { + "type": "object", + "additionalProperties": true + }, + "planned_start_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "planned_end_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "planned_effort": { + "type": ["string", "null"] + }, + "attachments": { + "type": ["null","array"], + "items": { + "additionalProperties": true + } + }, + "workspace_id": { + "type": ["null","integer"] + }, + "assets": { + "type": ["null","array"], + "items": { + "additionalProperties": true + } + } } } diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/purchase_orders.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/purchase_orders.json index 3de1cb4eccb1..fd80515db95b 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/purchase_orders.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/purchase_orders.json @@ -1,27 +1,76 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { - "id": { "type": "integer" }, - "vendor_id": { "type": ["null", "integer"] }, - "name": { "type": ["null", "string"] }, - "po_number": { "type": ["null", "string"] }, - "vendor_details": { "type": ["null", "string"] }, - "expected_delivery_date": { "type": ["null", "string"] }, - "created_at": { "type": ["null", "string"] }, - "updated_at": { "type": ["null", "string"] }, - "created_by": { "type": ["null", "string"] }, - "status": { "type": ["null", "integer"] }, - "shipping_address": { "type": ["null", "string"] }, - "billing_same_as_shipping": { "type": ["null", "integer"] }, - "billing_address": { "type": ["null", "string"] }, - "currency_code": { "type": ["null", "string"] }, - "conversion_rate": { "type": "number" }, - "department_id": { "type": ["null", "integer"] }, - "discount_percentage": { "type": ["null", "integer"] }, - "tax_percentage": { "type": ["null", "integer"] }, - "shipping_cost": { "type": ["null", "integer"] }, - "custom_fields": { "type": ["null", "object"] }, - "purchase_items": { "type": "array" } + "id": { + "type": "integer" + }, + "vendor_id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "po_number": { + "type": ["null", "string"] + }, + "vendor_details": { + "type": ["null", "string"] + }, + "expected_delivery_date": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "created_by": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "integer"] + }, + "shipping_address": { + "type": ["null", "string"] + }, + "billing_same_as_shipping": { + "type": ["null", "integer"] + }, + "billing_address": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + }, + "conversion_rate": { + "type": "number" + }, + "department_id": { + "type": ["null", "integer"] + }, + "discount_percentage": { + "type": ["null", "integer"] + }, + "tax_percentage": { + "type": ["null", "integer"] + }, + "shipping_cost": { + "type": ["null", "integer"] + }, + "workspace_id": { + "type": ["null", "integer"] + }, + "total_cost": { + "type": ["null", "number"] + }, + "custom_fields": { + "type": ["null", "object"] + }, + "purchase_items": { + "type": "array" + } } } diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/requesters.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/requesters.json index e77460367419..73cc035f2a95 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/requesters.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/requesters.json @@ -3,37 +3,92 @@ "type": "object", "additionalProperties": true, "properties": { - "id": { "type": "integer" }, - "first_name": { "type": ["null", "string"] }, - "last_name": { "type": ["null", "string"] }, - "job_title": { "type": ["null", "string"] }, - "primary_email": { "type": ["null", "string"] }, - "secondary_emails": { "type": ["null", "array"] }, - "work_phone_number": { "type": ["null", "string"] }, - "mobile_phone_number": { "type": ["null", "string"] }, - "department_ids": { "type": ["null", "array"] }, + "id": { + "type": "integer" + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "job_title": { + "type": ["null", "string"] + }, + "primary_email": { + "type": ["null", "string"] + }, + "secondary_emails": { + "type": ["null", "array"] + }, + "work_phone_number": { + "type": ["null", "string"] + }, + "mobile_phone_number": { + "type": ["null", "string"] + }, + "department_ids": { + "type": ["null", "array"] + }, "can_see_all_tickets_from_associated_departments": { "type": ["null", "boolean"] }, - "reporting_manager_id": { "type": ["null", "integer"] }, - "address": { "type": ["null", "string"] }, - "time_zone": { "type": ["null", "string"] }, - "time_format": { "type": ["null", "string"] }, - "language": { "type": ["null", "string"] }, - "location_id": { "type": ["null", "integer"] }, - "background_information": { "type": ["null", "string"] }, - "custom_fields": { "type": ["null", "object"] }, - "active": { "type": ["null", "boolean"] }, - "has_logged_in": { "type": ["null", "boolean"] }, - "created_at": { "type": ["null", "string"] }, - "updated_at": { "type": ["null", "string"] }, - "is_agent": { "type": ["null", "boolean"] }, - "department_names": { "type": ["null", "array"] }, - "vip_user": { "type": ["null", "boolean"] }, - "external_id": { "type": ["null", "string"] }, + "reporting_manager_id": { + "type": ["null", "integer"] + }, + "address": { + "type": ["null", "string"] + }, + "time_zone": { + "type": ["null", "string"] + }, + "time_format": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + }, + "location_id": { + "type": ["null", "integer"] + }, + "background_information": { + "type": ["null", "string"] + }, + "custom_fields": { + "type": ["null", "object"] + }, + "active": { + "type": ["null", "boolean"] + }, + "has_logged_in": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "is_agent": { + "type": ["null", "boolean"] + }, + "department_names": { + "type": ["null", "array"] + }, + "vip_user": { + "type": ["null", "boolean"] + }, + "external_id": { + "type": ["null", "string"] + }, "can_see_all_changes_from_associated_departments": { "type": ["null", "boolean"] }, - "location_name": { "type": ["null", "string"] } + "location_name": { + "type": ["null", "string"] + }, + "work_schedule_id": { + "type": ["null", "integer"] + } } } diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/software.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/software.json index 6652c10f11a9..2c83436b7bd9 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/software.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/software.json @@ -1,20 +1,58 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { - "id": { "type": "integer" }, - "name": { "type": ["null", "string"] }, - "description": { "type": ["null", "string"] }, - "application_type": { "type": ["null", "string"] }, - "status": { "type": ["null", "string"] }, - "publisher_id": { "type": ["null", "integer"] }, - "managed_by_id": { "type": ["null", "integer"] }, - "notes": { "type": ["null", "string"] }, - "category": { "type": ["null", "string"] }, - "sources": { "type": ["null", "string"] }, - "user_count": { "type": ["null", "integer"] }, - "installation_count": { "type": ["null", "integer"] }, - "created_at": { "type": ["null", "string"] }, - "updated_at": { "type": ["null", "string"] } + "id": { + "type": "integer" + }, + "name": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "application_type": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "publisher_id": { + "type": ["null", "integer"] + }, + "managed_by_id": { + "type": ["null", "integer"] + }, + "notes": { + "type": ["null", "string"] + }, + "category": { + "type": ["null", "string"] + }, + "sources": { + "type": ["null", "array"], + "items": { + "additionalProperties": true + } + }, + "user_count": { + "type": ["null", "integer"] + }, + "installation_count": { + "type": ["null", "integer"] + }, + "workspace_id": { + "type": ["null", "integer"] + }, + "additional_data": { + "type": ["null", "object"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + } } } diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/vendors.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/vendors.json index f68696260f7e..94984c8d005d 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/vendors.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/vendors.json @@ -1,17 +1,61 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { - "id": { "type": "integer" }, - "name": { "type": ["null", "string"] }, - "description": { "type": ["null", "string"] }, - "primary_contact_id": { "type": ["null", "integer"] }, - "line1": { "type": ["null", "string"] }, - "city": { "type": ["null", "string"] }, - "state": { "type": ["null", "string"] }, - "country": { "type": ["null", "string"] }, - "zipcode": { "type": ["null", "string"] }, - "created_at": { "type": ["null", "string"] }, - "updated_at": { "type": ["null", "string"] } + "id": { + "type": "integer" + }, + "name": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "contact_name": { + "type": ["null", "string"] + }, + "mobile": { + "type": ["null", "integer"] + }, + "phone": { + "type": ["null", "integer"] + }, + "primary_contact_id": { + "type": ["null", "integer"] + }, + "email": { + "type": ["null", "string"] + }, + "custom_fields": { + "type": ["null", "object"] + }, + "address": { + "type": "object", + "additionalProperties": true, + "properties": { + "line1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "zipcode": { + "type": ["null", "string"] + } + } + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + } } } diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/source.py b/airbyte-integrations/connectors/source-freshservice/source_freshservice/source.py index ec33e8cce3c6..d4cdeed0e652 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/source.py +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/source.py @@ -2,73 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import base64 -from typing import Any, List, Mapping, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -from .streams import ( - Agents, - Assets, - Changes, - Locations, - Problems, - Products, - PurchaseOrders, - Releases, - Requesters, - SatisfactionSurveyResponses, - Software, - Tickets, - Vendors, -) +WARNING: Do not modify this file. +""" -# Source -class HttpBasicAuthenticator(TokenAuthenticator): - def __init__(self, auth: Tuple[str, str], auth_method: str = "Basic", **kwargs): - auth_string = f"{auth[0]}:{auth[1]}".encode("utf8") - b64_encoded = base64.b64encode(auth_string).decode("utf8") - super().__init__(token=b64_encoded, auth_method=auth_method, **kwargs) - - -class SourceFreshservice(AbstractSource): - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: - kwargs = { - "authenticator": HttpBasicAuthenticator((config["api_key"], "")), - "start_date": config["start_date"], - "domain_name": config["domain_name"], - } - try: - tickets = Tickets(**kwargs).read_records(sync_mode=SyncMode.full_refresh) - next(tickets) - return True, None - except requests.exceptions.RequestException as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - kwargs = { - "authenticator": HttpBasicAuthenticator((config["api_key"], "")), - "start_date": config["start_date"], - "domain_name": config["domain_name"], - } - return [ - Tickets(**kwargs), - Problems(**kwargs), - Changes(**kwargs), - Releases(**kwargs), - Requesters(**kwargs), - Agents(**kwargs), - Locations(**kwargs), - Products(**kwargs), - Vendors(**kwargs), - Assets(**kwargs), - PurchaseOrders(**kwargs), - SatisfactionSurveyResponses(**kwargs), - Software(**kwargs), - ] +# Declarative Source +class SourceFreshservice(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/spec.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/spec.json deleted file mode 100644 index 1ff611c10e85..000000000000 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/spec.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/freshservice", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Freshservice Spec", - "type": "object", - "required": ["domain_name", "api_key", "start_date"], - "additionalProperties": true, - "properties": { - "domain_name": { - "type": "string", - "title": "Domain Name", - "description": "The name of your Freshservice domain", - "examples": ["mydomain.freshservice.com"] - }, - "api_key": { - "title": "API Key", - "type": "string", - "description": "Freshservice API Key. See here. The key is case sensitive.", - "airbyte_secret": true - }, - "start_date": { - "title": "Start Date", - "type": "string", - "description": "UTC date and time in the format 2020-10-01T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2020-10-01T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - } - } - } -} diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/spec.yaml b/airbyte-integrations/connectors/source-freshservice/source_freshservice/spec.yaml new file mode 100644 index 000000000000..50820e012689 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/spec.yaml @@ -0,0 +1,31 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/freshservice +connectionSpecification: + "$schema": http://json-schema.org/draft-07/schema# + title: Freshservice Spec + type: object + required: + - domain_name + - api_key + - start_date + additionalProperties: true + properties: + domain_name: + type: string + title: Domain Name + description: The name of your Freshservice domain + examples: + - mydomain.freshservice.com + api_key: + title: API Key + type: string + description: Freshservice API Key. See here. + The key is case sensitive. + airbyte_secret: true + start_date: + title: Start Date + type: string + description: UTC date and time in the format 2020-10-01T00:00:00Z. Any data + before this date will not be replicated. + examples: + - '2020-10-01T00:00:00Z' + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/streams.py b/airbyte-integrations/connectors/source-freshservice/source_freshservice/streams.py deleted file mode 100644 index bc5daec9ac34..000000000000 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/streams.py +++ /dev/null @@ -1,197 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC -from http import HTTPStatus -from typing import Any, Iterable, Mapping, MutableMapping, Optional -from urllib.parse import parse_qsl, urlparse - -import requests -from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer - - -# Basic full refresh stream -class FreshserviceStream(HttpStream, ABC): - primary_key = "id" - order_field = "updated_at" - page_size = 30 - transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - - def __init__(self, start_date: str = None, domain_name: str = None, **kwargs): - super().__init__(**kwargs) - self._start_date = start_date - self.domain_name = domain_name - - @property - def url_base(self) -> str: - return f"https://{self.domain_name}/api/v2/" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - next_page = response.links.get("next") - if next_page: - return {"page": dict(parse_qsl(urlparse(next_page.get("url")).query)).get("page")} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = {"per_page": self.page_size} - - if next_page_token: - params.update(next_page_token) - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - if response.status_code == HTTPStatus.NO_CONTENT: - return - json_response = response.json() - records = json_response.get(self.object_name, []) if self.object_name is not None else json_response - yield from records - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return self.object_name - - -# Basic incremental stream -class IncrementalFreshserviceStream(FreshserviceStream, ABC): - state_checkpoint_interval = 60 - cursor_field = "updated_at" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} - - def request_params(self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs): - params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) - # If there is a next page token then we should only send pagination-related parameters. - if not next_page_token: - params["order_by"] = self.order_field - params["order_type"] = "asc" - if stream_state: - params["updated_since"] = stream_state.get(self.cursor_field) - return params - - -class Tickets(IncrementalFreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#view_all_ticket - """ - - object_name = "tickets" - - -class SatisfactionSurveyResponses(IncrementalFreshserviceStream, HttpSubStream): - object_name = "csat_response" - """ - API docs: https://api.freshservice.com/#ticket_csat_attributes - - self.authenticator (which should be used as the - authenticator for Users) is object of NoAuth() - so self._session.auth is used instead - """ - - def __init__(self, **kwargs): - super().__init__(parent=Tickets(**kwargs), **kwargs) - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - ticket_id = stream_slice["parent"]["id"] - return f"tickets/{ticket_id}/csat_response" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - if response.status_code == HTTPStatus.NO_CONTENT: - return - json_response = response.json() - record = json_response.get(self.object_name, {}) if self.object_name is not None else json_response - yield record - - -class Problems(IncrementalFreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#problems - """ - - object_name = "problems" - - -class Changes(IncrementalFreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#changes - """ - - object_name = "changes" - - -class Releases(IncrementalFreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#releases - """ - - object_name = "releases" - - -class Requesters(FreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#requesters - """ - - object_name = "requesters" - - -class Agents(FreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#agents - """ - - object_name = "agents" - - -class Locations(FreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#locations - """ - - object_name = "locations" - - -class Products(FreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#products - """ - - object_name = "products" - - -class Vendors(FreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#vendors - """ - - object_name = "vendors" - - -class Assets(FreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#assets - """ - - object_name = "assets" - - -class PurchaseOrders(FreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#purchase-order - """ - - object_name = "purchase_orders" - - -class Software(FreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#software - """ - - object_name = "applications" diff --git a/airbyte-integrations/connectors/source-freshservice/unit_tests/__init__.py b/airbyte-integrations/connectors/source-freshservice/unit_tests/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/airbyte-integrations/connectors/source-freshservice/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-freshservice/unit_tests/conftest.py b/airbyte-integrations/connectors/source-freshservice/unit_tests/conftest.py deleted file mode 100644 index 5bb422d2b7f4..000000000000 --- a/airbyte-integrations/connectors/source-freshservice/unit_tests/conftest.py +++ /dev/null @@ -1,14 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest - - -@pytest.fixture -def test_config(): - return { - "domain_name": "test.freshservice.com", - "api_key": "test_api_key", - "start_date": "2021-05-07T00:00:00Z", - } diff --git a/airbyte-integrations/connectors/source-freshservice/unit_tests/test_source.py b/airbyte-integrations/connectors/source-freshservice/unit_tests/test_source.py deleted file mode 100644 index f540ab794e69..000000000000 --- a/airbyte-integrations/connectors/source-freshservice/unit_tests/test_source.py +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest import mock -from unittest.mock import MagicMock - -import responses -from requests.exceptions import HTTPError -from source_freshservice.source import SourceFreshservice - - -def setup_responses(): - responses.add( - responses.GET, - "https://test.freshservice.com/api/v2/tickets", - json={"per_page": 30, "order_by": "updated_at", "order_type": "asc", "updated_since": "2021-05-07T00:00:00Z"}, - ) - - -@mock.patch("source_freshservice.streams.Tickets.read_records", return_value=iter([1])) -def test_check_connection_success(mocker): - source = SourceFreshservice() - logger_mock = MagicMock() - test_config = MagicMock() - assert source.check_connection(logger_mock, test_config) == (True, None) - - -def test_check_connection_failure(mocker, test_config): - source = SourceFreshservice() - logger_mock = MagicMock() - response = source.check_connection(logger_mock, test_config) - - assert response[0] is False - assert type(response[1]) == HTTPError - - -def test_stream_count(mocker): - source = SourceFreshservice() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 13 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-freshservice/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-freshservice/unit_tests/test_stream.py deleted file mode 100644 index 01f17698a14f..000000000000 --- a/airbyte-integrations/connectors/source-freshservice/unit_tests/test_stream.py +++ /dev/null @@ -1,57 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest -import requests -from source_freshservice.streams import FreshserviceStream, Tickets - - -@pytest.fixture -def patch_base_class(mocker): - mocker.patch.object(FreshserviceStream, "path", "v0/example_endpoint") - # mocker.patch.object(FreshserviceStream, "domain_name", "https://example.example") - - -def test_request_params(patch_base_class): - stream = FreshserviceStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {"per_page": 30} - assert stream.request_params(**inputs) == expected_params - - -class Fake(object): - pass - - -def test_next_page_token(patch_base_class): - stream = FreshserviceStream() - response = Fake() - response.links = {'next': {'url': 'https://dummy?page=3'}} - inputs = {'response': response} - expected_token = {'page': '3'} - assert stream.next_page_token(**inputs) == expected_token - - -def test_next_page_token_when_no_next_page(patch_base_class): - stream = FreshserviceStream() - response = Fake() - response.links = {} - inputs = {'response': response} - assert stream.next_page_token(**inputs) is None - - -def test_parse_response(patch_base_class, requests_mock): - stream = Tickets() - requests_mock.get('https://dummy', json={ - "tickets": [ - { - "subject": "test", - "group_id": 18000074057 - } - ] - }) - res = requests.get('https://dummy') - inputs = {'response': res} - expected_parsed = {"subject": "test", "group_id": 18000074057} - assert next(stream.parse_response(**inputs)) == expected_parsed From 0593ebf4099328f4f1dc4a5f616843536d3a13b3 Mon Sep 17 00:00:00 2001 From: Aviraj Gour Date: Sun, 6 Aug 2023 16:46:09 +0530 Subject: [PATCH 2/6] doc changed --- docs/integrations/sources/freshservice.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/sources/freshservice.md b/docs/integrations/sources/freshservice.md index 7dbf88854194..a339768b0eaa 100644 --- a/docs/integrations/sources/freshservice.md +++ b/docs/integrations/sources/freshservice.md @@ -54,6 +54,7 @@ Please read [How to find your API key](https://api.freshservice.com/#authenticat | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 2.0.0 | 2023-08-06 | [29126](https://github.com/airbytehq/airbyte/pull/29126) | Migrated to Low-Code CDK | | 1.1.0 | 2023-05-09 | [25929](https://github.com/airbytehq/airbyte/pull/25929) | Add stream for customer satisfaction survey responses endpoint | | 1.0.0 | 2023-05-02 | [25743](https://github.com/airbytehq/airbyte/pull/25743) | Correct data types in tickets, agents and requesters schemas to match Freshservice API | | 0.1.1 | 2021-12-28 | [9143](https://github.com/airbytehq/airbyte/pull/9143) | Update titles and descriptions | From 3c61a86ff34510c3f31db846cdca3eb32ec4bc5e Mon Sep 17 00:00:00 2001 From: Aviraj Gour Date: Thu, 10 Aug 2023 23:39:05 +0530 Subject: [PATCH 3/6] suggested changes --- .../connectors/source-freshservice/Dockerfile | 2 +- .../connectors/source-freshservice/metadata.yaml | 4 ++-- docs/integrations/sources/freshservice.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-freshservice/Dockerfile b/airbyte-integrations/connectors/source-freshservice/Dockerfile index 965cbf8d720a..7732b3d3d243 100644 --- a/airbyte-integrations/connectors/source-freshservice/Dockerfile +++ b/airbyte-integrations/connectors/source-freshservice/Dockerfile @@ -34,5 +34,5 @@ COPY source_freshservice ./source_freshservice ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=2.0.0 +LABEL io.airbyte.version=1.2.0 LABEL io.airbyte.name=airbyte/source-freshservice diff --git a/airbyte-integrations/connectors/source-freshservice/metadata.yaml b/airbyte-integrations/connectors/source-freshservice/metadata.yaml index 6af4b29ccb30..61c73b4c6b10 100644 --- a/airbyte-integrations/connectors/source-freshservice/metadata.yaml +++ b/airbyte-integrations/connectors/source-freshservice/metadata.yaml @@ -10,13 +10,13 @@ data: connectorSubtype: api connectorType: source definitionId: 9bb85338-ea95-4c93-b267-6be89125b267 - dockerImageTag: 2.0.0 + dockerImageTag: 1.2.0 dockerRepository: airbyte/source-freshservice githubIssueLabel: source-freshservice icon: freshservice.svg license: MIT name: Freshservice - releaseDate: TODO + releaseDate: "2021-10-29" releaseStage: alpha supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/freshservice diff --git a/docs/integrations/sources/freshservice.md b/docs/integrations/sources/freshservice.md index a339768b0eaa..6481eb888c3f 100644 --- a/docs/integrations/sources/freshservice.md +++ b/docs/integrations/sources/freshservice.md @@ -54,7 +54,7 @@ Please read [How to find your API key](https://api.freshservice.com/#authenticat | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 2.0.0 | 2023-08-06 | [29126](https://github.com/airbytehq/airbyte/pull/29126) | Migrated to Low-Code CDK | +| 1.2.0 | 2023-08-06 | [29126](https://github.com/airbytehq/airbyte/pull/29126) | Migrated to Low-Code CDK | | 1.1.0 | 2023-05-09 | [25929](https://github.com/airbytehq/airbyte/pull/25929) | Add stream for customer satisfaction survey responses endpoint | | 1.0.0 | 2023-05-02 | [25743](https://github.com/airbytehq/airbyte/pull/25743) | Correct data types in tickets, agents and requesters schemas to match Freshservice API | | 0.1.1 | 2021-12-28 | [9143](https://github.com/airbytehq/airbyte/pull/9143) | Update titles and descriptions | From e882fa54954965f372a84e9c58229c3999e16a62 Mon Sep 17 00:00:00 2001 From: Aviraj Gour <100823015+avirajsingh7@users.noreply.github.com> Date: Wed, 16 Aug 2023 09:56:23 +0530 Subject: [PATCH 4/6] empty stream Co-authored-by: Mal Hancock --- .../acceptance-test-config.yml | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml b/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml index 339dbaaf6d69..9e3b75aa0788 100644 --- a/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml @@ -20,7 +20,25 @@ acceptance_tests: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["name":"satisfaction_survey_responses","name":"releases"] + empty_streams: + - name: satisfaction_survey_responses + bypass_reason: Test account does not have permissions + - name: releases + bypass_reason: Test account does not have permissions + - name: assets + bypass_reason: Test account does not have permissions + - name: problems + bypass_reason: Test account does not have permissions + - name: software + bypass_reason: Test account does not have permissions + - name: products + bypass_reason: Test account does not have permissions + - name: changes + bypass_reason: Test account does not have permissions + - name: vendors + bypass_reason: Test account does not have permissions + - name: purchase_orders + bypass_reason: Test account does not have permissions incremental: tests: - config_path: "secrets/config.json" From 18d2d39c47005ca6ec7100dd0ff56b10836db27b Mon Sep 17 00:00:00 2001 From: Aviraj Gour <100823015+avirajsingh7@users.noreply.github.com> Date: Wed, 16 Aug 2023 09:56:39 +0530 Subject: [PATCH 5/6] indentation Co-authored-by: Mal Hancock --- .../source-freshservice/acceptance-test-config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml b/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml index 9e3b75aa0788..5f2bc08d045b 100644 --- a/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml @@ -40,11 +40,11 @@ acceptance_tests: - name: purchase_orders bypass_reason: Test account does not have permissions incremental: - tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" From c5a3419a87760c0944ddc7bd57967956ea499b94 Mon Sep 17 00:00:00 2001 From: Aviraj Gour <100823015+avirajsingh7@users.noreply.github.com> Date: Wed, 16 Aug 2023 09:57:22 +0530 Subject: [PATCH 6/6] error handling Co-authored-by: Mal Hancock --- .../source-freshservice/source_freshservice/manifest.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/manifest.yaml b/airbyte-integrations/connectors/source-freshservice/source_freshservice/manifest.yaml index c5f74cc42ca5..c076f73789df 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/manifest.yaml +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/manifest.yaml @@ -14,6 +14,10 @@ definitions: type: "BasicHttpAuthenticator" username: "{{config['api_key']}}" password: "" + error_handler: + response_filters: + - http_codes: [403] + action: IGNORE retriever: type: SimpleRetriever record_selector: