From 2d4e596da952b0c06c3b8fddcec7dafd9f93c1e0 Mon Sep 17 00:00:00 2001 From: Elliot T Date: Sat, 15 Apr 2023 16:34:35 +0200 Subject: [PATCH 01/11] initiate connector --- .../source-captain-data/.dockerignore | 6 + .../connectors/source-captain-data/Dockerfile | 38 ++++ .../connectors/source-captain-data/README.md | 82 +++++++++ .../source-captain-data/__init__.py | 3 + .../acceptance-test-config.yml | 27 +++ .../acceptance-test-docker.sh | 3 + .../source-captain-data/build.gradle | 9 + .../integration_tests/__init__.py | 3 + .../integration_tests/acceptance.py | 14 ++ .../integration_tests/configured_catalog.json | 40 +++++ .../integration_tests/invalid_config.json | 4 + .../integration_tests/sample_config.json | 4 + .../connectors/source-captain-data/main.py | 13 ++ .../source-captain-data/requirements.txt | 2 + .../connectors/source-captain-data/setup.py | 29 +++ .../source_captain_data/__init__.py | 8 + .../source_captain_data/manifest.yaml | 168 ++++++++++++++++++ .../schemas/job_results.json | 19 ++ .../source_captain_data/schemas/jobs.json | 54 ++++++ .../schemas/workflows.json | 24 +++ .../schemas/workspace.json | 33 ++++ .../source_captain_data/source.py | 18 ++ 22 files changed, 601 insertions(+) create mode 100644 airbyte-integrations/connectors/source-captain-data/.dockerignore create mode 100644 airbyte-integrations/connectors/source-captain-data/Dockerfile create mode 100644 airbyte-integrations/connectors/source-captain-data/README.md create mode 100644 airbyte-integrations/connectors/source-captain-data/__init__.py create mode 100644 airbyte-integrations/connectors/source-captain-data/acceptance-test-config.yml create mode 100755 airbyte-integrations/connectors/source-captain-data/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-captain-data/build.gradle create mode 100644 airbyte-integrations/connectors/source-captain-data/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-captain-data/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-captain-data/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-captain-data/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-captain-data/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-captain-data/main.py create mode 100644 airbyte-integrations/connectors/source-captain-data/requirements.txt create mode 100644 airbyte-integrations/connectors/source-captain-data/setup.py create mode 100644 airbyte-integrations/connectors/source-captain-data/source_captain_data/__init__.py create mode 100644 airbyte-integrations/connectors/source-captain-data/source_captain_data/manifest.yaml create mode 100644 airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/job_results.json create mode 100644 airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/jobs.json create mode 100644 airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workflows.json create mode 100644 airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workspace.json create mode 100644 airbyte-integrations/connectors/source-captain-data/source_captain_data/source.py diff --git a/airbyte-integrations/connectors/source-captain-data/.dockerignore b/airbyte-integrations/connectors/source-captain-data/.dockerignore new file mode 100644 index 000000000000..f7cb8b7c520a --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_captain_data +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-captain-data/Dockerfile b/airbyte-integrations/connectors/source-captain-data/Dockerfile new file mode 100644 index 000000000000..3b3a4c8883ab --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_captain_data ./source_captain_data + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-captain-data diff --git a/airbyte-integrations/connectors/source-captain-data/README.md b/airbyte-integrations/connectors/source-captain-data/README.md new file mode 100644 index 000000000000..0d93d706ca0c --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/README.md @@ -0,0 +1,82 @@ +# Captain Data Source + +This is the repository for the Captain Data configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/captain-data). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-captain-data:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/captain-data) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_captain_data/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source captain-data test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-captain-data:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-captain-data:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-captain-data:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-captain-data:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-captain-data:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-captain-data:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with Docker, run: +``` +./acceptance-test-docker.sh +``` + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-captain-data:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-captain-data:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-captain-data/__init__.py b/airbyte-integrations/connectors/source-captain-data/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-captain-data/acceptance-test-config.yml b/airbyte-integrations/connectors/source-captain-data/acceptance-test-config.yml new file mode 100644 index 000000000000..8d8e51e6be24 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/acceptance-test-config.yml @@ -0,0 +1,27 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-captain-data:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_captain_data/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + incremental: + bypass_reason: "This connector does not implement incremental sync" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-captain-data/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-captain-data/acceptance-test-docker.sh new file mode 100755 index 000000000000..b6d65deeccb4 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/acceptance-test-docker.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-captain-data/build.gradle b/airbyte-integrations/connectors/source-captain-data/build.gradle new file mode 100644 index 000000000000..d92bf8ed87a3 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-connector-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_captain_data' +} diff --git a/airbyte-integrations/connectors/source-captain-data/integration_tests/__init__.py b/airbyte-integrations/connectors/source-captain-data/integration_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-captain-data/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-captain-data/integration_tests/acceptance.py new file mode 100644 index 000000000000..82823254d266 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-captain-data/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-captain-data/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..6813b6b503a3 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/integration_tests/configured_catalog.json @@ -0,0 +1,40 @@ +{ + "streams": [ + { + "stream": { + "name": "workspace", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "workflows", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "jobs", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "job_results", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-captain-data/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-captain-data/integration_tests/invalid_config.json new file mode 100644 index 000000000000..a8a304c99f51 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "bad-api-key", + "project_uid": "bad-project-uid" +} diff --git a/airbyte-integrations/connectors/source-captain-data/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-captain-data/integration_tests/sample_config.json new file mode 100644 index 000000000000..daca574b8635 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/integration_tests/sample_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "api-key", + "project_uid": "project-uid" +} diff --git a/airbyte-integrations/connectors/source-captain-data/main.py b/airbyte-integrations/connectors/source-captain-data/main.py new file mode 100644 index 000000000000..765d967fad15 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_captain_data import SourceCaptainData + +if __name__ == "__main__": + source = SourceCaptainData() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-captain-data/requirements.txt b/airbyte-integrations/connectors/source-captain-data/requirements.txt new file mode 100644 index 000000000000..cc57334ef619 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/connector-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-captain-data/setup.py b/airbyte-integrations/connectors/source-captain-data/setup.py new file mode 100644 index 000000000000..e74e6b5d45fc --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.2", + "pytest-mock~=3.6.1", + "connector-acceptance-test", +] + +setup( + name="source_captain_data", + description="Source implementation for Captain Data.", + author="Elliot Trabac", + author_email="elliot.trabac1@gmail.com", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/__init__.py b/airbyte-integrations/connectors/source-captain-data/source_captain_data/__init__.py new file mode 100644 index 000000000000..bdb8c926c0a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceCaptainData + +__all__ = ["SourceCaptainData"] diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/manifest.yaml b/airbyte-integrations/connectors/source-captain-data/source_captain_data/manifest.yaml new file mode 100644 index 000000000000..fe57c5289597 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/manifest.yaml @@ -0,0 +1,168 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + requester: + type: HttpRequester + url_base: "https://api.captaindata.co/v3/" + http_method: "GET" + authenticator: + type: ApiKeyAuthenticator + header: "x-api-key" + api_token: "{{ config['api_key'] }}" + request_headers: + x-project-id: "{{ config['project_uid'] }}" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + + captain_data_paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response['paging']['next']}}" + stop_condition: "{{ response['paging']['have_next_page'] is false }}" + page_token_option: + type: "RequestPath" + + # Base stream + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + # Streams + workspace_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "workspace" + primary_key: "name" + path: "/workspace" + transformations: + - type: AddFields + fields: + - path: ["project_uid"] + value: "{{ config['project_uid'] }}" + + workflows_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "workflows" + primary_key: "uid" + path: "/workflows" + + # Sliced streams + jobs_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "jobs" + primary_key: "uid" + path: "/workflows/{{ stream_slice.parent_id }}/jobs" + retriever: + requester: + $ref: "#/definitions/requester" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/workflows_stream" + parent_key: "uid" + partition_field: "parent_id" + record_selector: + $ref: "#/definitions/selector" + + successful_jobs_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "jobs" + primary_key: "uid" + path: "/workflows/{{ stream_slice.parent_id }}/jobs" + retriever: + requester: + $ref: "#/definitions/requester" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/workflows_stream" + parent_key: "uid" + partition_field: "parent_id" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + record_filter: + condition: "{{ record['total_row_count'] > 0 }}" + + job_results_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "job_results" + primary_key: "job_result_id" + path: "/jobs/{{ stream_slice.parent_id }}/results" + stream_cursor_field: "extracted_at" + retriever: + requester: + $ref: "#/definitions/requester" + paginator: + $ref: "#/definitions/captain_data_paginator" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/successful_jobs_stream" + parent_key: "uid" + partition_field: "parent_id" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["results"] + transformations: + - type: AddFields + fields: + - path: ["results"] + value: "{{ record }}" + - path: ["job_result_id"] + value: "{{ record|hash('md5') }}" + - path: ["job_uid"] + value: "{{ stream_slice.parent_id }}" + - path: ["extracted_at"] + value: "{{ now_utc() }}" + +streams: + - "#/definitions/workspace_stream" + - "#/definitions/workflows_stream" + - "#/definitions/jobs_stream" + - "#/definitions/job_results_stream" + +check: + type: CheckStream + stream_names: + - "workspace" + +spec: + type: Spec + documentationUrl: https://docs.captaindata.co/#1da39f5f-a26f-4a53-9e84-3e4b28077eae + connection_specification: + title: Captain Data Spec + type: object + required: + - api_key + - project_uid + additionalProperties: true + properties: + api_key: + type: string + description: Your Captain Data project API key. + airbyte_secret: true + project_uid: + type: string + description: Your Captain Data project uuid. diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/job_results.json b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/job_results.json new file mode 100644 index 000000000000..d975b10f74c5 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/job_results.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "job_uid": { + "type": ["null", "string"] + }, + "job_result_id": { + "type": ["null", "string"] + }, + "results": { + "type": ["null", "object"] + }, + "extracted_at": { + "type": ["null", "string"], + "format": "datetime" + } + } +} diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/jobs.json b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/jobs.json new file mode 100644 index 000000000000..91c403d850a6 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/jobs.json @@ -0,0 +1,54 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "accounts": { + "type": ["null", "array"] + }, + "configuration": { + "type": ["null", "array"] + }, + "error_message": { + "type": ["null", "string"] + }, + "finish_time": { + "type": ["null", "string"] + }, + "main_job_uid": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "number_inputs": { + "type": ["null", "integer"] + }, + "row_count": { + "type": ["null", "integer"] + }, + "scheduled_time": { + "type": ["null", "string"] + }, + "start_time": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "step_uid": { + "type": ["null", "string"] + }, + "total_row_count": { + "type": ["null", "integer"] + }, + "uid": { + "type": ["null", "string"] + }, + "workflow_name": { + "type": ["null", "string"] + }, + "workflow_uid": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workflows.json b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workflows.json new file mode 100644 index 000000000000..45b361fab9fe --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workflows.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "created_at": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "project_id": { + "type": ["null", "integer"] + }, + "steps": { + "type": ["null", "a"] + }, + "template_uid": { + "type": ["null", "string"] + }, + "uid": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workspace.json b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workspace.json new file mode 100644 index 000000000000..420960237e91 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workspace.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "project_uid": { + "type": ["null", "string"] + }, + "current_month_end": { + "type": ["null", "string"] + }, + "current_month_start": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "plan_name": { + "type": ["null", "string"] + }, + "tasks_left": { + "type": ["null", "integer"] + }, + "tasks_max": { + "type": ["null", "integer"] + }, + "tasks_used": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/source.py b/airbyte-integrations/connectors/source-captain-data/source_captain_data/source.py new file mode 100644 index 000000000000..648e1fb181b8 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceCaptainData(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) From 96d0b2f60b2458c352f956e069aeae679c79aefd Mon Sep 17 00:00:00 2001 From: Elliot T Date: Sat, 15 Apr 2023 16:45:36 +0200 Subject: [PATCH 02/11] add doc --- docs/integrations/sources/captain-data.md | 65 +++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 docs/integrations/sources/captain-data.md diff --git a/docs/integrations/sources/captain-data.md b/docs/integrations/sources/captain-data.md new file mode 100644 index 000000000000..9113b2e219da --- /dev/null +++ b/docs/integrations/sources/captain-data.md @@ -0,0 +1,65 @@ +# Captain Data + +This page contains the setup guide and reference information for the [Captain Data](https://docs.captaindata.co/#intro) source connector. + +## Prerequisites + +Api key and project UID are mandate for this connector to work, It could be generated from the dashboard settings (ref - https://app.captaindata.co/settings). + +## Setup guide + +### Step 1: Set up Captain Data connection + +- Available params + - api_key: The api_key + - project_uid: The project UID + +## Step 2: Set up the Captain Data connector in Airbyte + +### For Airbyte Cloud: + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. +3. On the Set up the source page, enter the name for the Captain Data connector and select **Captain Data** from the Source type dropdown. +4. Enter your `api_key` and `project_uid`. +5. Click **Set up source**. + +### For Airbyte OSS: + +1. Navigate to the Airbyte Open Source dashboard. +2. Set the name for your source. +3. Enter your `api_key` and `project_uid`. +4. Click **Set up source**. + +## Supported sync modes + +The Captain Data source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + +| Feature | Supported? | +| :---------------------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental Sync | No | +| Replicate Incremental Deletes | No | +| SSL connection | Yes | +| Namespaces | No | + +## Supported Streams + +- workspace +- workflows +- jobs +- job_results + +## API method example + +GET https://api.captaindata.co/v3/ + +## Performance considerations + +Captain Data [API reference](https://docs.captaindata.co/#intro) has v3 at present. The connector as default uses v3. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------- | :------------- | +| 0.1.0 | 2023-04-15 | [Init](https://github.com/airbytehq/airbyte/pull/) | Initial commit | From 98980682d7fb7153dec643b2ab7978dd0ca860c5 Mon Sep 17 00:00:00 2001 From: Elliot T Date: Sat, 15 Apr 2023 17:29:42 +0200 Subject: [PATCH 03/11] fix schema --- .../integration_tests/configured_catalog.json | 9 --------- .../source_captain_data/schemas/jobs.json | 5 ++++- .../source_captain_data/schemas/workflows.json | 2 +- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/airbyte-integrations/connectors/source-captain-data/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-captain-data/integration_tests/configured_catalog.json index 6813b6b503a3..a7114c8d99bc 100644 --- a/airbyte-integrations/connectors/source-captain-data/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-captain-data/integration_tests/configured_catalog.json @@ -26,15 +26,6 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "append" - }, - { - "stream": { - "name": "job_results", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/jobs.json b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/jobs.json index 91c403d850a6..1e302bfab8da 100644 --- a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/jobs.json +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/jobs.json @@ -6,7 +6,7 @@ "type": ["null", "array"] }, "configuration": { - "type": ["null", "array"] + "type": ["null", "object"] }, "error_message": { "type": ["null", "string"] @@ -17,6 +17,9 @@ "main_job_uid": { "type": ["null", "string"] }, + "main_job_name": { + "type": ["null", "string"] + }, "name": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workflows.json b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workflows.json index 45b361fab9fe..9674b453b953 100644 --- a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workflows.json +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workflows.json @@ -12,7 +12,7 @@ "type": ["null", "integer"] }, "steps": { - "type": ["null", "a"] + "type": ["null", "array"] }, "template_uid": { "type": ["null", "string"] From 2b3307ead1396e9c98d9f7dd7f7dd8f64e4dac34 Mon Sep 17 00:00:00 2001 From: Elliot T Date: Tue, 18 Apr 2023 09:08:26 +0200 Subject: [PATCH 04/11] add docs --- .../src/main/resources/seed/oss_catalog.json | 41 +++++++++++++++++++ .../resources/seed/source_definitions.yaml | 8 ++++ .../src/main/resources/seed/source_specs.yaml | 24 +++++++++++ .../source_captain_data/manifest.yaml | 2 + 4 files changed, 75 insertions(+) diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json b/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json index 8e90dc954187..c50a98e9bf8d 100644 --- a/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json +++ b/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json @@ -1015,6 +1015,47 @@ } } ] } + }, { + "sourceDefinitionId": "", + "name": "Captain Data", + "dockerRepository": "airbyte/source-captain-data", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://docs.airbyte.com/integrations/sources/captain-data", + "icon": "captain-data.svg", + "sourceType": "api", + "spec": { + "documentationUrl": "https://docs.airbyte.com/integrations/sources/captain-data", + "connectionSpecification": { + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Captain Data Source Spec", + "type": "object", + "additionalProperties": true, + "properties": { + "api_key": { + "title": "API Key", + "type": "string", + "description": "Your Captain Data project API key.", + "airbyte_secret": true + }, + "api_token": { + "title": "Project UID", + "type": "string", + "description": "Your Captain Data project uuid.", + "airbyte_secret": true + } + } + }, + "supportsNormalization": false, + "supportsDBT": false, + "supported_destination_sync_modes": [ ] + }, + "tombstone": false, + "public": true, + "custom": false, + "releaseStage": "alpha", + "allowedHosts": { + "hosts": [ "api.captaindata.co" ] + } }, { "destinationDefinitionId": "707456df-6f4f-4ced-b5c6-03f73bcad1c5", "name": "Cassandra", diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/source_definitions.yaml b/airbyte-config-oss/init-oss/src/main/resources/seed/source_definitions.yaml index 3a4678497b8b..54a3d6725f25 100644 --- a/airbyte-config-oss/init-oss/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config-oss/init-oss/src/main/resources/seed/source_definitions.yaml @@ -248,6 +248,14 @@ icon: callrail.svg sourceType: api releaseStage: alpha +- name: Captain Data + sourceDefinitionId: + dockerRepository: airbyte/source-captain-data + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/captain-data + icon: captain-data.svg + sourceType: api + releaseStage: alpha - name: Cart.com sourceDefinitionId: bb1a6d31-6879-4819-a2bd-3eed299ea8e2 dockerRepository: airbyte/source-cart diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/source_specs.yaml b/airbyte-config-oss/init-oss/src/main/resources/seed/source_specs.yaml index 3dbf916807e2..52b43df2a141 100644 --- a/airbyte-config-oss/init-oss/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config-oss/init-oss/src/main/resources/seed/source_specs.yaml @@ -2218,6 +2218,30 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-captain-data:0.1.0" + spec: + documentationUrl: https://docs.captaindata.co/#1da39f5f-a26f-4a53-9e84-3e4b28077eae + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: Captain Data Spec + type: object + required: + - api_key + - project_uid + additionalProperties: true + properties: + api_key: + title: API Key + type: string + description: Your Captain Data project API key. + airbyte_secret: true + project_uid: + title: Project UID + type: string + description: Your Captain Data project uuid. + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-cart:0.2.1" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/cart" diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/manifest.yaml b/airbyte-integrations/connectors/source-captain-data/source_captain_data/manifest.yaml index fe57c5289597..59ab5d6df3b5 100644 --- a/airbyte-integrations/connectors/source-captain-data/source_captain_data/manifest.yaml +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/manifest.yaml @@ -160,9 +160,11 @@ spec: additionalProperties: true properties: api_key: + title: API Key type: string description: Your Captain Data project API key. airbyte_secret: true project_uid: + title: Project UID type: string description: Your Captain Data project uuid. From 082a2ed6c2c0c93af2c4b04adc916135417e619f Mon Sep 17 00:00:00 2001 From: Elliot T Date: Thu, 20 Apr 2023 11:54:19 +0200 Subject: [PATCH 05/11] fix comments --- .../integration_tests/configured_catalog.json | 81 ++++++++++++++++--- .../source_captain_data/manifest.yaml | 2 +- .../schemas/job_results.json | 1 + .../source_captain_data/schemas/jobs.json | 5 +- .../schemas/workflows.json | 17 +++- .../schemas/workspace.json | 1 + 6 files changed, 92 insertions(+), 15 deletions(-) diff --git a/airbyte-integrations/connectors/source-captain-data/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-captain-data/integration_tests/configured_catalog.json index a7114c8d99bc..351c84710677 100644 --- a/airbyte-integrations/connectors/source-captain-data/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-captain-data/integration_tests/configured_catalog.json @@ -1,31 +1,88 @@ { "streams": [ { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "workspace", + "default_cursor_field": null, "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "name": "workspace", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [ + [ + "name" + ] + ], + "supported_sync_modes": [ + "full_refresh" + ] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "workflows", + "default_cursor_field": null, "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "name": "workflows", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [ + [ + "uid" + ] + ], + "supported_sync_modes": [ + "full_refresh" + ] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { + "default_cursor_field": null, + "json_schema": {}, "name": "jobs", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [ + [ + "uid" + ] + ], + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "name": "job_results", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [ + [ + "job_result_id" + ] + ], + "supported_sync_modes": [ + "full_refresh" + ] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "sync_mode": "full_refresh" } ] -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/manifest.yaml b/airbyte-integrations/connectors/source-captain-data/source_captain_data/manifest.yaml index 59ab5d6df3b5..4505a13181c7 100644 --- a/airbyte-integrations/connectors/source-captain-data/source_captain_data/manifest.yaml +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/manifest.yaml @@ -150,7 +150,7 @@ check: spec: type: Spec - documentationUrl: https://docs.captaindata.co/#1da39f5f-a26f-4a53-9e84-3e4b28077eae + documentationUrl: https://docs.airbyte.com/integrations/sources/captain-data connection_specification: title: Captain Data Spec type: object diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/job_results.json b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/job_results.json index d975b10f74c5..f487b33fbfdc 100644 --- a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/job_results.json +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/job_results.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "job_uid": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/jobs.json b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/jobs.json index 1e302bfab8da..50f12fda0fb9 100644 --- a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/jobs.json +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/jobs.json @@ -3,7 +3,10 @@ "type": "object", "properties": { "accounts": { - "type": ["null", "array"] + "items": { + "type": "string" + }, + "type": "array" }, "configuration": { "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workflows.json b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workflows.json index 9674b453b953..7548608790cc 100644 --- a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workflows.json +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workflows.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "created_at": { "type": ["null", "string"] @@ -12,7 +13,21 @@ "type": ["null", "integer"] }, "steps": { - "type": ["null", "array"] + "items": { + "properties": { + "name": { + "type": "string" + }, + "step_uid": { + "type": "string" + }, + "uid": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" }, "template_uid": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workspace.json b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workspace.json index 420960237e91..7644d353733f 100644 --- a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workspace.json +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workspace.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "project_uid": { "type": ["null", "string"] From 7158e9246c2674ed2c91d4b041efa342b29985e8 Mon Sep 17 00:00:00 2001 From: Elliot T Date: Thu, 20 Apr 2023 11:54:44 +0200 Subject: [PATCH 06/11] add logo --- .../init-oss/src/main/resources/icons/captain-data.svg | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 airbyte-config-oss/init-oss/src/main/resources/icons/captain-data.svg diff --git a/airbyte-config-oss/init-oss/src/main/resources/icons/captain-data.svg b/airbyte-config-oss/init-oss/src/main/resources/icons/captain-data.svg new file mode 100644 index 000000000000..7a54fe2fc550 --- /dev/null +++ b/airbyte-config-oss/init-oss/src/main/resources/icons/captain-data.svg @@ -0,0 +1,3 @@ + + + From b81e9acdde2cef3a374d4e05f0bac89dc9540b60 Mon Sep 17 00:00:00 2001 From: Elliot T Date: Mon, 12 Jun 2023 14:37:46 +0200 Subject: [PATCH 07/11] add finished job filter --- .../source-captain-data/source_captain_data/manifest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/manifest.yaml b/airbyte-integrations/connectors/source-captain-data/source_captain_data/manifest.yaml index 4505a13181c7..5cb698d2aa9f 100644 --- a/airbyte-integrations/connectors/source-captain-data/source_captain_data/manifest.yaml +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/manifest.yaml @@ -100,7 +100,7 @@ definitions: type: DpathExtractor field_path: [] record_filter: - condition: "{{ record['total_row_count'] > 0 }}" + condition: "{{ record['total_row_count'] > 0 and record['status'] == 'finished' }}" job_results_stream: $ref: "#/definitions/base_stream" From 814425dbdb04d290054d039ba83c172ec9948ea4 Mon Sep 17 00:00:00 2001 From: Elliot T Date: Mon, 12 Jun 2023 15:35:49 +0200 Subject: [PATCH 08/11] suggested changes --- .../source_captain_data/schemas/workflows.json | 6 +++--- docs/integrations/sources/captain-data.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workflows.json b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workflows.json index 7548608790cc..4709a5f21f51 100644 --- a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workflows.json +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workflows.json @@ -16,13 +16,13 @@ "items": { "properties": { "name": { - "type": "string" + "type": ["null", "string"] }, "step_uid": { - "type": "string" + "type": ["null", "string"] }, "uid": { - "type": "string" + "type": ["null", "string"] } }, "type": "object" diff --git a/docs/integrations/sources/captain-data.md b/docs/integrations/sources/captain-data.md index 9113b2e219da..dc7fff642f97 100644 --- a/docs/integrations/sources/captain-data.md +++ b/docs/integrations/sources/captain-data.md @@ -60,6 +60,6 @@ Captain Data [API reference](https://docs.captaindata.co/#intro) has v3 at prese ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------- | :------------- | -| 0.1.0 | 2023-04-15 | [Init](https://github.com/airbytehq/airbyte/pull/) | Initial commit | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------ | :------------- | +| 0.1.0 | 2023-04-15 | [Init](https://github.com/airbytehq/airbyte/pull/25230) | Initial commit | From 5555578b933b6f58c134e0582ecba35c271bf2b0 Mon Sep 17 00:00:00 2001 From: Elliot T Date: Mon, 12 Jun 2023 15:43:53 +0200 Subject: [PATCH 09/11] add metadata.yml --- .../connectors/source-captain-data/icon.svg | 0 .../source-captain-data/metadata.yaml | 21 +++++++++++++++++++ 2 files changed, 21 insertions(+) rename airbyte-config-oss/init-oss/src/main/resources/icons/captain-data.svg => airbyte-integrations/connectors/source-captain-data/icon.svg (100%) create mode 100644 airbyte-integrations/connectors/source-captain-data/metadata.yaml diff --git a/airbyte-config-oss/init-oss/src/main/resources/icons/captain-data.svg b/airbyte-integrations/connectors/source-captain-data/icon.svg similarity index 100% rename from airbyte-config-oss/init-oss/src/main/resources/icons/captain-data.svg rename to airbyte-integrations/connectors/source-captain-data/icon.svg diff --git a/airbyte-integrations/connectors/source-captain-data/metadata.yaml b/airbyte-integrations/connectors/source-captain-data/metadata.yaml new file mode 100644 index 000000000000..3247050ffa22 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/metadata.yaml @@ -0,0 +1,21 @@ +data: + connectorSubtype: api + connectorType: source + definitionId: ??? + dockerImageTag: 0.1.0 + dockerRepository: airbyte/source-captain-data + githubIssueLabel: source-captain-data + icon: captain-data.svg + license: MIT + name: Captain Data + registries: + cloud: + enabled: false + oss: + enabled: true + releaseStage: alpha + documentationUrl: https://docs.airbyte.com/integrations/sources/captain-data + tags: + - language:low-code + - language:python +metadataSpecVersion: "1.0" From 7936e8484c81eec77f3e4520302558a0e1f20709 Mon Sep 17 00:00:00 2001 From: Elliot T Date: Tue, 13 Jun 2023 09:26:49 +0200 Subject: [PATCH 10/11] add definition id --- .../connectors/source-captain-data/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-captain-data/metadata.yaml b/airbyte-integrations/connectors/source-captain-data/metadata.yaml index 3247050ffa22..a1ee258a6ed4 100644 --- a/airbyte-integrations/connectors/source-captain-data/metadata.yaml +++ b/airbyte-integrations/connectors/source-captain-data/metadata.yaml @@ -1,7 +1,7 @@ data: connectorSubtype: api connectorType: source - definitionId: ??? + definitionId: fa290790-1dca-43e7-8ced-6a40b2a66099 dockerImageTag: 0.1.0 dockerRepository: airbyte/source-captain-data githubIssueLabel: source-captain-data From 3320ffceb81bb42e6d4f6e0bc95876287eae2767 Mon Sep 17 00:00:00 2001 From: Elliot T Date: Tue, 13 Jun 2023 10:24:51 +0200 Subject: [PATCH 11/11] update schemas --- .../connectors/source-captain-data/acceptance-test-config.yml | 2 +- .../source-captain-data/source_captain_data/schemas/jobs.json | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-captain-data/acceptance-test-config.yml b/airbyte-integrations/connectors/source-captain-data/acceptance-test-config.yml index 8d8e51e6be24..9f4ac3d46b6e 100644 --- a/airbyte-integrations/connectors/source-captain-data/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-captain-data/acceptance-test-config.yml @@ -4,7 +4,7 @@ connector_image: airbyte/source-captain-data:dev acceptance_tests: spec: tests: - - spec_path: "source_captain_data/spec.yaml" + - spec_path: "source_captain_data/manifest.yaml" connection: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/jobs.json b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/jobs.json index 50f12fda0fb9..dd7cf2a8670c 100644 --- a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/jobs.json +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/jobs.json @@ -1,10 +1,11 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "accounts": { "items": { - "type": "string" + "type": ["null", "string"] }, "type": "array" },