From c5dfc98927b5d952512ba0b2228bb79c7b76664c Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Mon, 13 May 2024 16:50:37 +0200 Subject: [PATCH 1/4] Add new source --- .../connectors/source-workday-raas/README.md | 103 ++++++++ .../source-workday-raas/__init__.py | 3 + .../acceptance-test-config.yml | 32 +++ .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 0 .../integration_tests/acceptance.py | 14 ++ .../integration_tests/configured_catalog.json | 49 ++++ .../integration_tests/invalid_config.json | 3 + .../integration_tests/sample_config.json | 0 .../integration_tests/sample_state.json | 5 + .../connectors/source-workday-raas/main.py | 13 + .../source-workday-raas/metadata.yaml | 35 +++ .../source-workday-raas/pyproject.toml | 27 ++ .../source_workday_raas/__init__.py | 8 + .../source_workday_raas/manifest.yaml | 232 ++++++++++++++++++ .../source_workday_raas/run.py | 13 + .../source_workday_raas/source.py | 18 ++ docs/integrations/sources/workday-raas.md | 38 +++ 18 files changed, 596 insertions(+) create mode 100644 airbyte-integrations/connectors/source-workday-raas/README.md create mode 100644 airbyte-integrations/connectors/source-workday-raas/__init__.py create mode 100644 airbyte-integrations/connectors/source-workday-raas/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-workday-raas/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-workday-raas/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-workday-raas/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-workday-raas/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-workday-raas/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-workday-raas/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-workday-raas/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-workday-raas/main.py create mode 100644 airbyte-integrations/connectors/source-workday-raas/metadata.yaml create mode 100644 airbyte-integrations/connectors/source-workday-raas/pyproject.toml create mode 100644 airbyte-integrations/connectors/source-workday-raas/source_workday_raas/__init__.py create mode 100644 airbyte-integrations/connectors/source-workday-raas/source_workday_raas/manifest.yaml create mode 100644 airbyte-integrations/connectors/source-workday-raas/source_workday_raas/run.py create mode 100644 airbyte-integrations/connectors/source-workday-raas/source_workday_raas/source.py create mode 100644 docs/integrations/sources/workday-raas.md diff --git a/airbyte-integrations/connectors/source-workday-raas/README.md b/airbyte-integrations/connectors/source-workday-raas/README.md new file mode 100644 index 000000000000..c513af3c2f89 --- /dev/null +++ b/airbyte-integrations/connectors/source-workday-raas/README.md @@ -0,0 +1,103 @@ +# Workday Raas Source + +This is the repository for the Workday Raas configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/workday-raas). + +## Local development + +### Prerequisites + +* Python (`^3.9`) +* Poetry (`^1.7`) - installation instructions [here](https://python-poetry.org/docs/#installation) + + + +### Installing the connector + +From this connector directory, run: +```bash +poetry install --with dev +``` + + +### Create credentials + +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/workday-raas) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `src/source_workday_raas/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. + + +### Locally running the connector + +``` +poetry run source-workday-raas spec +poetry run source-workday-raas check --config secrets/config.json +poetry run source-workday-raas discover --config secrets/config.json +poetry run source-workday-raas read --config secrets/config.json --catalog sample_files/configured_catalog.json +``` + +### Running tests + +To run tests locally, from the connector directory run: + +``` +poetry run pytest tests +``` + +### Building the docker image + +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: +```bash +airbyte-ci connectors --name=source-workday-raas build +``` + +An image will be available on your host with the tag `airbyte/source-workday-raas:dev`. + + +### Running as a docker container + +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-workday-raas:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-workday-raas:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-workday-raas:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-workday-raas:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +### Running our CI test suite + +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-workday-raas test +``` + +### Customizing acceptance Tests + +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +### Dependency Management + +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. + +## Publishing a new version of the connector + +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-workday-raas test` +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` +3. Make sure the `metadata.yaml` content is up to date. +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/workday-raas.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-workday-raas/__init__.py b/airbyte-integrations/connectors/source-workday-raas/__init__.py new file mode 100644 index 000000000000..66f6de8cb2bb --- /dev/null +++ b/airbyte-integrations/connectors/source-workday-raas/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-workday-raas/acceptance-test-config.yml b/airbyte-integrations/connectors/source-workday-raas/acceptance-test-config.yml new file mode 100644 index 000000000000..be208eebc40b --- /dev/null +++ b/airbyte-integrations/connectors/source-workday-raas/acceptance-test-config.yml @@ -0,0 +1,32 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-workday-raas:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_workday_raas/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + incremental: + bypass_reason: "This connector does not implement incremental sync" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-workday-raas/integration_tests/__init__.py b/airbyte-integrations/connectors/source-workday-raas/integration_tests/__init__.py new file mode 100644 index 000000000000..66f6de8cb2bb --- /dev/null +++ b/airbyte-integrations/connectors/source-workday-raas/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-workday-raas/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-workday-raas/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-workday-raas/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-workday-raas/integration_tests/acceptance.py new file mode 100644 index 000000000000..efc25f08ce82 --- /dev/null +++ b/airbyte-integrations/connectors/source-workday-raas/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-workday-raas/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-workday-raas/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..3e6731e793b3 --- /dev/null +++ b/airbyte-integrations/connectors/source-workday-raas/integration_tests/configured_catalog.json @@ -0,0 +1,49 @@ +{ + "streams": [ + { + "stream": { + "name": "workers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "people", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "customers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "customers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "customers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + ] +} diff --git a/airbyte-integrations/connectors/source-workday-raas/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-workday-raas/integration_tests/invalid_config.json new file mode 100644 index 000000000000..f3732995784f --- /dev/null +++ b/airbyte-integrations/connectors/source-workday-raas/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "todo-wrong-field": "this should be an incomplete config file, used in standard tests" +} diff --git a/airbyte-integrations/connectors/source-workday-raas/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-workday-raas/integration_tests/sample_config.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-workday-raas/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-workday-raas/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-workday-raas/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-workday-raas/main.py b/airbyte-integrations/connectors/source-workday-raas/main.py new file mode 100644 index 000000000000..3dc5a38b4cb8 --- /dev/null +++ b/airbyte-integrations/connectors/source-workday-raas/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_workday_raas import SourceWorkdayRaas + +if __name__ == "__main__": + source = SourceWorkdayRaas() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-workday-raas/metadata.yaml b/airbyte-integrations/connectors/source-workday-raas/metadata.yaml new file mode 100644 index 000000000000..d9c415f9fc00 --- /dev/null +++ b/airbyte-integrations/connectors/source-workday-raas/metadata.yaml @@ -0,0 +1,35 @@ +data: + allowedHosts: + hosts: + - + registries: + oss: + enabled: true + cloud: + enabled: false + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-workday-raas + connectorBuildOptions: + # Please update to the latest version of the connector base image. + # https://hub.docker.com/r/airbyte/python-connector-base + # Please use the full address with sha256 hash to guarantee build reproducibility. + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 + connectorSubtype: api + connectorType: source + definitionId: 7b8b9550-331c-46c8-a299-943fb6ae2a72 + dockerImageTag: 0.1.0 + dockerRepository: airbyte/source-workday-raas + githubIssueLabel: source-workday-raas + icon: workday-raas.svg + license: MIT + name: Workday Raas + releaseDate: + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/sources/workday-raas + tags: + - language:python + - cdk:low-code +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-workday-raas/pyproject.toml b/airbyte-integrations/connectors/source-workday-raas/pyproject.toml new file mode 100644 index 000000000000..701dcb95cc17 --- /dev/null +++ b/airbyte-integrations/connectors/source-workday-raas/pyproject.toml @@ -0,0 +1,27 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.1.0" +name = "source-workday-raas" +description = "Source implementation for workday-raas." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/workday-raas" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +packages = [ { include = "source_workday_raas" }, {include = "main.py" } ] + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "^0" + +[tool.poetry.scripts] +source-workday-raas = "source_workday_raas.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "*" +pytest-mock = "*" +pytest = "*" diff --git a/airbyte-integrations/connectors/source-workday-raas/source_workday_raas/__init__.py b/airbyte-integrations/connectors/source-workday-raas/source_workday_raas/__init__.py new file mode 100644 index 000000000000..7648df9907c4 --- /dev/null +++ b/airbyte-integrations/connectors/source-workday-raas/source_workday_raas/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceWorkdayRaas + +__all__ = ["SourceWorkdayRaas"] diff --git a/airbyte-integrations/connectors/source-workday-raas/source_workday_raas/manifest.yaml b/airbyte-integrations/connectors/source-workday-raas/source_workday_raas/manifest.yaml new file mode 100644 index 000000000000..d91070d812f6 --- /dev/null +++ b/airbyte-integrations/connectors/source-workday-raas/source_workday_raas/manifest.yaml @@ -0,0 +1,232 @@ +version: 0.85.0 + +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - workers + +definitions: + streams: + workers: + type: DeclarativeStream + name: workers + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: '#/definitions/base_requester' + path: workers + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + inject_into: request_parameter + field_name: Limit + pagination_strategy: + type: OffsetIncrement + page_size: 100 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: '#/schemas/workers' + people: + type: DeclarativeStream + name: people + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: '#/definitions/base_requester' + path: people + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + inject_into: request_parameter + field_name: Limit + pagination_strategy: + type: OffsetIncrement + page_size: 100 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: '#/schemas/people' + supervisory_organizations: + type: DeclarativeStream + name: supervisory_organizations + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: '#/definitions/base_requester' + path: supervisory_organizations + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + inject_into: request_parameter + field_name: Limit + pagination_strategy: + type: OffsetIncrement + page_size: 100 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: '#/schemas/supervisory_organizations' + org_charts: + type: DeclarativeStream + name: org_charts + retriever: + type: SimpleRetriever + requester: + $ref: '#/definitions/base_requester' + path: /supervisoryOrganizations/{org_id}/orgChart + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + partition_router: + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: org_id + stream: + $ref: '#/definitions/streams/supervisory_organizations' + schema_loader: + type: InlineSchemaLoader + schema: + $ref: '#/schemas/org_charts' + custom_reports: + type: DeclarativeStream + name: custom_reports + retriever: + type: SimpleRetriever + requester: + $ref: '#/definitions/base_requester' + path: /service/customreport2/{tenant_id}/{report_name} + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + schema_loader: + type: InlineSchemaLoader + schema: + $ref: '#/schemas/custom_reports' + base_requester: + type: HttpRequester + url_base: ///ccx + authenticator: + type: BasicHttpAuthenticator + username: '{{ config["username"] }}' + password: '{{ config["password"] }}' + +streams: + - $ref: '#/definitions/streams/workers' + - $ref: '#/definitions/streams/people' + - $ref: '#/definitions/streams/supervisory_organizations' + - $ref: '#/definitions/streams/org_charts' + - $ref: '#/definitions/streams/custom_reports' + +spec: + type: Spec + connection_specification: + type: object + $schema: http://json-schema.org/draft-07/schema# + required: + - username + - host + - tenant_id + - custom_report_name + properties: + username: + type: string + title: Username + order: 0 + password: + type: string + title: Password + always_show: true + airbyte_secret: true + order: 1 + host: + type: string + title: Host + order: 2 + tenant_id: + type: string + title: Tenant Id + order: 3 + custom_report_name: + type: array + title: Custom Report Name + order: 4 + additionalProperties: true + +metadata: + autoImportSchema: + workers: true + people: true + supervisory_organizations: true + org_charts: true + custom_reports: true + +schemas: + workers: + type: object + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + properties: {} + people: + type: object + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + properties: {} + supervisory_organizations: + type: object + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + properties: {} + org_charts: + type: object + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + properties: {} + custom_reports: + type: object + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + properties: {} diff --git a/airbyte-integrations/connectors/source-workday-raas/source_workday_raas/run.py b/airbyte-integrations/connectors/source-workday-raas/source_workday_raas/run.py new file mode 100644 index 000000000000..824dbb4486b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-workday-raas/source_workday_raas/run.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from .source import SourceWorkdayRaas + +def run(): + source = SourceWorkdayRaas() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-workday-raas/source_workday_raas/source.py b/airbyte-integrations/connectors/source-workday-raas/source_workday_raas/source.py new file mode 100644 index 000000000000..bbfe01c17e24 --- /dev/null +++ b/airbyte-integrations/connectors/source-workday-raas/source_workday_raas/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceWorkdayRaas(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/docs/integrations/sources/workday-raas.md b/docs/integrations/sources/workday-raas.md new file mode 100644 index 000000000000..505d471efa79 --- /dev/null +++ b/docs/integrations/sources/workday-raas.md @@ -0,0 +1,38 @@ +# Workday RAAS + +## Overview + +The Workday RAAS source currently supports Full Refresh syncs only. This means that all contents for all chosen streams will be replaced with every sync. + +### Output schema + +This Source is capable of syncing the following core Streams: + +- + +If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose) + +### Features + +| Feature | Supported? | | +| :---------------------------- | :--------- | :-- | +| Full Refresh Sync | Yes | | +| Incremental Sync | No | | +| Replicate Incremental Deletes | No | | +| SSL connection | Yes | | +| Namespaces | No | | + +## Getting started + +Workday RAAS facilitates resource planning. With it you can manage your employee's skills and schedule assignment of +your employees to the right projects. + +### Requirements + +- + +## CHANGELOG + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:-----------------------------------------------|:-----------------------| +| 0.1.0 | 2024-05-13 | [](https://github.com/airbytehq/airbyte/pull/) | Initial implementation | From 0cf0e9531cadedd562760dae959ae9dc5fd99f28 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Tue, 9 Jul 2024 12:22:48 -0700 Subject: [PATCH 2/4] Update airbyte-integrations/connectors/source-workday-raas/metadata.yaml --- .../connectors/source-workday-raas/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-workday-raas/metadata.yaml b/airbyte-integrations/connectors/source-workday-raas/metadata.yaml index d9c415f9fc00..7c1f3b29db3c 100644 --- a/airbyte-integrations/connectors/source-workday-raas/metadata.yaml +++ b/airbyte-integrations/connectors/source-workday-raas/metadata.yaml @@ -1,7 +1,7 @@ data: allowedHosts: hosts: - - + - "*" registries: oss: enabled: true From 4e34efb4d167dc7f01689d0fc68abff071c9596e Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 5 Aug 2024 10:42:32 -0700 Subject: [PATCH 3/4] Update manifest. only keep the custom reports endpoint --- .../source_workday_raas/manifest.yaml | 275 +++++++----------- 1 file changed, 107 insertions(+), 168 deletions(-) diff --git a/airbyte-integrations/connectors/source-workday-raas/source_workday_raas/manifest.yaml b/airbyte-integrations/connectors/source-workday-raas/source_workday_raas/manifest.yaml index d91070d812f6..20d74e1e788f 100644 --- a/airbyte-integrations/connectors/source-workday-raas/source_workday_raas/manifest.yaml +++ b/airbyte-integrations/connectors/source-workday-raas/source_workday_raas/manifest.yaml @@ -1,165 +1,49 @@ -version: 0.85.0 +version: 3.9.6 type: DeclarativeSource check: type: CheckStream stream_names: - - workers + - Custom reports definitions: streams: - workers: + Custom reports: type: DeclarativeStream - name: workers - primary_key: - - id + name: Custom reports retriever: type: SimpleRetriever requester: - $ref: '#/definitions/base_requester' - path: workers + $ref: "#/definitions/base_requester" + path: "{{stream_partition.report}}" http_method: GET + request_parameters: + format: json record_selector: type: RecordSelector extractor: type: DpathExtractor field_path: - - data - paginator: - type: DefaultPaginator - page_size_option: - type: RequestOption - inject_into: request_parameter - field_name: Limit - pagination_strategy: - type: OffsetIncrement - page_size: 100 - inject_on_first_request: true - schema_loader: - type: InlineSchemaLoader - schema: - $ref: '#/schemas/workers' - people: - type: DeclarativeStream - name: people - primary_key: - - id - retriever: - type: SimpleRetriever - requester: - $ref: '#/definitions/base_requester' - path: people - http_method: GET - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - data - paginator: - type: DefaultPaginator - page_size_option: - type: RequestOption - inject_into: request_parameter - field_name: Limit - pagination_strategy: - type: OffsetIncrement - page_size: 100 - inject_on_first_request: true - schema_loader: - type: InlineSchemaLoader - schema: - $ref: '#/schemas/people' - supervisory_organizations: - type: DeclarativeStream - name: supervisory_organizations - primary_key: - - id - retriever: - type: SimpleRetriever - requester: - $ref: '#/definitions/base_requester' - path: supervisory_organizations - http_method: GET - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - data - paginator: - type: DefaultPaginator - page_size_option: - type: RequestOption - inject_into: request_parameter - field_name: Limit - pagination_strategy: - type: OffsetIncrement - page_size: 100 - inject_on_first_request: true - schema_loader: - type: InlineSchemaLoader - schema: - $ref: '#/schemas/supervisory_organizations' - org_charts: - type: DeclarativeStream - name: org_charts - retriever: - type: SimpleRetriever - requester: - $ref: '#/definitions/base_requester' - path: /supervisoryOrganizations/{org_id}/orgChart - http_method: GET - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: [] + - Report_Entry partition_router: - - type: SubstreamPartitionRouter - parent_stream_configs: - - type: ParentStreamConfig - parent_key: id - partition_field: org_id - stream: - $ref: '#/definitions/streams/supervisory_organizations' - schema_loader: - type: InlineSchemaLoader - schema: - $ref: '#/schemas/org_charts' - custom_reports: - type: DeclarativeStream - name: custom_reports - retriever: - type: SimpleRetriever - requester: - $ref: '#/definitions/base_requester' - path: /service/customreport2/{tenant_id}/{report_name} - http_method: GET - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: [] + type: ListPartitionRouter + values: "{{config.reports}}" + cursor_field: report schema_loader: type: InlineSchemaLoader schema: - $ref: '#/schemas/custom_reports' + $ref: "#/schemas/Custom reports" base_requester: type: HttpRequester - url_base: ///ccx + url_base: https://{{config.host}}/ccx/service/customreport2/{{config.tenant_id}}/ authenticator: type: BasicHttpAuthenticator - username: '{{ config["username"] }}' - password: '{{ config["password"] }}' + password: "{{ config[\"password\"] }}" + username: "{{ config[\"username\"] }}" streams: - - $ref: '#/definitions/streams/workers' - - $ref: '#/definitions/streams/people' - - $ref: '#/definitions/streams/supervisory_organizations' - - $ref: '#/definitions/streams/org_charts' - - $ref: '#/definitions/streams/custom_reports' + - $ref: "#/definitions/streams/Custom reports" spec: type: Spec @@ -170,63 +54,118 @@ spec: - username - host - tenant_id - - custom_report_name properties: username: type: string - title: Username order: 0 + title: Username password: type: string + order: 1 title: Password always_show: true airbyte_secret: true - order: 1 host: type: string - title: Host order: 2 + title: Host tenant_id: type: string - title: Tenant Id order: 3 - custom_report_name: + title: Tenant ID + reports: type: array - title: Custom Report Name order: 4 + title: Reports additionalProperties: true metadata: autoImportSchema: - workers: true - people: true - supervisory_organizations: true - org_charts: true - custom_reports: true + Custom reports: true + testedStreams: + Custom reports: + streamHash: 375faa072cf11e83f7c37c06bb06c549b2e48d1b + hasResponse: true + responsesAreSuccessful: true + hasRecords: true + primaryKeysArePresent: true + primaryKeysAreUnique: true schemas: - workers: + Custom reports: type: object - $schema: http://json-schema.org/draft-07/schema# + $schema: http://json-schema.org/schema# additionalProperties: true - properties: {} - people: - type: object - $schema: http://json-schema.org/draft-07/schema# - additionalProperties: true - properties: {} - supervisory_organizations: - type: object - $schema: http://json-schema.org/draft-07/schema# - additionalProperties: true - properties: {} - org_charts: - type: object - $schema: http://json-schema.org/draft-07/schema# - additionalProperties: true - properties: {} - custom_reports: - type: object - $schema: http://json-schema.org/draft-07/schema# - additionalProperties: true - properties: {} + properties: + Business_Title: + type: + - string + - "null" + Department: + type: + - string + - "null" + Department_ID: + type: + - string + - "null" + Division: + type: + - string + - "null" + Division_ID: + type: + - string + - "null" + Employee_ID: + type: + - string + - "null" + First_Name: + type: + - string + - "null" + Hire_Date: + type: + - string + - "null" + Last_Name: + type: + - string + - "null" + Manager_Email: + type: + - string + - "null" + Manager_Flag: + type: + - string + - "null" + Preferred_First_Name: + type: + - string + - "null" + Preferred_Last_Name: + type: + - string + - "null" + Primary_Work_Location: + type: + - string + - "null" + Remote_Flag: + type: + - string + - "null" + Work_Email: + type: + - string + - "null" + Worker_Sub_Type: + type: + - string + - "null" + Worker_Type: + type: + - string + - "null" From 07ccb92ae08e1432a90e101ade69b1d22ff4baef Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 5 Aug 2024 10:56:33 -0700 Subject: [PATCH 4/4] Update airbyte-integrations/connectors/source-workday-raas/pyproject.toml --- .../connectors/source-workday-raas/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-workday-raas/pyproject.toml b/airbyte-integrations/connectors/source-workday-raas/pyproject.toml index 701dcb95cc17..93b4288fb45a 100644 --- a/airbyte-integrations/connectors/source-workday-raas/pyproject.toml +++ b/airbyte-integrations/connectors/source-workday-raas/pyproject.toml @@ -16,7 +16,7 @@ packages = [ { include = "source_workday_raas" }, {include = "main.py" } ] [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = "^0" +airbyte-cdk = "3.9.6" [tool.poetry.scripts] source-workday-raas = "source_workday_raas.run:run"