diff --git a/airbyte-config/init/src/main/resources/icons/clockify.svg b/airbyte-config/init/src/main/resources/icons/clockify.svg new file mode 100644 index 000000000000..562fbab20a7e --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/clockify.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index cb3f6a0e3597..35de0b71653b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -220,6 +220,14 @@ documentationUrl: https://docs.airbyte.com/integrations/sources/courier sourceType: api releaseStage: alpha +- name: Clockify + sourceDefinitionId: e71aae8a-5143-11ed-bdc3-0242ac120002 + dockerRepository: airbyte/source-clockify + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/clockify + icon: clockify.svg + sourceType: api + releaseStage: alpha - name: Customer.io sourceDefinitionId: c47d6804-8b98-449f-970a-5ddb5cb5d7aa dockerRepository: farosai/airbyte-customer-io-source diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 34b6ea17cc2d..3c949efa6894 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2187,6 +2187,31 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-clockify:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/clockify" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Clockify Spec" + type: "object" + required: + - "workspace_id" + - "api_key" + additionalProperties: true + properties: + workspace_id: + title: "Workspace Id" + description: "WorkSpace Id" + type: "string" + api_key: + title: "API Key" + description: "You can get your api access_key here This API is Case Sensitive." + type: "string" + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "farosai/airbyte-customer-io-source:0.1.23" spec: documentationUrl: "https://docs.faros.ai" diff --git a/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py b/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py index dd13b87ef40d..666c5aa26d9b 100644 --- a/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py @@ -1,7 +1,8 @@ # # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from unittest.mock import patch, PropertyMock + +from unittest.mock import PropertyMock, patch from airbyte_cdk.logger import AirbyteLogger from source_asana.source import SourceAsana @@ -27,7 +28,7 @@ def test_check_connection_empty_config(config): def test_check_connection_exception(config): - with patch('source_asana.streams.Workspaces.use_cache', new_callable=PropertyMock, return_value=False): + with patch("source_asana.streams.Workspaces.use_cache", new_callable=PropertyMock, return_value=False): ok, error_msg = SourceAsana().check_connection(logger, config=config) assert not ok diff --git a/airbyte-integrations/connectors/source-clockify/.dockerignore b/airbyte-integrations/connectors/source-clockify/.dockerignore new file mode 100644 index 000000000000..a5f918cda764 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_clockify +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-clockify/.gitignore b/airbyte-integrations/connectors/source-clockify/.gitignore new file mode 100644 index 000000000000..945168c8f81b --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/.gitignore @@ -0,0 +1,3 @@ +users.yml +projects.yml +schemas diff --git a/airbyte-integrations/connectors/source-clockify/Dockerfile b/airbyte-integrations/connectors/source-clockify/Dockerfile new file mode 100644 index 000000000000..a19cd8507ded --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.13-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_clockify ./source_clockify + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-clockify diff --git a/airbyte-integrations/connectors/source-clockify/README.md b/airbyte-integrations/connectors/source-clockify/README.md new file mode 100644 index 000000000000..0f4544e0896e --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/README.md @@ -0,0 +1,132 @@ +# Clockify Source + +This is the repository for the Clockify source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/clockify). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-clockify:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/clockify) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_clockify/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source clockify test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-clockify:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-clockify:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-clockify:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-clockify:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-clockify:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-clockify:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-clockify:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-clockify:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-clockify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-clockify/acceptance-test-config.yml new file mode 100644 index 000000000000..451c4f772c54 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/acceptance-test-config.yml @@ -0,0 +1,19 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-clockify:dev +tests: + spec: + - spec_path: "source_clockify/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-clockify/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-clockify/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-clockify/build.gradle b/airbyte-integrations/connectors/source-clockify/build.gradle new file mode 100644 index 000000000000..9fbbaa9b16f7 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_clockify' +} diff --git a/airbyte-integrations/connectors/source-clockify/integration_tests/__init__.py b/airbyte-integrations/connectors/source-clockify/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-clockify/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-clockify/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-clockify/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-clockify/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..a939aad354a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/integration_tests/configured_catalog.json @@ -0,0 +1,109 @@ +{ + "streams": [ + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "users", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": null, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "projects", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": null, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "clients", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": null, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "tags", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": null, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "user_groups", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": null, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "time_entries", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": null, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "tasks", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": null, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + } + ] +} diff --git a/airbyte-integrations/connectors/source-clockify/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-clockify/integration_tests/invalid_config.json new file mode 100644 index 000000000000..306220396277 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "", + "workspace_id": "" +} diff --git a/airbyte-integrations/connectors/source-clockify/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-clockify/integration_tests/sample_config.json new file mode 100644 index 000000000000..b5ef13de8006 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/integration_tests/sample_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "", + "workspace_id": "" +} diff --git a/airbyte-integrations/connectors/source-clockify/main.py b/airbyte-integrations/connectors/source-clockify/main.py new file mode 100644 index 000000000000..72002deb5c04 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_clockify import SourceClockify + +if __name__ == "__main__": + source = SourceClockify() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-clockify/requirements.txt b/airbyte-integrations/connectors/source-clockify/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-clockify/setup.py b/airbyte-integrations/connectors/source-clockify/setup.py new file mode 100644 index 000000000000..4e83a518e06e --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/setup.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.2.0", +] + +TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "source-acceptance-test", "responses"] + +setup( + name="source_clockify", + description="Source implementation for Clockify.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/__init__.py b/airbyte-integrations/connectors/source-clockify/source_clockify/__init__.py new file mode 100644 index 000000000000..952d1a5a6623 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceClockify + +__all__ = ["SourceClockify"] diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/clients.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/clients.json new file mode 100644 index 000000000000..00a10979e40c --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/clients.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/schema#", + "properties": { + "address": { + "type": ["null", "string"] + }, + "archived": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "note": { + "type": ["null", "string"] + }, + "workspaceId": { + "type": "string" + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/projects.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/projects.json new file mode 100644 index 000000000000..faa01b58a0fd --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/projects.json @@ -0,0 +1,173 @@ +{ + "$schema": "http://json-schema.org/schema#", + "properties": { + "archived": { + "type": "boolean" + }, + "billable": { + "type": "boolean" + }, + "budgetEstimate": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "integer" + }, + { + "properties": { + "estimate": { + "type": ["null", "string"] + }, + "type": { + "type": "string" + }, + "resetOption": { + "type": ["null", "string"] + }, + "active": { + "type": "boolean" + } + }, + "type": "object" + } + ] + }, + "clientId": { + "type": "string" + }, + "clientName": { + "type": "string" + }, + "color": { + "type": "string" + }, + "costRate": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "properties": { + "amount": { + "type": ["null", "string", "integer"] + }, + "currency": { + "type": ["null", "string"] + } + }, + "type": "object" + } + ] + }, + "duration": { + "type": "string" + }, + "estimate": { + "properties": { + "estimate": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "hourlyRate": { + "properties": { + "amount": { + "type": "integer" + }, + "currency": { + "type": "string" + } + }, + "type": "object" + }, + "id": { + "type": "string" + }, + "memberships": { + "items": { + "properties": { + "costRate": { + "type": "null" + }, + "hourlyRate": { + "anyOf": [ + { + "type": "null" + }, + { + "properties": { + "amount": { + "type": "integer" + }, + "currency": { + "type": "string" + } + }, + "type": "object" + } + ] + }, + "membershipStatus": { + "type": "string" + }, + "membershipType": { + "type": "string" + }, + "targetId": { + "type": "string" + }, + "userId": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "name": { + "type": "string" + }, + "note": { + "type": "string" + }, + "public": { + "type": "boolean" + }, + "template": { + "type": "boolean" + }, + "timeEstimate": { + "properties": { + "active": { + "type": "boolean" + }, + "estimate": { + "type": "string" + }, + "includeNonBillable": { + "type": "boolean" + }, + "resetOption": { + "type": ["null", "string"] + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "workspaceId": { + "type": "string" + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tags.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tags.json new file mode 100644 index 000000000000..75b53dd8cfea --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tags.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/schema#", + "properties": { + "archived": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "workspaceId": { + "type": "string" + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tasks.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tasks.json new file mode 100644 index 000000000000..441785586cce --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tasks.json @@ -0,0 +1,78 @@ +{ + "$schema": "http://json-schema.org/schema#", + "properties": { + "assigneeId": { + "type": ["null", "string"] + }, + "assigneeIds": { + "items": { + "type": "string" + }, + "type": "array" + }, + "billable": { + "type": "boolean" + }, + "costRate": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string" + }, + { + "properties": { + "amount": { + "type": ["null", "string", "integer"] + }, + "currency": { + "type": ["null", "string"] + } + }, + "type": "object" + } + ] + }, + "duration": { + "type": ["null", "string"] + }, + "estimate": { + "type": "string" + }, + "hourlyRate": { + "anyOf": [ + { + "type": "null" + }, + { + "properties": { + "amount": { + "type": "integer" + }, + "currency": { + "type": "string" + } + }, + "type": "object" + } + ] + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "projectId": { + "type": "string" + }, + "status": { + "type": "string" + }, + "userGroupIds": { + "type": "array" + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/time_entries.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/time_entries.json new file mode 100644 index 000000000000..8fb163736583 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/time_entries.json @@ -0,0 +1,66 @@ +{ + "$schema": "http://json-schema.org/schema#", + "properties": { + "billable": { + "type": "boolean" + }, + "customFieldValues": { + "type": "array" + }, + "description": { + "type": "string" + }, + "id": { + "type": "string" + }, + "isLocked": { + "type": "boolean" + }, + "kioskId": { + "type": ["null", "string"] + }, + "projectId": { + "type": ["null", "string"] + }, + "tagIds": { + "anyOf": [ + { + "type": "null" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ] + }, + "taskId": { + "type": ["null", "string"] + }, + "timeInterval": { + "properties": { + "duration": { + "type": "string" + }, + "end": { + "type": "string" + }, + "start": { + "type": "string" + } + }, + "type": "object" + }, + "type": { + "type": "string" + }, + "userId": { + "type": "string" + }, + "workspaceId": { + "type": "string" + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/user_groups.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/user_groups.json new file mode 100644 index 000000000000..f7ffaae51e6b --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/user_groups.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/schema#", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "userIds": { + "items": { + "type": "string" + }, + "type": "array" + }, + "workspaceId": { + "type": "string" + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/users.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/users.json new file mode 100644 index 000000000000..7d3d6d27cbd7 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/users.json @@ -0,0 +1,133 @@ +{ + "$schema": "http://json-schema.org/schema#", + "properties": { + "activeWorkspace": { + "type": "string" + }, + "customFields": { + "type": "array" + }, + "defaultWorkspace": { + "type": "string" + }, + "email": { + "type": "string" + }, + "id": { + "type": "string" + }, + "memberships": { + "type": "array" + }, + "name": { + "type": "string" + }, + "profilePicture": { + "type": "string" + }, + "settings": { + "properties": { + "alerts": { + "type": "boolean" + }, + "approval": { + "type": "boolean" + }, + "collapseAllProjectLists": { + "type": "boolean" + }, + "dashboardPinToTop": { + "type": "boolean" + }, + "dashboardSelection": { + "type": "string" + }, + "dashboardViewType": { + "type": "string" + }, + "dateFormat": { + "type": "string" + }, + "groupSimilarEntriesDisabled": { + "type": "boolean" + }, + "isCompactViewOn": { + "type": "boolean" + }, + "lang": { + "type": "string" + }, + "longRunning": { + "type": "boolean" + }, + "multiFactorEnabled": { + "type": "boolean" + }, + "myStartOfDay": { + "type": "string" + }, + "onboarding": { + "type": "boolean" + }, + "projectListCollapse": { + "type": "integer" + }, + "projectPickerTaskFilter": { + "type": "boolean" + }, + "pto": { + "type": "boolean" + }, + "reminders": { + "type": "boolean" + }, + "scheduledReports": { + "type": "boolean" + }, + "scheduling": { + "type": "boolean" + }, + "sendNewsletter": { + "type": "boolean" + }, + "showOnlyWorkingDays": { + "type": "boolean" + }, + "summaryReportSettings": { + "properties": { + "group": { + "type": "string" + }, + "subgroup": { + "type": "string" + } + }, + "type": "object" + }, + "theme": { + "type": "string" + }, + "timeFormat": { + "type": "string" + }, + "timeTrackingManual": { + "type": "boolean" + }, + "timeZone": { + "type": "string" + }, + "weekStart": { + "type": "string" + }, + "weeklyUpdates": { + "type": "boolean" + } + }, + "type": "object" + }, + "status": { + "type": "string" + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/source.py b/airbyte-integrations/connectors/source-clockify/source_clockify/source.py new file mode 100644 index 000000000000..b4bcd649b241 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/source.py @@ -0,0 +1,33 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from typing import Any, List, Mapping, Tuple + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator + +from .streams import Clients, Projects, Tags, Tasks, TimeEntries, UserGroups, Users + + +# Source +class SourceClockify(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + try: + workspace_stream = Users( + authenticator=TokenAuthenticator(token=config["api_key"], auth_header="X-Api-Key", auth_method=""), + workspace_id=config["workspace_id"], + ) + next(workspace_stream.read_records(sync_mode=SyncMode.full_refresh)) + return True, None + except Exception as e: + return False, f"Please check that your API key and workspace id are entered correctly: {repr(e)}" + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + authenticator = TokenAuthenticator(token=config["api_key"], auth_header="X-Api-Key", auth_method="") + + args = {"authenticator": authenticator, "workspace_id": config["workspace_id"]} + + return [Users(**args), Projects(**args), Clients(**args), Tags(**args), UserGroups(**args), TimeEntries(**args), Tasks(**args)] diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/spec.json b/airbyte-integrations/connectors/source-clockify/source_clockify/spec.json new file mode 100644 index 000000000000..ecd182c8e160 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/spec.json @@ -0,0 +1,23 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/clockify", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Clockify Spec", + "type": "object", + "required": ["workspace_id", "api_key"], + "additionalProperties": true, + "properties": { + "workspace_id": { + "title": "Workspace Id", + "description": "WorkSpace Id", + "type": "string" + }, + "api_key": { + "title": "API Key", + "description": "You can get your api access_key here This API is Case Sensitive.", + "type": "string", + "airbyte_secret": true + } + } + } +} diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/streams.py b/airbyte-integrations/connectors/source-clockify/source_clockify/streams.py new file mode 100644 index 000000000000..4fb36a3977fa --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/streams.py @@ -0,0 +1,123 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, Mapping, MutableMapping, Optional + +import requests +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream +from requests.auth import AuthBase + + +class ClockifyStream(HttpStream, ABC): + url_base = "https://api.clockify.me/api/v1/" + page_size = 50 + page = 1 + primary_key = None + + def __init__(self, workspace_id: str, **kwargs): + super().__init__(**kwargs) + self.workspace_id = workspace_id + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + next_page = response.json() + self.page = self.page + 1 + if next_page: + return {"page": self.page} + + def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = { + "page-size": self.page_size, + } + + if next_page_token: + params.update(next_page_token) + + return params + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + yield from response.json() + + +class Users(ClockifyStream): + @property + def use_cache(self) -> bool: + return True + + def path(self, **kwargs) -> str: + return f"workspaces/{self.workspace_id}/users" + + +class Projects(ClockifyStream): + @property + def use_cache(self) -> bool: + return True + + def path(self, **kwargs) -> str: + return f"workspaces/{self.workspace_id}/projects" + + +class Clients(ClockifyStream): + def path(self, **kwargs) -> str: + return f"workspaces/{self.workspace_id}/clients" + + +class Tags(ClockifyStream): + def path(self, **kwargs) -> str: + return f"workspaces/{self.workspace_id}/tags" + + +class UserGroups(ClockifyStream): + def path(self, **kwargs) -> str: + return f"workspaces/{self.workspace_id}/user-groups" + + +class TimeEntries(HttpSubStream, ClockifyStream): + def __init__(self, authenticator: AuthBase, workspace_id: Mapping[str, Any], **kwargs): + super().__init__( + authenticator=authenticator, + workspace_id=workspace_id, + parent=Users(authenticator=authenticator, workspace_id=workspace_id, **kwargs), + ) + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + """ + self.authenticator (which should be used as the + authenticator for Users) is object of NoAuth() + + so self._session.auth is used instead + """ + users_stream = Users(authenticator=self._session.auth, workspace_id=self.workspace_id) + for user in users_stream.read_records(sync_mode=SyncMode.full_refresh): + yield {"user_id": user["id"]} + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + user_id = stream_slice["user_id"] + return f"workspaces/{self.workspace_id}/user/{user_id}/time-entries" + + +class Tasks(HttpSubStream, ClockifyStream): + def __init__(self, authenticator: AuthBase, workspace_id: Mapping[str, Any], **kwargs): + super().__init__( + authenticator=authenticator, + workspace_id=workspace_id, + parent=Projects(authenticator=authenticator, workspace_id=workspace_id, **kwargs), + ) + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + """ + self.authenticator (which should be used as the + authenticator for Projects) is object of NoAuth() + + so self._session.auth is used instead + """ + projects_stream = Projects(authenticator=self._session.auth, workspace_id=self.workspace_id) + for project in projects_stream.read_records(sync_mode=SyncMode.full_refresh): + yield {"project_id": project["id"]} + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + project_id = stream_slice["project_id"] + return f"workspaces/{self.workspace_id}/projects/{project_id}/tasks" diff --git a/airbyte-integrations/connectors/source-clockify/unit_tests/__init__.py b/airbyte-integrations/connectors/source-clockify/unit_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-clockify/unit_tests/conftest.py b/airbyte-integrations/connectors/source-clockify/unit_tests/conftest.py new file mode 100644 index 000000000000..ae1ad3fd9d3d --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/unit_tests/conftest.py @@ -0,0 +1,10 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import pytest + + +@pytest.fixture(scope="session", name="config") +def config_fixture(): + return {"api_key": "test_api_key", "workspace_id": "workspace_id"} diff --git a/airbyte-integrations/connectors/source-clockify/unit_tests/test_source.py b/airbyte-integrations/connectors/source-clockify/unit_tests/test_source.py new file mode 100644 index 000000000000..3f6200a9ef68 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/unit_tests/test_source.py @@ -0,0 +1,33 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +import responses +from source_clockify.source import SourceClockify + + +def setup_responses(): + responses.add( + responses.GET, + "https://api.clockify.me/api/v1/workspaces/workspace_id/users", + json={"access_token": "test_api_key", "expires_in": 3600}, + ) + + +@responses.activate +def test_check_connection(config): + setup_responses() + source = SourceClockify() + logger_mock = MagicMock() + assert source.check_connection(logger_mock, config) == (True, None) + + +def test_streams(mocker): + source = SourceClockify() + config_mock = MagicMock() + streams = source.streams(config_mock) + + expected_streams_number = 7 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-clockify/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-clockify/unit_tests/test_streams.py new file mode 100644 index 000000000000..e64e95119dc0 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/unit_tests/test_streams.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +import pytest +from airbyte_cdk.models import SyncMode +from source_clockify.streams import ClockifyStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(ClockifyStream, "path", "v0/example_endpoint") + mocker.patch.object(ClockifyStream, "primary_key", "test_primary_key") + mocker.patch.object(ClockifyStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = ClockifyStream(workspace_id=MagicMock()) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {"page-size": 50} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = ClockifyStream(workspace_id=MagicMock()) + inputs = {"response": MagicMock()} + expected_token = {"page": 2} + assert stream.next_page_token(**inputs) == expected_token + + +def test_read_records(patch_base_class): + stream = ClockifyStream(workspace_id=MagicMock()) + assert stream.read_records(sync_mode=SyncMode.full_refresh) + + +def test_request_headers(patch_base_class): + stream = ClockifyStream(workspace_id=MagicMock()) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = ClockifyStream(workspace_id=MagicMock()) + expected_method = "GET" + assert stream.http_method == expected_method diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py index d0b8b899456a..00315a9bc5fa 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py @@ -4,7 +4,7 @@ import random from typing import Any, MutableMapping -from unittest.mock import patch, PropertyMock +from unittest.mock import PropertyMock, patch import pytest from airbyte_cdk.models import SyncMode @@ -126,7 +126,7 @@ def test_incremental(stream, resource, authenticator, config, requests_mock): highest_updated_at = "2022-04-25T22:00:00Z" other_updated_at = "2022-04-01T00:00:00Z" highest_index = random.randint(0, 24) - with patch(f'source_freshdesk.streams.{stream.__name__}.use_cache', new_callable=PropertyMock, return_value=False): + with patch(f"source_freshdesk.streams.{stream.__name__}.use_cache", new_callable=PropertyMock, return_value=False): requests_mock.register_uri( "GET", f"/api/{resource}",