From 5ec28b3e92e1834d63e0a679919d9d9c9294917c Mon Sep 17 00:00:00 2001 From: sbrickel-nimble <113671803+sbrickel-nimble@users.noreply.github.com> Date: Fri, 28 Oct 2022 19:13:14 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20New=20Source:=20Waiteraid=20[low?= =?UTF-8?q?-code=20cdk]=20(#18165)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * added source waiteraid * 🎉 New Source: Waiteraid * add searchBookings stream * add P/R number * add SUMMARY entry * add docs/integrations/README.md entry * add builds.md entry * add docs to each endpoint * fix schema ~ * Update airbyte-integrations/builds.md added web address Co-authored-by: Marcos Marx <marcosmarxm@users.noreply.github.com> * Delete catalog.json * Delete TODO.md * Update spec.yaml * add waiteraid to source def seed * auto-bump connector version Co-authored-by: Sebastian Brickel <sebastianbrickel@Sebastians-MacBook-Air.local> Co-authored-by: Marcos Marx <marcosmarxm@users.noreply.github.com> Co-authored-by: marcosmarxm <marcosmarxm@gmail.com> Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com> --- .../resources/seed/source_definitions.yaml | 7 + .../src/main/resources/seed/source_specs.yaml | 34 ++++ airbyte-integrations/builds.md | 1 + .../connectors/source-waiteraid/.dockerignore | 6 + .../connectors/source-waiteraid/Dockerfile | 38 +++++ .../connectors/source-waiteraid/README.md | 129 +++++++++++++++ .../connectors/source-waiteraid/__init__.py | 3 + .../acceptance-test-config.yml | 30 ++++ .../acceptance-test-docker.sh | 16 ++ .../connectors/source-waiteraid/bootstrap.md | 11 ++ .../connectors/source-waiteraid/build.gradle | 9 ++ .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 16 ++ .../integration_tests/configured_catalog.json | 13 ++ .../integration_tests/invalid_config.json | 1 + .../integration_tests/sample_config.json | 3 + .../integration_tests/sample_state.json | 5 + .../connectors/source-waiteraid/main.py | 13 ++ .../source-waiteraid/requirements.txt | 2 + .../connectors/source-waiteraid/setup.py | 29 ++++ .../source_waiteraid/__init__.py | 8 + .../source_waiteraid/schemas/booking.json | 151 ++++++++++++++++++ .../source_waiteraid/source.py | 18 +++ .../source_waiteraid/spec.yaml | 28 ++++ .../source_waiteraid/waiteraid.yaml | 48 ++++++ docs/integrations/README.md | 1 + docs/integrations/sources/waiteraid.md | 57 +++++++ 28 files changed, 685 insertions(+) create mode 100644 airbyte-integrations/connectors/source-waiteraid/.dockerignore create mode 100644 airbyte-integrations/connectors/source-waiteraid/Dockerfile create mode 100644 airbyte-integrations/connectors/source-waiteraid/README.md create mode 100644 airbyte-integrations/connectors/source-waiteraid/__init__.py create mode 100644 airbyte-integrations/connectors/source-waiteraid/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-waiteraid/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-waiteraid/bootstrap.md create mode 100644 airbyte-integrations/connectors/source-waiteraid/build.gradle create mode 100644 airbyte-integrations/connectors/source-waiteraid/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-waiteraid/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-waiteraid/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-waiteraid/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-waiteraid/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-waiteraid/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-waiteraid/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-waiteraid/main.py create mode 100644 airbyte-integrations/connectors/source-waiteraid/requirements.txt create mode 100644 airbyte-integrations/connectors/source-waiteraid/setup.py create mode 100644 airbyte-integrations/connectors/source-waiteraid/source_waiteraid/__init__.py create mode 100644 airbyte-integrations/connectors/source-waiteraid/source_waiteraid/schemas/booking.json create mode 100644 airbyte-integrations/connectors/source-waiteraid/source_waiteraid/source.py create mode 100644 airbyte-integrations/connectors/source-waiteraid/source_waiteraid/spec.yaml create mode 100644 airbyte-integrations/connectors/source-waiteraid/source_waiteraid/waiteraid.yaml create mode 100644 docs/integrations/sources/waiteraid.md diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 37d226008ecd..33a126256226 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1413,6 +1413,13 @@ documentationUrl: https://docs.airbyte.com/integrations/sources/elasticsearch sourceType: api releaseStage: alpha +- name: Waiteraid + sourceDefinitionId: 03a53b13-794a-4d6b-8544-3b36ed8f3ce4 + dockerRepository: airbyte/source-waiteraid + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/waiteraid + sourceType: api + releaseStage: alpha - name: Yandex Metrica sourceDefinitionId: 7865dce4-2211-4f6a-88e5-9d0fe161afe7 dockerRepository: airbyte/source-yandex-metrica diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 28e09f19d102..de2d93a704ed 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -13861,6 +13861,40 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-waiteraid:0.1.0" + spec: + documentationUrl: "https://docsurl.com" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Waiteraid Spec" + type: "object" + required: + - "start_date" + - "auth_hash" + - "restid" + additionalProperties: true + properties: + start_date: + title: "Start Date" + type: "string" + description: "Start getting data from that date." + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + examples: + - "YYYY-MM-DD" + auth_hash: + title: "Authentication Hash" + type: "string" + description: "Your WaiterAid API key, obtained from API request with Username\ + \ and Password" + airbyte_secret: true + restid: + title: "Restaurant ID" + type: "string" + description: "Your WaiterAid restaurant id from API request to getRestaurants" + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-yandex-metrica:0.1.0" spec: documentationUrl: "https://docsurl.com" diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 1c8051245533..ab4c3417064d 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -121,6 +121,7 @@ | Twilio | [![source-twilio](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-twilio%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-twilio) | | Typeform | [![source-typeform](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-typeform%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-typeform) | | US Census | [![source-us-census](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-us-census%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-us-census) | +| Waiteraid | [![source-waiteraid]()](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-waiteraid%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-waiteraid) | | Whisky Hunter | [![source-whisky-hunter](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-whisky-hunter%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-whisky-hunter) | | Wrike | [![source-wrike](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-wrike%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-wrike) | | YouTube Analytics | [![source-youtube-analytics](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-youtube-analytics%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-youtube-analytics) | diff --git a/airbyte-integrations/connectors/source-waiteraid/.dockerignore b/airbyte-integrations/connectors/source-waiteraid/.dockerignore new file mode 100644 index 000000000000..a89f0645c28d --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_waiteraid +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-waiteraid/Dockerfile b/airbyte-integrations/connectors/source-waiteraid/Dockerfile new file mode 100644 index 000000000000..a0d4d3a3c919 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_waiteraid ./source_waiteraid + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-waiteraid diff --git a/airbyte-integrations/connectors/source-waiteraid/README.md b/airbyte-integrations/connectors/source-waiteraid/README.md new file mode 100644 index 000000000000..a4f376768025 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/README.md @@ -0,0 +1,129 @@ +# Waiteraid Source + +This is the repository for the Waiteraid configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/waiteraid). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-waiteraid:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/waiteraid) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_waiteraid/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source waiteraid test creds` +and place them into `secrets/config.json`. +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-waiteraid:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-waiteraid:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-waiteraid:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-waiteraid:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-waiteraid:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-waiteraid:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-waiteraid:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-waiteraid:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-waiteraid/__init__.py b/airbyte-integrations/connectors/source-waiteraid/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-waiteraid/acceptance-test-config.yml b/airbyte-integrations/connectors/source-waiteraid/acceptance-test-config.yml new file mode 100644 index 000000000000..c5d65168c7ef --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-waiteraid:dev +tests: + spec: + - spec_path: "source_waiteraid/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.txt" + # extra_fields: no + # exact_order: no + # extra_records: yes + #incremental: # TODO if your connector does not implement incremental sync, remove this block + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-waiteraid/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-waiteraid/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-waiteraid/bootstrap.md b/airbyte-integrations/connectors/source-waiteraid/bootstrap.md new file mode 100644 index 000000000000..a92233214a44 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/bootstrap.md @@ -0,0 +1,11 @@ +## Streams + +Waiteraid is a REST API. Connector has the following streams, and all of them support full refresh only. + +* [Bookings](https://app.waiteraid.com/api-docs/index.html#api_get_bookings) + +## Authentication +Waiteraid API offers two types of [authentication methods](https://app.waiteraid.com/api-docs/index.html#auth_call). + +* API Keys - Keys are passed using HTTP Basic auth. +* Username and Password - Not supported by this connector. diff --git a/airbyte-integrations/connectors/source-waiteraid/build.gradle b/airbyte-integrations/connectors/source-waiteraid/build.gradle new file mode 100644 index 000000000000..6ea94362d508 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_waiteraid' +} diff --git a/airbyte-integrations/connectors/source-waiteraid/integration_tests/__init__.py b/airbyte-integrations/connectors/source-waiteraid/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-waiteraid/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-waiteraid/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..e6ccaa81eced --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "booking": { + "date": "2999-12-31" + } +} diff --git a/airbyte-integrations/connectors/source-waiteraid/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-waiteraid/integration_tests/acceptance.py new file mode 100644 index 000000000000..1302b2f57e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-waiteraid/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-waiteraid/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..7b8975e6ceac --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/integration_tests/configured_catalog.json @@ -0,0 +1,13 @@ +{ + "streams": [ + { + "stream": { + "name": "booking", + "json_schema": {}, + "supported_sync_modes": ["full_refresh","incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-waiteraid/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-waiteraid/integration_tests/invalid_config.json new file mode 100644 index 000000000000..1f9795718747 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/integration_tests/invalid_config.json @@ -0,0 +1 @@ +{"start_date": "2022-09-01", "auth_hash": "1nval1dk3y", "restid": "666"} diff --git a/airbyte-integrations/connectors/source-waiteraid/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-waiteraid/integration_tests/sample_config.json new file mode 100644 index 000000000000..ecc4913b84c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-waiteraid/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-waiteraid/integration_tests/sample_state.json new file mode 100644 index 000000000000..49e5c722434b --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "booking": { + "date": "2022-10-01" + } +} diff --git a/airbyte-integrations/connectors/source-waiteraid/main.py b/airbyte-integrations/connectors/source-waiteraid/main.py new file mode 100644 index 000000000000..c176f331c485 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_waiteraid import SourceWaiteraid + +if __name__ == "__main__": + source = SourceWaiteraid() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-waiteraid/requirements.txt b/airbyte-integrations/connectors/source-waiteraid/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-waiteraid/setup.py b/airbyte-integrations/connectors/source-waiteraid/setup.py new file mode 100644 index 000000000000..bb67ca6ef48a --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_waiteraid", + description="Source implementation for Waiteraid.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/__init__.py b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/__init__.py new file mode 100644 index 000000000000..0730ddfa3771 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceWaiteraid + +__all__ = ["SourceWaiteraid"] diff --git a/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/schemas/booking.json b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/schemas/booking.json new file mode 100644 index 000000000000..ec922ebe34dd --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/schemas/booking.json @@ -0,0 +1,151 @@ +{ + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "amount": { + "type": ["null", "number"] + }, + "children_amount": { + "type": ["null", "number"] + }, + "placed": { + "type": ["null", "number"] + }, + "placed_manually": { + "type": ["null", "number"] + }, + "start": { + "type": ["null", "number"] + }, + "end": { + "type": ["null", "number"] + }, + "length": { + "type": ["null", "number"] + }, + "status": { + "type": ["null", "string"] + }, + "arrived": { + "type": ["null", "number"] + }, + "all_seated": { + "type": ["null", "number"] + }, + "guest_left": { + "type": ["null", "number"] + }, + "comment": { + "type": ["null", "string"] + }, + "confirmed": { + "type": ["null", "number"] + }, + "waitinbar": { + "type": ["null", "number"] + }, + "internet_booking": { + "type": ["null", "number"] + }, + "internet_booking_confirmed": { + "type": ["null", "number"] + }, + "paid": { + "type": ["null", "number"] + }, + "langid": { + "type": ["null", "number"] + }, + "meal": { + "type": ["null", "string"] + }, + "tables": { + "type": ["null", "number"] + }, + "meal_abbr": { + "type": ["null", "string"] + }, + "table_ids": { + "type": ["null", "number"] + }, + "products": { + "type": ["null", "number"] + }, + "waitinlist": { + "type": ["null", "number"] + }, + "date": { + "type": ["null", "string"] + }, + "time": { + "type": ["null", "string"] + }, + "guest": { + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "firstname": { + "type": ["null", "string"] + }, + "lastname": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "postalcode": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "telephone": { + "type": ["null", "string"] + }, + "mobile": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "comment": { + "type": ["null", "string"] + }, + "other_contact": { + "type": ["null", "string"] + }, + "restaurant_newsletter": { + "type": ["null", "boolean"] + } + } + }, + "booking_date": { + "type": ["null", "number"] + }, + "payStarted": { + "type": ["null", "boolean"] + }, + "payClosed": { + "type": ["null", "boolean"] + }, + "payCharged": { + "type": ["null", "boolean"] + }, + "payActivated": { + "type": ["null", "boolean"] + }, + "has_message": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/source.py b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/source.py new file mode 100644 index 000000000000..7e2b4027a8ed --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceWaiteraid(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "waiteraid.yaml"}) diff --git a/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/spec.yaml b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/spec.yaml new file mode 100644 index 000000000000..7f00092a24d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/spec.yaml @@ -0,0 +1,28 @@ +documentationUrl: https://docsurl.com +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Waiteraid Spec + type: object + required: + - start_date + - auth_hash + - restid + additionalProperties: true + properties: + start_date: + title: Start Date + type: string + description: Start getting data from that date. + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + examples: + - YYYY-MM-DD + auth_hash: + title: Authentication Hash + type: string + description: Your WaiterAid API key, obtained from API request with Username and Password + airbyte_secret: true + restid: + title: Restaurant ID + type: string + description: Your WaiterAid restaurant id from API request to getRestaurants + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/waiteraid.yaml b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/waiteraid.yaml new file mode 100644 index 000000000000..123c84e0f031 --- /dev/null +++ b/airbyte-integrations/connectors/source-waiteraid/source_waiteraid/waiteraid.yaml @@ -0,0 +1,48 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: [] + requester: + url_base: "https://app.waiteraid.com" + http_method: "POST" + request_options_provider: + request_parameters: + date: "{{ config['start_date'] }}" + auth_hash: "{{ config['auth_hash'] }}" + restid: "{{ config['restid'] }}" + stream_slicer: + type: "DatetimeStreamSlicer" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%d" + end_datetime: + datetime: "{{ now_utc() }}" + datetime_format: "%Y-%m-%d %H:%M:%S.%f" + step: "1d" + datetime_format: "%Y-%m-%d" + cursor_field: "{{ options['stream_cursor_field'] }}" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + requester: + $ref: "*ref(definitions.requester)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + booking_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "booking" + path: "/wa-api/searchBooking" + stream_cursor_field: "date" + +streams: + - "*ref(definitions.booking_stream)" + +check: + stream_names: + - "booking" diff --git a/docs/integrations/README.md b/docs/integrations/README.md index a67b98dc9590..91cb1d75cb6b 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -181,6 +181,7 @@ For more information about the grading system, see [Product Release Stages](http | [Typeform](sources/typeform.md) | Alpha | Yes | | [US Census](sources/us-census.md) | Alpha | Yes | | [VictorOps](sources/victorops.md) | Alpha | No | +| [Waiteraid](sources/waiteraid.md) | Alpha | Yes | | [Webflow](sources/webflow.md ) | Alpha | Yes | | [Whisky Hunter](sources/whisky-hunter.md ) | Alpha | No | | [WooCommerce](sources/woocommerce.md) | Alpha | No | diff --git a/docs/integrations/sources/waiteraid.md b/docs/integrations/sources/waiteraid.md new file mode 100644 index 000000000000..7b0411ac7fa2 --- /dev/null +++ b/docs/integrations/sources/waiteraid.md @@ -0,0 +1,57 @@ +# Waiteraid + +This page contains the setup guide and reference information for the Waiteraid source connector. + +## Prerequisites + +You can find or create authentication tokens within [Waiteraid](https://app.waiteraid.com/api-docs/index.html#auth_call). + +## Setup guide +## Step 1: Set up the Waiteraid connector in Airbyte + +### For Airbyte Cloud: + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. +3. On the Set up the source page, enter the name for the Waiteraid connector and select **Waiteraid** from the Source type dropdown. +4. Enter your `auth_token` - Waiteraid Authentication Token. +5. Enter your `restaurant ID` - The Waiteraid ID of the Restaurant you wanto sync. +6. Click **Set up source**. + +### For Airbyte OSS: + +1. Navigate to the Airbyte Open Source dashboard. +2. Set the name for your source. +3. Enter your `auth_token` - Waiteraid Authentication Token. +4. Enter your `restaurant ID` - The Waiteraid ID of the Restaurant you wanto sync. +5. Click **Set up source**. + +## Supported sync modes + +The Waiteraid source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + +| Feature | Supported? | +| :---------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental Sync | No | +| SSL connection | No | +| Namespaces | No | + +## Supported Streams + +* [Bookings](https://app.waiteraid.com/api-docs/index.html#api_get_bookings) + +## Data type map + +| Integration Type | Airbyte Type | +| :------------------ | :----------- | +| `string` | `string` | +| `integer`, `number` | `number` | +| `array` | `array` | +| `object` | `object` | + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------| +| 0.1.0 | 2022-10-QQ | [QQQQ](https://github.com/airbytehq/airbyte/pull/QQQQ) | New Source: Waiteraid |