diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 5e137848006f..62daadc7e336 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1143,6 +1143,14 @@ icon: trelllo.svg sourceType: api releaseStage: alpha +- name: TVMaze Schedule + sourceDefinitionId: bd14b08f-9f43-400f-b2b6-7248b5c72561 + dockerRepository: airbyte/source-tvmaze-schedule + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/tvmaze-schedule + icon: trelllo.svg + sourceType: api + releaseStage: alpha - name: Twilio sourceDefinitionId: b9dc6155-672e-42ea-b10d-9f1f1fb95ab1 dockerRepository: airbyte/source-twilio diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 58510e415ca4..21ff56090cf0 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -11994,6 +11994,47 @@ oauthFlowOutputParameters: - - "token" - - "key" +- dockerImage: "airbyte/source-tvmaze-schedule:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/tvmaze-schedule" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "TVMaze Schedule Spec" + type: "object" + required: + - "start_date" + - "domestic_schedule_country_code" + additionalProperties: true + properties: + start_date: + type: "string" + description: "Start date for TV schedule retrieval. May be in the future." + order: 0 + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + end_date: + type: "string" + description: "End date for TV schedule retrieval. May be in the future.\ + \ Optional.\n" + order: 1 + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + domestic_schedule_country_code: + type: "string" + description: "Country code for domestic TV schedule retrieval." + examples: + - "US" + - "GB" + web_schedule_country_code: + type: "string" + description: "ISO 3166-1 country code for web TV schedule retrieval. Leave\ + \ blank for\nall countries plus global web channels (e.g. Netflix). Alternatively,\n\ + set to 'global' for just global web channels.\n" + examples: + - "US" + - "GB" + - "global" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-twilio:0.1.13" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/twilio" diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/.dockerignore b/airbyte-integrations/connectors/source-tvmaze-schedule/.dockerignore new file mode 100644 index 000000000000..2aaed5f638fb --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_tvmaze_schedule +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/Dockerfile b/airbyte-integrations/connectors/source-tvmaze-schedule/Dockerfile new file mode 100644 index 000000000000..4df013196a63 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_tvmaze_schedule ./source_tvmaze_schedule + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-tvmaze-schedule diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/README.md b/airbyte-integrations/connectors/source-tvmaze-schedule/README.md new file mode 100644 index 000000000000..8f6ee3c87b0c --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/README.md @@ -0,0 +1,79 @@ +# Tvmaze Schedule Source + +This is the repository for the Tvmaze Schedule configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/tvmaze-schedule). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-tvmaze-schedule:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/tvmaze-schedule) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_tvmaze_schedule/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source tvmaze-schedule test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-tvmaze-schedule:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-tvmaze-schedule:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-tvmaze-schedule:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-tvmaze-schedule:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-tvmaze-schedule:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-tvmaze-schedule:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-tvmaze-schedule:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-tvmaze-schedule:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/__init__.py b/airbyte-integrations/connectors/source-tvmaze-schedule/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/acceptance-test-config.yml b/airbyte-integrations/connectors/source-tvmaze-schedule/acceptance-test-config.yml new file mode 100644 index 000000000000..3a7509bbf359 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/acceptance-test-config.yml @@ -0,0 +1,20 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-tvmaze-schedule:dev +tests: + spec: + - spec_path: "source_tvmaze_schedule/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-tvmaze-schedule/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/build.gradle b/airbyte-integrations/connectors/source-tvmaze-schedule/build.gradle new file mode 100644 index 000000000000..0d46db9972fb --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_tvmaze_schedule' +} diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/__init__.py b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..3a09e12ca7c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/configured_catalog.json @@ -0,0 +1,31 @@ +{ + "streams": [ + { + "stream": { + "name": "domestic", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "web", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "future", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/invalid_config.json new file mode 100644 index 000000000000..8a0545197c81 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/invalid_config.json @@ -0,0 +1,6 @@ +{ + "start_date": "2022-10-20", + "end_date": "2022-10-20", + "domestic_schedule_country_code": "something-wrong", + "web_schedule_country_code": "something-wrong" +} diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/sample_config.json new file mode 100644 index 000000000000..c56b1884ef5d --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/sample_config.json @@ -0,0 +1,6 @@ +{ + "start_date": "2022-10-20", + "end_date": "2022-10-20", + "domestic_schedule_country_code": "GB", + "web_schedule_country_code": "NL" +} diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/main.py b/airbyte-integrations/connectors/source-tvmaze-schedule/main.py new file mode 100644 index 000000000000..7b7ed5efaffe --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_tvmaze_schedule import SourceTvmazeSchedule + +if __name__ == "__main__": + source = SourceTvmazeSchedule() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/requirements.txt b/airbyte-integrations/connectors/source-tvmaze-schedule/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/setup.py b/airbyte-integrations/connectors/source-tvmaze-schedule/setup.py new file mode 100644 index 000000000000..90cb9a01f789 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_tvmaze_schedule", + description="Source implementation for Tvmaze Schedule.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/__init__.py b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/__init__.py new file mode 100644 index 000000000000..21885552e59c --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceTvmazeSchedule + +__all__ = ["SourceTvmazeSchedule"] diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/schemas/domestic.json b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/schemas/domestic.json new file mode 100644 index 000000000000..f422bec2b627 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/schemas/domestic.json @@ -0,0 +1,218 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["number", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "season": { + "type": ["number", "null"] + }, + "number": { + "type": ["number", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "airdate": { + "type": ["string", "null"] + }, + "airtime": { + "type": ["string", "null"] + }, + "airstamp": { + "type": ["string", "null"] + }, + "runtime": { + "type": ["number", "null"] + }, + "rating": { + "type": "object", + "properties": { + "average": { + "type": ["number", "null"] + } + } + }, + "image": { + "type": ["object", "null"], + "properties": { + "medium": { + "type": ["string", "null"] + }, + "original": { + "type": ["string", "null"] + } + } + }, + "summary": { + "type": ["string", "null"] + }, + "show": { + "type": "object", + "properties": { + "id": { + "type": ["number", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "language": { + "type": ["string", "null"] + }, + "genres": { + "type": "array", + "items": { + "type": ["string", "null"] + } + }, + "status": { + "type": ["string", "null"] + }, + "runtime": { + "type": ["number", "null"] + }, + "averageRuntime": { + "type": ["number", "null"] + }, + "premiered": { + "type": ["string", "null"] + }, + "ended": { + "type": ["string", "null"] + }, + "officialSite": { "type": ["string", "null"] }, + "schedule": { + "type": "object", + "properties": { + "time": { + "type": ["string", "null"] + }, + "days": { + "type": "array", + "items": { + "type": ["string", "null"] + } + } + } + }, + "rating": { + "type": "object", + "properties": { + "average": { + "type": ["number", "null"] + } + } + }, + "weight": { + "type": ["number", "null"] + }, + "network": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["number", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "country": { + "type": "object", + "properties": { + "name": { + "type": ["string", "null"] + }, + "code": { + "type": ["string", "null"] + }, + "timezone": { + "type": ["string", "null"] + } + } + }, + "officialSite": {} + } + }, + "webChannel": {}, + "dvdCountry": {}, + "externals": { + "type": "object", + "properties": { + "tvrage": { + "type": ["number", "null"] + }, + "thetvdb": { + "type": ["number", "null"] + }, + "imdb": { + "type": ["string", "null"] + } + } + }, + "image": { + "type": ["object", "null"], + "properties": { + "medium": { + "type": ["string", "null"] + }, + "original": { + "type": ["string", "null"] + } + } + }, + "summary": { + "type": ["string", "null"] + }, + "updated": { + "type": ["number", "null"] + }, + "_links": { + "type": "object", + "properties": { + "self": { + "type": "object", + "properties": { + "href": { + "type": ["string", "null"] + } + } + }, + "previousepisode": { + "type": "object", + "properties": { + "href": { + "type": ["string", "null"] + } + } + } + } + } + } + }, + "_links": { + "type": "object", + "properties": { + "self": { + "type": "object", + "properties": { + "href": { + "type": ["string", "null"] + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/schemas/future.json b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/schemas/future.json new file mode 100644 index 000000000000..568914ae1711 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/schemas/future.json @@ -0,0 +1,202 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["number", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "season": { + "type": ["number", "null"] + }, + "number": { + "type": ["number", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "airdate": { + "type": ["string", "null"] + }, + "airtime": { + "type": ["string", "null"] + }, + "airstamp": { + "type": ["string", "null"] + }, + "runtime": { + "type": ["number", "null"] + }, + "rating": { + "type": "object", + "properties": { + "average": {} + } + }, + "image": {}, + "summary": { + "type": ["string", "null"] + }, + "show": { + "type": "object", + "properties": { + "id": { + "type": ["number", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "language": { + "type": ["string", "null"] + }, + "genres": { + "type": "array", + "items": { + "type": ["string", "null"] + } + }, + "status": { + "type": ["string", "null"] + }, + "runtime": { + "type": ["number", "null"] + }, + "averageRuntime": { + "type": ["number", "null"] + }, + "premiered": { + "type": ["string", "null"] + }, + "ended": { + "type": ["string", "null"] + }, + "officialSite": {}, + "schedule": { + "type": "object", + "properties": { + "time": { + "type": ["string", "null"] + }, + "days": { + "type": "array", + "items": { + "type": ["string", "null"] + } + } + } + }, + "rating": { + "type": "object", + "properties": { + "average": {} + } + }, + "weight": { + "type": ["number", "null"] + }, + "network": { + "type": "object", + "properties": { + "id": { + "type": ["number", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "country": { + "type": "object", + "properties": { + "name": { + "type": ["string", "null"] + }, + "code": { + "type": ["string", "null"] + }, + "timezone": { + "type": ["string", "null"] + } + } + }, + "officialSite": {} + } + }, + "webChannel": {}, + "dvdCountry": {}, + "externals": { + "type": "object", + "properties": { + "tvrage": {}, + "thetvdb": { + "type": ["number", "null"] + }, + "imdb": { + "type": ["string", "null"] + } + } + }, + "image": { + "type": "object", + "properties": { + "medium": { + "type": ["string", "null"] + }, + "original": { + "type": ["string", "null"] + } + } + }, + "summary": { + "type": ["string", "null"] + }, + "updated": { + "type": ["number", "null"] + }, + "_links": { + "type": "object", + "properties": { + "self": { + "type": "object", + "properties": { + "href": { + "type": ["string", "null"] + } + } + }, + "previousepisode": { + "type": "object", + "properties": { + "href": { + "type": ["string", "null"] + } + } + } + } + } + } + }, + "_links": { + "type": "object", + "properties": { + "self": { + "type": "object", + "properties": { + "href": { + "type": ["string", "null"] + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/schemas/web.json b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/schemas/web.json new file mode 100644 index 000000000000..4be56bdc5ca7 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/schemas/web.json @@ -0,0 +1,208 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["number", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "season": { + "type": ["number", "null"] + }, + "number": { + "type": ["number", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "airdate": { + "type": ["string", "null"] + }, + "airtime": { + "type": ["string", "null"] + }, + "airstamp": { + "type": ["string", "null"] + }, + "runtime": { + "type": ["number", "null"] + }, + "rating": { + "type": "object", + "properties": { + "average": { + "type": ["number", "null"] + } + } + }, + "image": { + "type": ["object", "null"] + }, + "summary": { + "type": ["string", "null"] + }, + "show": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["number", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "language": { + "type": ["string", "null"] + }, + "genres": { + "type": "array", + "items": { + "type": ["string", "null"] + } + }, + "status": { + "type": ["string", "null"] + }, + "runtime": { + "type": ["number", "null"] + }, + "averageRuntime": { + "type": ["number", "null"] + }, + "premiered": { + "type": ["string", "null"] + }, + "ended": { + "type": ["string", "null"] + }, + "officialSite": {}, + "schedule": { + "type": "object", + "properties": { + "time": { + "type": ["string", "null"] + }, + "days": { + "type": "array", + "items": { + "type": ["string", "null"] + } + } + } + }, + "rating": { + "type": "object", + "properties": { + "average": { + "type": ["number", "null"] + } + } + }, + "weight": { + "type": ["number", "null"] + }, + "network": { + "type": "object", + "properties": { + "id": { + "type": ["number", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "country": { + "type": "object", + "properties": { + "name": { + "type": ["string", "null"] + }, + "code": { + "type": ["string", "null"] + }, + "timezone": { + "type": ["string", "null"] + } + } + }, + "officialSite": {} + } + }, + "webChannel": {}, + "dvdCountry": {}, + "externals": { + "type": "object", + "properties": { + "tvrage": {}, + "thetvdb": { + "type": ["number", "null"] + }, + "imdb": { + "type": ["string", "null"] + } + } + }, + "image": { + "type": ["object", "null"], + "properties": { + "medium": { + "type": ["string", "null"] + }, + "original": { + "type": ["string", "null"] + } + } + }, + "summary": { + "type": ["string", "null"] + }, + "updated": { + "type": ["number", "null"] + }, + "_links": { + "type": "object", + "properties": { + "self": { + "type": "object", + "properties": { + "href": { + "type": ["string", "null"] + } + } + }, + "previousepisode": { + "type": "object", + "properties": { + "href": { + "type": ["string", "null"] + } + } + } + } + } + } + }, + "_links": { + "type": "object", + "properties": { + "self": { + "type": "object", + "properties": { + "href": { + "type": ["string", "null"] + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/source.py b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/source.py new file mode 100644 index 000000000000..49dc934451b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceTvmazeSchedule(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "tvmaze_schedule.yaml"}) diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/spec.yaml b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/spec.yaml new file mode 100644 index 000000000000..f53564a37652 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/spec.yaml @@ -0,0 +1,37 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/tvmaze-schedule +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: TVMaze Schedule Spec + type: object + required: + - start_date + - domestic_schedule_country_code + additionalProperties: true + properties: + start_date: + type: string + description: Start date for TV schedule retrieval. May be in the future. + order: 0 + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + end_date: + type: string + description: | + End date for TV schedule retrieval. May be in the future. Optional. + order: 1 + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + domestic_schedule_country_code: + type: string + description: Country code for domestic TV schedule retrieval. + examples: + - US + - GB + web_schedule_country_code: + type: string + description: | + ISO 3166-1 country code for web TV schedule retrieval. Leave blank for + all countries plus global web channels (e.g. Netflix). Alternatively, + set to 'global' for just global web channels. + examples: + - US + - GB + - global diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/tvmaze_schedule.yaml b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/tvmaze_schedule.yaml new file mode 100644 index 000000000000..0b39d36dece0 --- /dev/null +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/source_tvmaze_schedule/tvmaze_schedule.yaml @@ -0,0 +1,72 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: [] + requester: + url_base: "https://api.tvmaze.com" + http_method: "GET" + request_options_provider: + request_parameters: + country: | + {{ + config['domestic_schedule_country_code'] + if options['name'] == 'domestic' + else config['web_schedule_country_code'].replace('global', '') + if options['name'] == 'web' + else '' + }} + stream_slicer: + type: DatetimeStreamSlicer + start_datetime: + datetime: "{{ config['start_date'] }}" + format: "%Y-%m-%d" + end_datetime: + datetime: "{{ config['end_date'] or today_utc() }}" + format: "%Y-%m-%d" + step: 1d + datetime_format: "%Y-%m-%d" + cursor_field: "airdate" + start_time_option: + field_name: "date" + inject_into: "request_parameter" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + requester: + $ref: "*ref(definitions.requester)" + stream_slicer: + $ref: "*ref(definitions.stream_slicer)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + domestic_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "domestic" + primary_key: "id" + path: "/schedule" + web_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "web" + primary_key: "id" + path: "/schedule" + future_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "future" + primary_key: "id" + path: "/schedule/full" + +streams: + - "*ref(definitions.domestic_stream)" + - "*ref(definitions.web_stream)" + - "*ref(definitions.future_stream)" + +check: + stream_names: + - "domestic" diff --git a/docs/integrations/sources/tvmaze-schedule.md b/docs/integrations/sources/tvmaze-schedule.md new file mode 100644 index 000000000000..a46aa435bb0b --- /dev/null +++ b/docs/integrations/sources/tvmaze-schedule.md @@ -0,0 +1,52 @@ +# TVMaze Schedule + +## Sync overview + +This source retrieves historical and future TV scheduling data using the +[TVMaze](https://www.tvmaze.com/) schedule API. + + +### Output schema + +This source is capable of syncing the following streams: + +* `domestic` +* `web` +* `future` + +### Features + +| Feature | Supported? \(Yes/No\) | Notes | +|:------------------|:----------------------|:------| +| Full Refresh Sync | Yes | | +| Incremental Sync | No | | + +### Performance considerations + +TVMaze has a rate limit of 20 requests per 10 seconds. This source should not +run into this limit. + +## Getting started + +### Requirements + +1. Choose a start date for your sync. This may be in the future. +2. Choose an ISO 3166-1 country code for domestic schedule syncs. + +### Setup guide + +The following fields are required fields for the connector to work: + +- `start_date`: The start date to pull `history` data from. +- (optional) `end_date`: The end date to pull `history` data until. +- `domestic_schedule_country_code`: The ISO 3166-1 country code to pull domestic + schedule data for. +- (optional) `web_schedule_country_code`: The ISO 3166-1 country code to pull + web schedule data for. Can be left blank for all countries and global + channels, or set to 'global' for only global channels. + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-----------| +| 0.1.0 | 2022-10-22 | [18333](https://github.com/airbytehq/airbyte/pull/18333) | New source |