From a6974a7ba93b7f1c62f17b37ca45a0bfcb456061 Mon Sep 17 00:00:00 2001 From: Xabier Lahuerta Vazquez Date: Fri, 28 Oct 2022 02:40:57 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20New=20Source:=20Nasa=20APOD=20[p?= =?UTF-8?q?ython=20cdk]=20(#18394)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * NASA APOD connector first implementation * Fix Acceptance and Unit tests * Fix DecrecationWarning on NoAuth * Update changelog with PR id * `date` input removal in favor of `start_date` + Minor fixes * Connector renaming to support different Nasa streams * Update docs and source definitions with connector rename * Support incremental updates * Fix typo on abnormal_state.json, making acceptance tests to fail * auto-bump connector version Co-authored-by: Octavia Squidington III --- .../init/src/main/resources/icons/nasa.svg | 52 +++++ .../resources/seed/source_definitions.yaml | 8 + .../src/main/resources/seed/source_specs.yaml | 54 +++++ .../connectors/source-nasa/.dockerignore | 6 + .../connectors/source-nasa/Dockerfile | 38 ++++ .../connectors/source-nasa/README.md | 132 +++++++++++ .../source-nasa/acceptance-test-config.yml | 29 +++ .../source-nasa/acceptance-test-docker.sh | 16 ++ .../connectors/source-nasa/build.gradle | 9 + .../source-nasa/integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 14 ++ .../source-nasa/integration_tests/config.json | 5 + .../integration_tests/configured_catalog.json | 26 +++ .../integration_tests/expected_records.txt | 6 + .../integration_tests/invalid_config.json | 4 + .../integration_tests/sample_config.json | 5 + .../integration_tests/sample_state.json | 5 + .../connectors/source-nasa/main.py | 13 ++ .../connectors/source-nasa/requirements.txt | 2 + .../connectors/source-nasa/setup.py | 29 +++ .../source-nasa/source_nasa/__init__.py | 8 + .../source_nasa/schemas/nasa_apod.json | 61 +++++ .../source-nasa/source_nasa/source.py | 209 ++++++++++++++++++ .../source-nasa/source_nasa/spec.yaml | 51 +++++ .../source-nasa/unit_tests/__init__.py | 3 + .../unit_tests/test_incremental_streams.py | 51 +++++ .../source-nasa/unit_tests/test_source.py | 58 +++++ .../source-nasa/unit_tests/test_streams.py | 80 +++++++ docs/integrations/README.md | 1 + docs/integrations/sources/nasa.md | 42 ++++ 31 files changed, 1025 insertions(+) create mode 100644 airbyte-config/init/src/main/resources/icons/nasa.svg create mode 100644 airbyte-integrations/connectors/source-nasa/.dockerignore create mode 100644 airbyte-integrations/connectors/source-nasa/Dockerfile create mode 100644 airbyte-integrations/connectors/source-nasa/README.md create mode 100644 airbyte-integrations/connectors/source-nasa/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-nasa/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-nasa/build.gradle create mode 100644 airbyte-integrations/connectors/source-nasa/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-nasa/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-nasa/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-nasa/integration_tests/config.json create mode 100644 airbyte-integrations/connectors/source-nasa/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-nasa/integration_tests/expected_records.txt create mode 100644 airbyte-integrations/connectors/source-nasa/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-nasa/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-nasa/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-nasa/main.py create mode 100644 airbyte-integrations/connectors/source-nasa/requirements.txt create mode 100644 airbyte-integrations/connectors/source-nasa/setup.py create mode 100644 airbyte-integrations/connectors/source-nasa/source_nasa/__init__.py create mode 100644 airbyte-integrations/connectors/source-nasa/source_nasa/schemas/nasa_apod.json create mode 100644 airbyte-integrations/connectors/source-nasa/source_nasa/source.py create mode 100644 airbyte-integrations/connectors/source-nasa/source_nasa/spec.yaml create mode 100644 airbyte-integrations/connectors/source-nasa/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-nasa/unit_tests/test_incremental_streams.py create mode 100644 airbyte-integrations/connectors/source-nasa/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-nasa/unit_tests/test_streams.py create mode 100644 docs/integrations/sources/nasa.md diff --git a/airbyte-config/init/src/main/resources/icons/nasa.svg b/airbyte-config/init/src/main/resources/icons/nasa.svg new file mode 100644 index 000000000000..e776a4be47d1 --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/nasa.svg @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index cca6dc9684b3..d41908063518 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -747,6 +747,14 @@ icon: mysql.svg sourceType: database releaseStage: beta +- name: NASA + sourceDefinitionId: 1a8667d7-7978-43cd-ba4d-d32cbd478971 + dockerRepository: airbyte/source-nasa + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/nasa + icon: nasa.svg + sourceType: api + releaseStage: alpha - name: Netsuite sourceDefinitionId: 4f2f093d-ce44-4121-8118-9d13b7bfccd0 dockerRepository: airbyte/source-netsuite diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index d421eddce129..7f4c930abb76 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -7726,6 +7726,60 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-nasa:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/nasa-apod" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "NASA spec" + type: "object" + required: + - "api_key" + properties: + api_key: + type: "string" + description: "API access key used to retrieve data from the NASA APOD API." + airbyte_secret: true + concept_tags: + type: "boolean" + default: false + description: "Indicates whether concept tags should be returned with the\ + \ rest of the response. The concept tags are not necessarily included\ + \ in the explanation, but rather derived from common search tags that\ + \ are associated with the description text. (Better than just pure text\ + \ search.) Defaults to False." + count: + type: "integer" + minimum: 1 + maximum: 100 + description: "A positive integer, no greater than 100. If this is specified\ + \ then `count` randomly chosen images will be returned in a JSON array.\ + \ Cannot be used in conjunction with `date` or `start_date` and `end_date`." + start_date: + type: "string" + description: "Indicates the start of a date range. All images in the range\ + \ from `start_date` to `end_date` will be returned in a JSON array. Must\ + \ be after 1995-06-16, the first day an APOD picture was posted. There\ + \ are no images for tomorrow available through this API." + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + examples: + - "2022-10-20" + end_date: + type: "string" + description: "Indicates that end of a date range. If `start_date` is specified\ + \ without an `end_date` then `end_date` defaults to the current date." + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + examples: + - "2022-10-20" + thumbs: + type: "boolean" + default: false + description: "Indicates whether the API should return a thumbnail image\ + \ URL for video files. If set to True, the API returns URL of video thumbnail.\ + \ If an APOD is not a video, this parameter is ignored." + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-netsuite:0.1.1" spec: documentationUrl: "https://docsurl.com" diff --git a/airbyte-integrations/connectors/source-nasa/.dockerignore b/airbyte-integrations/connectors/source-nasa/.dockerignore new file mode 100644 index 000000000000..9e72055452fa --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_nasa +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-nasa/Dockerfile b/airbyte-integrations/connectors/source-nasa/Dockerfile new file mode 100644 index 000000000000..82be10b93751 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.13-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_nasa ./source_nasa + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-nasa diff --git a/airbyte-integrations/connectors/source-nasa/README.md b/airbyte-integrations/connectors/source-nasa/README.md new file mode 100644 index 000000000000..532154929ece --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/README.md @@ -0,0 +1,132 @@ +# Nasa Source + +This is the repository for the Nasa source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/nasa). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-nasa:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/nasa) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_nasa/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source nasa test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-nasa:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-nasa:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-nasa:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-nasa:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-nasa:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-nasa:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-nasa:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-nasa:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-nasa/acceptance-test-config.yml b/airbyte-integrations/connectors/source-nasa/acceptance-test-config.yml new file mode 100644 index 000000000000..e9fdfb3e37ed --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/acceptance-test-config.yml @@ -0,0 +1,29 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-nasa:dev +tests: + spec: + - spec_path: "source_nasa/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "integration_tests/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + expect_records: + path: "integration_tests/expected_records.txt" + extra_fields: no + exact_order: no + extra_records: yes + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-nasa/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-nasa/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-nasa/build.gradle b/airbyte-integrations/connectors/source-nasa/build.gradle new file mode 100644 index 000000000000..84acad7f37c4 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_nasa' +} diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/__init__.py b/airbyte-integrations/connectors/source-nasa/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-nasa/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..a73baeb404d6 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "nasa_apod": { + "date": "9999-12-31" + } +} diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-nasa/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/config.json b/airbyte-integrations/connectors/source-nasa/integration_tests/config.json new file mode 100644 index 000000000000..1a58cb54281b --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/config.json @@ -0,0 +1,5 @@ +{ + "api_key": "DEMO_KEY", + "start_date": "2022-09-10", + "end_date": "2022-09-15" +} diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-nasa/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..caa03877beac --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/configured_catalog.json @@ -0,0 +1,26 @@ +{ + "streams": [ + { + "stream": { + "name": "nasa_apod", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "date" + ], + "source_defined_primary_key": [ + [ + "date" + ] + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + } + ] +} + diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-nasa/integration_tests/expected_records.txt new file mode 100644 index 000000000000..93467678bace --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/expected_records.txt @@ -0,0 +1,6 @@ +{"stream": "nasa_apod", "data": {"copyright": "Gerardo Ferrarino", "date": "2022-09-10", "explanation": "This 180 degree panoramic night skyscape captures our Milky Way Galaxy as it arcs above the horizon on a winter's night in August. Near midnight, the galactic center is close to the zenith with the clear waters of Lake Traful, Neuquen, Argentina, South America, planet Earth below. Zodiacal light, dust reflected sunlight along the Solar System's ecliptic plane, is also visible in the region's very dark night sky. The faint band of light reaches up from the distant snowy peaks toward the galaxy's center. Follow the arc of the Milky Way to the left to find the southern hemisphere stellar beacons Alpha and Beta Centauri. Close to the horizon bright star Vega is reflected in the calm mountain lake.", "hdurl": "https://apod.nasa.gov/apod/image/2209/Traful-Lake.jpg", "media_type": "image", "service_version": "v1", "title": "Galaxy by the Lake", "url": "https://apod.nasa.gov/apod/image/2209/Traful-Lake1024.jpg"}, "emitted_at": 1666637798520} +{"stream": "nasa_apod", "data": {"date": "2022-09-11", "explanation": "How does your favorite planet spin? Does it spin rapidly around a nearly vertical axis, or horizontally, or backwards? The featured video animates NASA images of all eight planets in our Solar System to show them spinning side-by-side for an easy comparison. In the time-lapse video, a day on Earth -- one Earth rotation -- takes just a few seconds. Jupiter rotates the fastest, while Venus spins not only the slowest (can you see it?), but backwards. The inner rocky planets across the top underwent dramatic spin-altering collisions during the early days of the Solar System. Why planets spin and tilt as they do remains a topic of research with much insight gained from modern computer modeling and the recent discovery and analysis of hundreds of exoplanets: planets orbiting other stars.", "media_type": "video", "service_version": "v1", "title": "Planets of the Solar System: Tilts and Spins", "url": "https://www.youtube.com/embed/my1euFQHH-o?rel=0"}, "emitted_at": 1666637798523} +{"stream": "nasa_apod", "data": {"copyright": "Daniel \u0160\u010derba", "date": "2022-09-12", "explanation": "What are those red filaments in the sky? They are a rarely seen form of lightning confirmed only about 35 years ago: red sprites. Research has shown that following a powerful positive cloud-to-ground lightning strike, red sprites may start as 100-meter balls of ionized air that shoot down from about 80-km high at 10 percent the speed of light. They are quickly followed by a group of upward streaking ionized balls. The featured image was taken late last month from the Jeseniky Mountains in northern Moravia in the Czech Republic. The distance to the red sprites is about 200 kilometers. Red sprites take only a fraction of a second to occur and are best seen when powerful thunderstorms are visible from the side. APOD in world languages: Arabic, Bulgarian, Catalan, Chinese (Beijing), Chinese (Taiwan), Croatian, Czech, Dutch, Farsi, French, French (Canada), German, Hebrew, Indonesian, Japanese, Korean, Montenegrin, Polish, Russian, Serbian, Slovenian, Spanish, Taiwanese, Turkish, and Ukrainian", "hdurl": "https://apod.nasa.gov/apod/image/2209/sprites_scerba_4240.jpg", "media_type": "image", "service_version": "v1", "title": "Red Sprite Lightning over the Czech Republic", "url": "https://apod.nasa.gov/apod/image/2209/sprites_scerba_960.jpg"}, "emitted_at": 1666637798524} +{"stream": "nasa_apod", "data": {"copyright": "Alan FriedmanAverted Imagination", "date": "2022-09-13", "explanation": "rlier this month, the Sun exhibited one of the longer filaments on record. Visible as the bright curving streak around the image center, the snaking filament's full extent was estimated to be over half of the Sun's radius -- more than 350,000 kilometers long. A filament is composed of hot gas held aloft by the Sun's magnetic field, so that viewed from the side it would appear as a raised prominence. A different, smaller prominence is simultaneously visible at the Sun's edge. The featured image is in false-color and color-inverted to highlight not only the filament but the Sun's carpet chromosphere. The bright dot on the upper right is actually a dark sunspot about the size of the Earth. Solar filaments typically last from hours to days, eventually collapsing to return hot plasma back to the Sun. Sometimes, though, they explode and expel particles into the Solar System, some of which trigger auroras on Earth. The pictured filament appeared in early September and continued to hold steady for about a week.", "hdurl": "https://apod.nasa.gov/apod/image/2209/SnakingFilament_Friedman_960.jpg", "media_type": "image", "service_version": "v1", "title": "A Long Snaking Filament on the Sun", "url": "https://apod.nasa.gov/apod/image/2209/SnakingFilament_Friedman_960.jpg"}, "emitted_at": 1666637798524} +{"stream": "nasa_apod", "data": {"copyright": "Jarmo Ruuth Text: Ata SarajediniFlorida Atlantic U.Astronomy Minute", "date": "2022-09-14", "explanation": "It is one of the largest nebulas on the sky -- why isn't it better known? Roughly the same angular size as the Andromeda Galaxy, the Great Lacerta Nebula can be found toward the constellation of the Lizard (Lacerta). The emission nebula is difficult to see with wide-field binoculars because it is so faint, but also usually difficult to see with a large telescope because it is so great in angle -- spanning about three degrees. The depth, breadth, waves, and beauty of the nebula -- cataloged as Sharpless 126 (Sh2-126) -- can best be seen and appreciated with a long duration camera exposure. The featured image is one such combined exposure -- in this case 10 hours over five different colors and over six nights during this past June and July at the IC Astronomy Observatory in Spain. The hydrogen gas in the Great Lacerta Nebula glows red because it is excited by light from the bright star 10 Lacertae, one of the bright blue stars just above the red-glowing nebula's center. The stars and nebula are about 1,200 light years distant. Harvest Full Moon 2022: Notable Submissions to APOD", "hdurl": "https://apod.nasa.gov/apod/image/2209/GreatLacerta_Ruuth_3719.jpg", "media_type": "image", "service_version": "v1", "title": "Waves of the Great Lacerta Nebula", "url": "https://apod.nasa.gov/apod/image/2209/GreatLacerta_Ruuth_960.jpg"}, "emitted_at": 1666637798524} +{"stream": "nasa_apod", "data": {"copyright": "Dario Giannobile", "date": "2022-09-15", "explanation": "For northern hemisphere dwellers, September's Full Moon was the Harvest Moon. Reflecting warm hues at sunset it rises over the historic town of Castiglione di Sicilia in this telephoto view from September 9. Famed in festival, story, and song Harvest Moon is just the traditional name of the full moon nearest the autumnal equinox. According to lore the name is a fitting one. Despite the diminishing daylight hours as the growing season drew to a close, farmers could harvest crops by the light of a full moon shining on from dusk to dawn. Harvest Full Moon 2022: Notable Submissions to APOD", "hdurl": "https://apod.nasa.gov/apod/image/2209/HarvestMoonCastiglioneSicilyLD.jpg", "media_type": "image", "service_version": "v1", "title": "Harvest Moon over Sicily", "url": "https://apod.nasa.gov/apod/image/2209/HarvestMoonCastiglioneSicily1024.jpg"}, "emitted_at": 1666637798525} diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-nasa/integration_tests/invalid_config.json new file mode 100644 index 000000000000..20e5c97d5ff1 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "DEMO_KEY", + "date": "xxx" +} diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-nasa/integration_tests/sample_config.json new file mode 100644 index 000000000000..2f3a9a920871 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "api_key": "DEMO_KEY", + "date": "2022-10-20", + "concept_tags": true +} diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-nasa/integration_tests/sample_state.json new file mode 100644 index 000000000000..6f6300ecb66b --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "nasa-apod": { + "date": "2022-10-15" + } +} diff --git a/airbyte-integrations/connectors/source-nasa/main.py b/airbyte-integrations/connectors/source-nasa/main.py new file mode 100644 index 000000000000..9ed7e7381e18 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_nasa import SourceNasa + +if __name__ == "__main__": + source = SourceNasa() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-nasa/requirements.txt b/airbyte-integrations/connectors/source-nasa/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-nasa/setup.py b/airbyte-integrations/connectors/source-nasa/setup.py new file mode 100644 index 000000000000..bde7a24e5245 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.2", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_nasa", + description="Source implementation for Nasa.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-nasa/source_nasa/__init__.py b/airbyte-integrations/connectors/source-nasa/source_nasa/__init__.py new file mode 100644 index 000000000000..6f155993533b --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/source_nasa/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceNasa + +__all__ = ["SourceNasa"] diff --git a/airbyte-integrations/connectors/source-nasa/source_nasa/schemas/nasa_apod.json b/airbyte-integrations/connectors/source-nasa/source_nasa/schemas/nasa_apod.json new file mode 100644 index 000000000000..fc6ee91c2b8f --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/source_nasa/schemas/nasa_apod.json @@ -0,0 +1,61 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "resource": { + "type": ["null", "object"], + "properties": { + "image_set": { + "type": ["null", "string"] + }, + "planet": { + "type": ["null", "string"] + } + } + }, + "concept_tags": { + "type": ["null", "boolean"] + }, + "title": { + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"], + "format": "%Y-%m-%d" + }, + "url": { + "type": ["null", "string"], + "format": "uri" + }, + "hdurl": { + "type": ["null", "string"], + "format": "uri" + }, + "media_type": { + "type": ["null", "string"], + "enum": ["image", "video"] + }, + "explanation": { + "type": ["null", "string"] + }, + "concepts": { + "type": ["null", "object", "string"], + "patternProperties": { + "^[0-9]+$": { + "type": ["null", "string"] + } + } + }, + "thumbnail_url": { + "type": ["null", "string"], + "format": "uri" + }, + "copyright": { + "type": ["null", "string"] + }, + "service_version": { + "type": ["null", "string"], + "pattern": "^v[0-9]$" + } + } +} diff --git a/airbyte-integrations/connectors/source-nasa/source_nasa/source.py b/airbyte-integrations/connectors/source-nasa/source_nasa/source.py new file mode 100644 index 000000000000..3f47309ab7c3 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/source_nasa/source.py @@ -0,0 +1,209 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from abc import ABC +from datetime import datetime, time, timedelta +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union + +import requests +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import IncrementalMixin, Stream +from airbyte_cdk.sources.streams.http import HttpStream + +date_format = "%Y-%m-%d" + + +class NasaStream(HttpStream, ABC): + + api_key = "api_key" + url_base = "https://api.nasa.gov/" + + def __init__(self, config: Mapping[str, any], **kwargs): + super().__init__() + self.config = config + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {self.api_key: self.config[self.api_key]} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + r = response.json() + if type(r) is dict: + yield r + else: # We got a list + yield from r + + +class NasaApod(NasaStream, IncrementalMixin): + + cursor_field = "date" + primary_key = "date" + start_date_key = "start_date" + end_date_key = "end_date" + + def __init__(self, config: Mapping[str, any], **kwargs): + super().__init__(config) + self.start_date = ( + datetime.strptime(config.pop(self.start_date_key), date_format) if self.start_date_key in config else datetime.now() + ) + self.end_date = datetime.strptime(config.pop(self.end_date_key), date_format) if self.end_date_key in config else datetime.now() + self.sync_mode = SyncMode.full_refresh + self._cursor_value = self.start_date + + @property + def state(self) -> Mapping[str, Any]: + return {self.cursor_field: self._cursor_value.strftime(date_format)} + + @state.setter + def state(self, value: Mapping[str, Any]): + self._cursor_value = datetime.strptime(value[self.cursor_field], date_format) + + def _chunk_date_range(self, start_date: datetime) -> List[Mapping[str, Any]]: + """ + Returns a list of each day between the start date and end date. + The return value is a list of dicts {'date': date_string}. + """ + dates = [] + while start_date <= self.end_date: + dates.append({self.cursor_field: start_date.strftime(date_format)}) + start_date += timedelta(days=1) + return dates + + def stream_slices( + self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + if ( + stream_state + and self.cursor_field in stream_state + and datetime.strptime(stream_state[self.cursor_field], date_format) > self.end_date + ): + return [] + if sync_mode == SyncMode.full_refresh: + return [self.start_date] + + start_date = ( + datetime.strptime(stream_state[self.cursor_field], date_format) + if stream_state and self.cursor_field in stream_state + else self.start_date + ) + return self._chunk_date_range(start_date) + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "planetary/apod" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + request_dict = {**self.config, **super().request_params(stream_state, stream_slice, next_page_token)} + if self.sync_mode == SyncMode.full_refresh: + request_dict[self.start_date_key] = self.start_date.strftime(date_format) + request_dict[self.end_date_key] = self.end_date.strftime(date_format) + else: + request_dict[self.primary_key] = stream_slice[self.cursor_field] + return request_dict + + def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: + self.sync_mode = kwargs.get("sync_mode", SyncMode.full_refresh) + if self._cursor_value and self._cursor_value > self.end_date: + yield [] + else: + for record in super().read_records(*args, **kwargs): + if self._cursor_value: + latest_record_date = datetime.strptime(record[self.cursor_field], date_format) + self._cursor_value = max(self._cursor_value, latest_record_date) + yield record + + +# Source +class SourceNasa(AbstractSource): + + count_key = "count" + start_date_key = "start_date" + end_date_key = "end_date" + min_count_value, max_count_value = 1, 101 + min_date = datetime.strptime("1995-06-16", date_format) + max_date = datetime.combine(datetime.today(), time(0, 0)) + timedelta(days=1) + invalid_conbination_message_template = "Invalid parameter combination. Cannot use {} and {} together." + invalid_parameter_value_template = "Invalid {} value: {}. {}." + invalid_parameter_value_range_template = "The value should be in the range [{},{})" + + def _parse_date(self, date_str: str) -> Union[datetime, str]: + """ + Parses the date string into a datetime object. + + :param date_str: string containing the date according to DATE_FORMAT + :return Union[datetime, str]: str if not correctly formatted or if it does not satify the constraints [self.MIN_DATE, self.MAX_DATE), datetime otherwise. + """ + try: + date = datetime.strptime(date_str, date_format) + if date < self.min_date or date >= self.max_date: + return self.invalid_parameter_value_template.format( + self.date_key, date_str, self.invalid_parameter_value_range_template.format(self.min_date, self.max_date) + ) + else: + return date + except ValueError: + return self.invalid_parameter_value_template.format(self.date_key, date_str, f"It should be formatted as '{date_format}'") + + def check_connection(self, logger, config) -> Tuple[bool, any]: + """ + Verifies that the input configuration supplied by the user can be used to connect to the underlying data source. + + :param config: the user-input config object conforming to the connector's spec.yaml + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + if self.start_date_key in config: + start_date = self._parse_date(config[self.start_date_key]) + if type(start_date) is not datetime: + return False, start_date + + if self.count_key in config: + return False, self.invalid_conbination_message_template.format(self.start_date_key, self.count_key) + + if self.end_date_key in config: + end_date = self._parse_date(config[self.end_date_key]) + if type(end_date) is not datetime: + return False, end_date + + if self.count_key in config: + return False, self.invalid_conbination_message_template.format(self.end_date_key, self.count_key) + + if self.start_date_key not in config: + return False, f"Cannot use {self.end_date_key} without specifying {self.start_date_key}." + + if start_date > end_date: + return False, f"Invalid values. start_date ({start_date}) needs to be lower than or equal to end_date ({end_date})." + + if self.count_key in config: + count_value = config[self.count_key] + if count_value < self.min_count_value or count_value >= self.max_count_value: + return False, self.invalid_parameter_value_template.format( + self.count_key, + count_value, + self.invalid_parameter_value_range_template.format(self.min_count_value, self.max_count_value), + ) + + try: + stream = NasaApod(authenticator=None, config=config) + records = stream.read_records(sync_mode=SyncMode.full_refresh) + next(records) + return True, None + except requests.exceptions.RequestException as e: + return False, e + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + return [NasaApod(authenticator=None, config=config)] diff --git a/airbyte-integrations/connectors/source-nasa/source_nasa/spec.yaml b/airbyte-integrations/connectors/source-nasa/source_nasa/spec.yaml new file mode 100644 index 000000000000..ace88f9ff111 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/source_nasa/spec.yaml @@ -0,0 +1,51 @@ +documentationUrl: https://docs.airbyte.io/integrations/sources/nasa-apod +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: NASA spec + type: object + required: + - api_key + properties: + api_key: + type: string + description: API access key used to retrieve data from the NASA APOD API. + airbyte_secret: true + concept_tags: + type: boolean + default: false + description: >- + Indicates whether concept tags should be returned with the rest of the response. + The concept tags are not necessarily included in the explanation, but rather derived + from common search tags that are associated with the description text. (Better than + just pure text search.) Defaults to False. + count: + type: integer + minimum: 1 + maximum: 100 + description: >- + A positive integer, no greater than 100. If this is specified then `count` randomly + chosen images will be returned in a JSON array. Cannot be used in conjunction with + `date` or `start_date` and `end_date`. + start_date: + type: string + description: >- + Indicates the start of a date range. All images in the range from `start_date` to + `end_date` will be returned in a JSON array. Must be after 1995-06-16, the first day + an APOD picture was posted. There are no images for tomorrow available through this API. + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + examples: + - "2022-10-20" + end_date: + type: string + description: >- + Indicates that end of a date range. If `start_date` is specified without an `end_date` + then `end_date` defaults to the current date. + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + examples: + - "2022-10-20" + thumbs: + type: boolean + default: false + description: >- + Indicates whether the API should return a thumbnail image URL for video files. If set to True, + the API returns URL of video thumbnail. If an APOD is not a video, this parameter is ignored. diff --git a/airbyte-integrations/connectors/source-nasa/unit_tests/__init__.py b/airbyte-integrations/connectors/source-nasa/unit_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-nasa/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-nasa/unit_tests/test_incremental_streams.py new file mode 100644 index 000000000000..be0de14c84bb --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/unit_tests/test_incremental_streams.py @@ -0,0 +1,51 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from datetime import datetime, timedelta + +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_nasa.source import NasaApod + +config = {"api_key": "foobar"} + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(NasaApod, "path", "v0/example_endpoint") + mocker.patch.object(NasaApod, "primary_key", "test_primary_key") + mocker.patch.object(NasaApod, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + stream = NasaApod(config=config) + expected_cursor_field = "date" + assert stream.cursor_field == expected_cursor_field + + +def test_stream_slices(patch_incremental_base_class): + stream = NasaApod(config=config) + start_date = datetime.now() - timedelta(days=3) + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": ["date"], "stream_state": {"date": start_date.strftime("%Y-%m-%d")}} + expected_stream_slice = [{"date": (start_date + timedelta(days=x)).strftime("%Y-%m-%d")} for x in range(4)] + assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + mocker.patch.object(NasaApod, "cursor_field", "dummy_field") + stream = NasaApod(config=config) + assert stream.supports_incremental + + +def test_source_defined_cursor(patch_incremental_base_class): + stream = NasaApod(config=config) + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(patch_incremental_base_class): + stream = NasaApod(config=config) + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-nasa/unit_tests/test_source.py b/airbyte-integrations/connectors/source-nasa/unit_tests/test_source.py new file mode 100644 index 000000000000..49c4570bd888 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/unit_tests/test_source.py @@ -0,0 +1,58 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from datetime import datetime, timedelta +from unittest.mock import MagicMock, patch + +import pytest +from source_nasa.source import NasaApod, SourceNasa + +date_format = "%Y-%m-%d" +min_date = datetime.strptime("1995-06-16", date_format) +tomorrow = SourceNasa().max_date +after_tomorrow_str = (tomorrow + timedelta(days=1)).strftime(date_format) +valid_date_str = (min_date + timedelta(days=10)).strftime(date_format) + + +@pytest.mark.parametrize( + ("config", "expected_return"), + [ + ({"api_key": "foobar"}, (True, None)), + ({"api_key": "foobar", "start_date": valid_date_str}, (True, None)), + ( + {"api_key": "foobar", "start_date": valid_date_str, "count": 5}, + (False, "Invalid parameter combination. Cannot use start_date and count together."), + ), + ( + {"api_key": "foobar", "end_date": valid_date_str, "count": 5}, + (False, "Invalid parameter combination. Cannot use end_date and count together."), + ), + ({"api_key": "foobar", "end_date": valid_date_str}, (False, "Cannot use end_date without specifying start_date.")), + ( + {"api_key": "foobar", "start_date": valid_date_str, "end_date": min_date.strftime(date_format)}, + ( + False, + f"Invalid values. start_date ({datetime.strptime(valid_date_str, date_format)}) needs to be lower than or equal to end_date ({min_date}).", + ), + ), + ({"api_key": "foobar", "start_date": min_date.strftime(date_format), "end_date": valid_date_str}, (True, None)), + ({"api_key": "foobar", "count": 0}, (False, "Invalid count value: 0. The value should be in the range [1,101).")), + ({"api_key": "foobar", "count": 101}, (False, "Invalid count value: 101. The value should be in the range [1,101).")), + ({"api_key": "foobar", "count": 1}, (True, None)), + ], +) +def test_check_connection(mocker, config, expected_return): + with patch.object(NasaApod, "read_records") as mock_http_request: + mock_http_request.return_value = iter([None]) + source = SourceNasa() + logger_mock = MagicMock() + assert source.check_connection(logger_mock, config) == expected_return + + +def test_streams(mocker): + source = SourceNasa() + config_mock = MagicMock() + streams = source.streams(config_mock) + expected_streams_number = 1 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-nasa/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-nasa/unit_tests/test_streams.py new file mode 100644 index 000000000000..0a6ae568ef8c --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/unit_tests/test_streams.py @@ -0,0 +1,80 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from datetime import datetime +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_nasa.source import NasaApod + +api_key_value = "foobar" +config = {"api_key": api_key_value} + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(NasaApod, "path", "v0/example_endpoint") + mocker.patch.object(NasaApod, "primary_key", "test_primary_key") + mocker.patch.object(NasaApod, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = NasaApod(config={**config, "start_date": "2022-09-10"}) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {"api_key": api_key_value, "start_date": "2022-09-10", "end_date": datetime.now().strftime("%Y-%m-%d")} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = NasaApod(config=config) + inputs = {"response": MagicMock()} + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = NasaApod(config=config) + response_object = [{"foo": "bar", "baz": ["qux"]}] + response_mock = MagicMock() + response_mock.configure_mock(**{"json.return_value": response_object}) + inputs = {"response": response_mock} + assert next(stream.parse_response(**inputs)) == response_object[0] + + +def test_request_headers(patch_base_class): + stream = NasaApod(config=config) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = NasaApod(config=config) + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = NasaApod(config=config) + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = NasaApod(config=config) + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 11ba90803f8d..d2f9a72127dc 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -114,6 +114,7 @@ For more information about the grading system, see [Product Release Stages](http | [Mongo DB](sources/mongodb-v2.md) | Alpha | Yes | | [My Hours](sources/my-hours.md) | Alpha | Yes | | [MySQL](sources/mysql.md) | Beta | Yes | +| [NASA](sources/nasa.md) | Alpha | Yes | | [Notion](sources/notion.md) | Generally Available | Yes | | [Okta](sources/okta.md) | Alpha | Yes | | [OneSignal](sources/onesignal.md) | Alpha | No | diff --git a/docs/integrations/sources/nasa.md b/docs/integrations/sources/nasa.md new file mode 100644 index 000000000000..5f0db94697df --- /dev/null +++ b/docs/integrations/sources/nasa.md @@ -0,0 +1,42 @@ +# NASA + +## Overview + +The NASA source supports full refresh syncs + +### Output schema + +Asingle output stream is available (at the moment) from this source: + +*[APOD](https://github.com/nasa/apod-api#docs-). + +If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose) + +### Features + +| Feature | Supported? | +|:------------------|:-----------| +| Full Refresh Sync | Yes | +| Incremental Sync | Yes | +| SSL connection | No | +| Namespaces | No | + +### Performance considerations + +The NASA connector should not run into NASA API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. + +## Getting started + +### Requirements + +* NASA API Key. You can use `DEMO_KEY` (see rate limits [here](https://api.nasa.gov/)). + +### Connect using `API Key`: +1. Generate an API Key as described [here](https://api.nasa.gov/). +2. Use the generated `API Key` in the Airbyte connection. + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------| +| 0.1.0 | 2022-10-24 | [18394](https://github.com/airbytehq/airbyte/pull/18394) | 🎉 New Source: NASA APOD |