From 395eca4dbfec259c1f6497aef1fbe20fadb8e579 Mon Sep 17 00:00:00 2001 From: Haithem SOUALA Date: Sat, 22 Oct 2022 16:03:38 +0200 Subject: [PATCH 01/19] init commit --- .vscode/settings.json | 2 +- .../source-mailjet-mail/.dockerignore | 6 ++ .../connectors/source-mailjet-mail/Dockerfile | 38 +++++++++ .../connectors/source-mailjet-mail/README.md | 79 +++++++++++++++++++ .../source-mailjet-mail/__init__.py | 3 + .../acceptance-test-config.yml | 26 ++++++ .../acceptance-test-docker.sh | 16 ++++ .../source-mailjet-mail/build.gradle | 9 +++ .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 4 + .../integration_tests/acceptance.py | 16 ++++ .../integration_tests/catalog.json | 12 +++ .../integration_tests/configured_catalog.json | 15 ++++ .../integration_tests/invalid_config.json | 3 + .../integration_tests/sample_config.json | 2 + .../integration_tests/sample_state.json | 5 ++ .../connectors/source-mailjet-mail/main.py | 13 +++ .../source-mailjet-mail/requirements.txt | 2 + .../connectors/source-mailjet-mail/setup.py | 29 +++++++ .../source_mailjet_mail/__init__.py | 8 ++ .../source_mailjet_mail/mailjet_mail.yaml | 36 +++++++++ .../source_mailjet_mail/schemas/TODO.md | 16 ++++ .../source_mailjet_mail/schemas/contacts.json | 22 ++++++ .../source_mailjet_mail/source.py | 18 +++++ .../source_mailjet_mail/spec.yaml | 21 +++++ 25 files changed, 403 insertions(+), 1 deletion(-) create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/.dockerignore create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/Dockerfile create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/README.md create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/__init__.py create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/build.gradle create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/main.py create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/requirements.txt create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/setup.py create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/__init__.py create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/mailjet_mail.yaml create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contacts.json create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/source.py create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml diff --git a/.vscode/settings.json b/.vscode/settings.json index e52044ed1069..a328b59d4c58 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -33,7 +33,7 @@ }, "[json]": { "editor.formatOnSave": true, - "editor.defaultFormatter": "esbenp.prettier-vscode" + "editor.defaultFormatter": "vscode.json-language-features" }, "[scss]": { "editor.formatOnSave": true, diff --git a/airbyte-integrations/connectors/source-mailjet-mail/.dockerignore b/airbyte-integrations/connectors/source-mailjet-mail/.dockerignore new file mode 100644 index 000000000000..ebac9a93e9f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_mailjet_mail +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-mailjet-mail/Dockerfile b/airbyte-integrations/connectors/source-mailjet-mail/Dockerfile new file mode 100644 index 000000000000..c65f2c6a7eaf --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_mailjet_mail ./source_mailjet_mail + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-mailjet-mail diff --git a/airbyte-integrations/connectors/source-mailjet-mail/README.md b/airbyte-integrations/connectors/source-mailjet-mail/README.md new file mode 100644 index 000000000000..e2814ba1bb13 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/README.md @@ -0,0 +1,79 @@ +# Mailjet Mail Source + +This is the repository for the Mailjet Mail configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/mailjet-mail). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-mailjet-mail:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/mailjet-mail) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_mailjet_mail/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source mailjet-mail test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-mailjet-mail:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-mailjet-mail:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-mailjet-mail:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailjet-mail:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailjet-mail:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-mailjet-mail:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-mailjet-mail:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-mailjet-mail:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-mailjet-mail/__init__.py b/airbyte-integrations/connectors/source-mailjet-mail/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml new file mode 100644 index 000000000000..18f11912ffcf --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml @@ -0,0 +1,26 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-mailjet-mail:dev +tests: + spec: + - spec_path: "source_mailjet_mail/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.txt" + # extra_fields: no + # exact_order: no + # extra_records: yes + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-mailjet-mail/build.gradle b/airbyte-integrations/connectors/source-mailjet-mail/build.gradle new file mode 100644 index 000000000000..2078df5f5d93 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_mailjet_mail' +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/__init__.py b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..71710348aa30 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/abnormal_state.json @@ -0,0 +1,4 @@ +{ + "contacts": { + } +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/acceptance.py new file mode 100644 index 000000000000..1302b2f57e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/catalog.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/catalog.json new file mode 100644 index 000000000000..ccbe8f6b30e2 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/catalog.json @@ -0,0 +1,12 @@ +{ + "streams": [ + { + "name": "contacts", + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_cursor": false, + "json_schema": {} + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..99e5dda19789 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json @@ -0,0 +1,15 @@ +{ + "streams": [ + { + "stream": { + "name": "contacts", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/invalid_config.json new file mode 100644 index 000000000000..6ef46db35028 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "date":"xxx" +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json new file mode 100644 index 000000000000..7a73a41bfdf7 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json @@ -0,0 +1,2 @@ +{ +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/main.py b/airbyte-integrations/connectors/source-mailjet-mail/main.py new file mode 100644 index 000000000000..58793767365f --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_mailjet_mail import SourceMailjetMail + +if __name__ == "__main__": + source = SourceMailjetMail() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mailjet-mail/requirements.txt b/airbyte-integrations/connectors/source-mailjet-mail/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-mailjet-mail/setup.py b/airbyte-integrations/connectors/source-mailjet-mail/setup.py new file mode 100644 index 000000000000..9fc525ecd862 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_mailjet_mail", + description="Source implementation for Mailjet Mail.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/__init__.py b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/__init__.py new file mode 100644 index 000000000000..a02451809fe7 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceMailjetMail + +__all__ = ["SourceMailjetMail"] diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/mailjet_mail.yaml b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/mailjet_mail.yaml new file mode 100644 index 000000000000..7025ece36227 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/mailjet_mail.yaml @@ -0,0 +1,36 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: ["Data"] + requester: + url_base: "https://api.mailjet.com/v3/REST" + http_method: "GET" + authenticator: + type: BasicHttpAuthenticator + username: "{{ config['username'] }}" + password: "{{ config['password'] }}" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + requester: + $ref: "*ref(definitions.requester)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + contacts_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "contacts" + primary_key: "ID" + path: "/contactslist" + +streams: + - "*ref(definitions.contacts_stream)" + +check: + stream_names: + - "contacts" diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/TODO.md b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/TODO.md new file mode 100644 index 000000000000..a36641189a0a --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/TODO.md @@ -0,0 +1,16 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +You can describe the schema of your streams using one `.json` file per stream. + +## Static schemas +From the `mailjet_mail.yaml` configuration file, you read the `.json` files in the `schemas/` directory. You can refer to a schema in your configuration file using the `schema_loader` component's `file_path` field. For example: +``` +schema_loader: + type: JsonSchema + file_path: "./source_mailjet_mail/schemas/customers.json" +``` +Every stream specified in the configuration file should have a corresponding `.json` schema file. + +Delete this file once you're done. Or don't. Up to you :) + diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contacts.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contacts.json new file mode 100644 index 000000000000..5a604f3dd8b0 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contacts.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "Name": { + "type": ["null", "string"] + }, + "Address": { + "type": ["null", "string"] + }, + "CreatedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "ID": { + "type": ["null", "number"] + }, + "SubscriberCount": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/source.py b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/source.py new file mode 100644 index 000000000000..2f3fa93c6441 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceMailjetMail(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "mailjet_mail.yaml"}) diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml new file mode 100644 index 000000000000..f46b553c473d --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml @@ -0,0 +1,21 @@ +documentationUrl: https://docsurl.com +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Mailjet Mail Spec + type: object + required: + - username + - password + additionalProperties: true + properties: + username: + type: string + description: >- + The username is your API Key. See here. + password: + type: string + description: >- + The password is your API Secret Key. See here. + airbyte_secret: true From 413f52e32bf9dca43e6079c4146882fc48e7b99d Mon Sep 17 00:00:00 2001 From: Haithem SOUALA Date: Sun, 23 Oct 2022 20:43:40 +0200 Subject: [PATCH 02/19] add streams --- .vscode/frontend.code-workspace | 71 ------------------- .vscode/settings.json | 45 ------------ .../integration_tests/abnormal_state.json | 3 +- .../integration_tests/catalog.json | 37 ++++++++-- .../integration_tests/configured_catalog.json | 44 ++++++++++++ .../integration_tests/invalid_config.json | 3 +- .../integration_tests/sample_config.json | 1 + .../source_mailjet_mail/mailjet_mail.yaml | 59 ++++++++++++++- .../source_mailjet_mail/schemas/contacts.json | 59 ++++++++++----- 9 files changed, 178 insertions(+), 144 deletions(-) delete mode 100644 .vscode/frontend.code-workspace delete mode 100644 .vscode/settings.json diff --git a/.vscode/frontend.code-workspace b/.vscode/frontend.code-workspace deleted file mode 100644 index c55ade0551d9..000000000000 --- a/.vscode/frontend.code-workspace +++ /dev/null @@ -1,71 +0,0 @@ -{ - "folders": [ - { - "path": "../airbyte-webapp" - }, - { - "path": "../airbyte-webapp-e2e-tests" - }, - { - "name": "docs/integrations/destinations", - "path": "../docs/integrations/destinations" - }, - { - "name": "docs/integrations/sources", - "path": "../docs/integrations/sources" - } - ], - "extensions": { - "recommendations": [ - "dbaeumer.vscode-eslint", - "stylelint.vscode-stylelint", - "esbenp.prettier-vscode", - "eamodio.gitlens", - ] - }, - "settings": { - "javascript.preferences.quoteStyle": "double", - "typescript.preferences.quoteStyle": "double", - "javascript.preferences.importModuleSpecifier": "shortest", - "typescript.preferences.importModuleSpecifier": "shortest", - "javascript.updateImportsOnFileMove.enabled": "always", - "typescript.updateImportsOnFileMove.enabled": "always", - "editor.detectIndentation": true, - "eslint.format.enable": true, - "eslint.run": "onType", - "stylelint.enable": true, - "stylelint.validate": ["css", "scss"], - "[javascript]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "dbaeumer.vscode-eslint", - "editor.codeActionsOnSave": { - "source.organizeImports": false - } - }, - "[typescript]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "dbaeumer.vscode-eslint", - "editor.codeActionsOnSave": { - "source.organizeImports": false - } - }, - "[typescriptreact]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "dbaeumer.vscode-eslint", - "editor.codeActionsOnSave": { - "source.organizeImports": false - } - }, - "[json]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "esbenp.prettier-vscode" - }, - "[scss]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "esbenp.prettier-vscode", - "editor.codeActionsOnSave": { - "source.fixAll.stylelint": true - } - } - } -} diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index a328b59d4c58..000000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "javascript.preferences.quoteStyle": "double", - "typescript.preferences.quoteStyle": "double", - "javascript.preferences.importModuleSpecifier": "shortest", - "typescript.preferences.importModuleSpecifier": "shortest", - "javascript.updateImportsOnFileMove.enabled": "always", - "typescript.updateImportsOnFileMove.enabled": "always", - "editor.detectIndentation": true, - "eslint.format.enable": true, - "eslint.run": "onType", - "stylelint.enable": true, - "stylelint.validate": ["css", "scss"], - "[javascript]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "dbaeumer.vscode-eslint", - "editor.codeActionsOnSave": { - "source.organizeImports": false - } - }, - "[typescript]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "dbaeumer.vscode-eslint", - "editor.codeActionsOnSave": { - "source.organizeImports": false - } - }, - "[typescriptreact]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "dbaeumer.vscode-eslint", - "editor.codeActionsOnSave": { - "source.organizeImports": false - } - }, - "[json]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "vscode.json-language-features" - }, - "[scss]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "esbenp.prettier-vscode", - "editor.codeActionsOnSave": { - "source.fixAll.stylelint": true - } - } -} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/abnormal_state.json index 71710348aa30..52b0f2c2118f 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/abnormal_state.json @@ -1,4 +1,5 @@ { - "contacts": { + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" } } diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/catalog.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/catalog.json index ccbe8f6b30e2..439ddd3d700d 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/catalog.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/catalog.json @@ -1,12 +1,39 @@ { "streams": [ { - "name": "contacts", - "supported_sync_modes": [ - "full_refresh" - ], + "name": "TODO fix this file", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": "column1", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + }, + { + "name": "table1", + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": false, - "json_schema": {} + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } } ] } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json index 99e5dda19789..3a4087104e0a 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json @@ -1,5 +1,27 @@ { "streams": [ + { + "stream": { + "name": "contactslist", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "stats", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "contacts", @@ -10,6 +32,28 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "campaign", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "message", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/invalid_config.json index 6ef46db35028..38bd19e0204a 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/invalid_config.json @@ -1,3 +1,4 @@ { - "date":"xxx" + "username": "ooo2e2eec3fb42181773d75dae6ab140cf8", + "password": "ppp0d6be348aaea7ee4196a73c7608e0ffb" } diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json index 7a73a41bfdf7..b9a1d25d959d 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json @@ -1,2 +1,3 @@ { + "fix-me": "TODO" } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/mailjet_mail.yaml b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/mailjet_mail.yaml index 7025ece36227..02750b9c0074 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/mailjet_mail.yaml +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/mailjet_mail.yaml @@ -11,6 +11,19 @@ definitions: type: BasicHttpAuthenticator username: "{{ config['username'] }}" password: "{{ config['password'] }}" + offset_paginator: + type: DefaultPaginator + $options: + url_base: "*ref(definitions.requester.url_base)" + pagination_strategy: + type: "OffsetIncrement" + page_size: 50 + page_token_option: + field_name: "offset" + inject_into: "request_parameter" + page_size_option: + inject_into: "request_parameter" + field_name: "limit" retriever: record_selector: $ref: "*ref(definitions.selector)" @@ -21,16 +34,58 @@ definitions: base_stream: retriever: $ref: "*ref(definitions.retriever)" + contactslist_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "contactslist" + primary_key: "ID" + path: "/contactslist?limit=100" contacts_stream: $ref: "*ref(definitions.base_stream)" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + $ref: "*ref(definitions.offset_paginator)" + requester: + $ref: "*ref(definitions.requester)" $options: name: "contacts" primary_key: "ID" - path: "/contactslist" + path: "/contact" + stats_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "stats" + primary_key: "APIKeyID" + path: "/statcounters?CounterSource=APIKey&CounterResolution=Lifetime&CounterTiming=Message" + campaign_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "campaign" + primary_key: "ID" + path: "/campaign?FromTS=972314998" + message_stream: + $ref: "*ref(definitions.base_stream)" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + $ref: "*ref(definitions.offset_paginator)" + requester: + $ref: "*ref(definitions.requester)" + $options: + name: "message" + primary_key: "ID" + path: "/message" streams: + - "*ref(definitions.contactslist_stream)" - "*ref(definitions.contacts_stream)" + - "*ref(definitions.stats_stream)" + - "*ref(definitions.campaign_stream)" + - "*ref(definitions.message_stream)" check: stream_names: - - "contacts" + - "contactslist" diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contacts.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contacts.json index 5a604f3dd8b0..9bc816265663 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contacts.json @@ -1,22 +1,43 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "Name": { - "type": ["null", "string"] - }, - "Address": { - "type": ["null", "string"] - }, - "CreatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "ID": { - "type": ["null", "number"] - }, - "SubscriberCount": { - "type": ["null", "number"] + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "CreatedAt": { + "type": "string" + }, + "DeliveredCount": { + "type": "integer" + }, + "Email": { + "type": "string" + }, + "ID": { + "type": "integer" + }, + "IsOptInPending": { + "type": "boolean" + }, + "IsSpamComplaining": { + "type": "boolean" + }, + "LastActivityAt": { + "type": "string" + }, + "LastUpdateAt": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "UnsubscribedAt": { + "type": "string" + }, + "UnsubscribedBy": { + "type": "string" + }, + "IsExcludedFromCampaigns": { + "type": "boolean" + } } } -} + \ No newline at end of file From 76145f0dd2dfd9baa6f7852d81635efe2d76e7e4 Mon Sep 17 00:00:00 2001 From: Haithem SOUALA Date: Sun, 23 Oct 2022 20:53:05 +0200 Subject: [PATCH 03/19] add schemas --- .../source_mailjet_mail/schemas/campaign.json | 73 +++++++++++++++++++ .../schemas/contactslist.json | 24 ++++++ .../source_mailjet_mail/schemas/message.json | 73 +++++++++++++++++++ .../source_mailjet_mail/schemas/stats.json | 69 ++++++++++++++++++ 4 files changed, 239 insertions(+) create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/campaign.json create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contactslist.json create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/message.json create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats.json diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/campaign.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/campaign.json new file mode 100644 index 000000000000..c25780e63a66 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/campaign.json @@ -0,0 +1,73 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "CampaignType": { + "type": "integer" + }, + "ClickTracked": { + "type": "integer" + }, + "CreatedAt": { + "type": "string" + }, + "CustomValue": { + "type": "integer" + }, + "FirstMessageID": { + "type": "integer" + }, + "FromEmail": { + "type": "boolean" + }, + "FromID": { + "type": "string" + }, + "FromName": { + "type": "string" + }, + "HasHtmlCount": { + "type": "string" + }, + "HasTxtCount": { + "type": "string" + }, + "ID": { + "type": "integer" + }, + "IsDeleted": { + "type": "boolean" + }, + "IsStarred": { + "type": "boolean" + }, + "ListID": { + "type": "integer" + }, + "NewsLetterID": { + "type": "integer" + }, + "OpenTracked": { + "type": "string" + }, + "SendEndAt": { + "type": "string" + }, + "SendStartAt": { + "type": "boolean" + }, + "SpamassScore": { + "type": "integer" + }, + "Status": { + "type": "integer" + }, + "Subject": { + "type": "string" + }, + "UnsubscribeTrackedCount": { + "type": "integer" + } + } + } + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contactslist.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contactslist.json new file mode 100644 index 000000000000..8e373f00d21f --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contactslist.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "Address": { + "type": "string" + }, + "CreatedAt": { + "type": "string" + }, + "ID": { + "type": "integer" + }, + "IsDeleted": { + "type": "boolean" + }, + "Name": { + "type": "string" + }, + "SubscriberCount": { + "type": "integer" + } + } +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/message.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/message.json new file mode 100644 index 000000000000..c6a472f813a4 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/message.json @@ -0,0 +1,73 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ArrivedAt": { + "type": "string" + }, + "AttachmentCount": { + "type": "integer" + }, + "AttemptCount": { + "type": "integer" + }, + "ContactAlt": { + "type": "string" + }, + "ContactID": { + "type": "integer" + }, + "Delay": { + "type": "integer" + }, + "DestinationID": { + "type": "integer" + }, + "FilterTime": { + "type": "integer" + }, + "ID": { + "type": "integer" + }, + "IsClickTracked": { + "type": "boolean" + }, + "IsHTMLPartIncluded": { + "type": "boolean" + }, + "IsOpenTracked": { + "type": "boolean" + }, + "IsTextPartIncluded": { + "type": "boolean" + }, + "IsUnsubTracked": { + "type": "string" + }, + "MessageSize": { + "type": "integer" + }, + "SenderID": { + "type": "integer" + }, + "SpamassassinScore": { + "type": "integer" + }, + "SpamassRules": { + "type": "string" + }, + "StatePermanent": { + "type": "string" + }, + "Status": { + "type": "string" + }, + "Subject": { + "type": "string" + }, + "UUID": { + "type": "string" + } + } + } + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats.json new file mode 100644 index 000000000000..c2fe4a0a8f20 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats.json @@ -0,0 +1,69 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "APIKeyID": { + "type": "integer" + }, + "EventClickDelay": { + "type": "integer" + }, + "EventClickedCount": { + "type": "integer" + }, + "EventOpenDelay": { + "type": "integer" + }, + "EventOpenedCount": { + "type": "integer" + }, + "EventUnsubscribedCount": { + "type": "integer" + }, + "EventWorkflowExitedCount": { + "type": "integer" + }, + "MessageBlockedCount": { + "type": "integer" + }, + "MessageClickedCount": { + "type": "integer" + }, + "MessageDeferredCount": { + "type": "integer" + }, + "MessageHardBouncedCount": { + "type": "integer" + }, + "MessageOpenedCount": { + "type": "integer" + }, + "MessageQueuedCount": { + "type": "integer" + }, + "MessageSentCount": { + "type": "integer" + }, + "MessageSoftBouncedCount": { + "type": "integer" + }, + "MessageSpamCount": { + "type": "integer" + }, + "MessageUnsubscribedCount": { + "type": "integer" + }, + "MessageWorkFlowExitedCount": { + "type": "integer" + }, + "SourceID": { + "type": "integer" + }, + "Timeslice": { + "type": "string" + }, + "Total": { + "type": "integer" + } + } + } \ No newline at end of file From 2e929674f3d240d86ea7c247d225b513da510bca Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 27 Oct 2022 23:08:06 -0300 Subject: [PATCH 04/19] solve conflict --- .../source-mailjet-mail/integration_tests/invalid_config.json | 4 ++-- docs/integrations/README.md | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/invalid_config.json index 38bd19e0204a..decdd1d2a0e5 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/invalid_config.json @@ -1,4 +1,4 @@ { - "username": "ooo2e2eec3fb42181773d75dae6ab140cf8", - "password": "ppp0d6be348aaea7ee4196a73c7608e0ffb" + "username": "ooo2e2eec3frrrrrrf8", + "password": "ppp0d6be348arrrre0ffb" } diff --git a/docs/integrations/README.md b/docs/integrations/README.md index c242e2149abf..6981092c46de 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -103,6 +103,7 @@ For more information about the grading system, see [Product Release Stages](http | [Magento](sources/magento.md) | Alpha | No | | [Mailchimp](sources/mailchimp.md) | Generally Available | Yes | | [Mailerlite](sources/mailerlite.md) | Alpha | No | +| [Mailjet Mail](sources/mailjet-mail.md) | Alpha | No | | [Marketo](sources/marketo.md) | Generally Available | Yes | | [Metabase](sources/metabase.md) | Alpha | Yes | | [Microsoft Dynamics AX](sources/microsoft-dynamics-ax.md) | Alpha | No | From edd81c72e2a3a613d622a902b4a1f5df299610a2 Mon Sep 17 00:00:00 2001 From: Haithem SOUALA Date: Sun, 23 Oct 2022 21:25:10 +0200 Subject: [PATCH 05/19] add docs --- docs/integrations/sources/mailjet-mail.md | 31 +++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 docs/integrations/sources/mailjet-mail.md diff --git a/docs/integrations/sources/mailjet-mail.md b/docs/integrations/sources/mailjet-mail.md new file mode 100644 index 000000000000..d17f66395f1c --- /dev/null +++ b/docs/integrations/sources/mailjet-mail.md @@ -0,0 +1,31 @@ +# Mailjet - Mail API + +## Sync overview + +This source can sync data from the [Mailjet Mail API](https://dev.mailjet.com/email). At present this connector only supports full refresh syncs meaning that each time you use the connector it will sync all available records from scratch. Please use cautiously if you expect your API to have a lot of records. + +## This Source Supports the Following Streams + +* contact list +* contacts +* messages +* campaigns +* stats + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | Yes | | +| Incremental Sync | No | | + +### Performance considerations + +Mailjet APIs are under rate limits for the number of API calls allowed per API keys per second. If you reach a rate limit, API will return a 429 HTTP error code. See [here](https://dev.mailjet.com/email/reference/overview/rate-limits/) + +## Getting started + +### Requirements + +* Mailjet Mail API_KEY (username) +* Mailjet Mail SECRET_KEY (password) \ No newline at end of file From 0d2120129b4188220e91330f11cf38b91ff26e17 Mon Sep 17 00:00:00 2001 From: Haithem SOUALA Date: Tue, 25 Oct 2022 10:57:08 +0200 Subject: [PATCH 06/19] fix comments --- .../acceptance-test-config.yml | 2 +- .../integration_tests/configured_catalog.json | 2 +- .../integration_tests/invalid_config.json | 4 +- .../source_mailjet_mail/mailjet_mail.yaml | 23 ++++--- .../source_mailjet_mail/schemas/TODO.md | 16 ----- .../source_mailjet_mail/schemas/stats.json | 69 ------------------- .../source_mailjet_mail/spec.yaml | 12 ++-- docs/integrations/sources/mailjet-mail.md | 10 ++- 8 files changed, 33 insertions(+), 105 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/TODO.md delete mode 100644 airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats.json diff --git a/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml index 18f11912ffcf..ad650b63af16 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml @@ -14,7 +14,7 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + empty_streams: ["contactslist", "contacts", "stats_api_lifetime_message", "campaign", "message"] # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file # expect_records: # path: "integration_tests/expected_records.txt" diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json index 3a4087104e0a..b7fdde85fbee 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json @@ -13,7 +13,7 @@ }, { "stream": { - "name": "stats", + "name": "stats_api_lifetime_message", "json_schema": {}, "supported_sync_modes": [ "full_refresh" diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/invalid_config.json index decdd1d2a0e5..1d6826198820 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/invalid_config.json @@ -1,4 +1,4 @@ { - "username": "ooo2e2eec3frrrrrrf8", - "password": "ppp0d6be348arrrre0ffb" + "api_key": "ooo2e2eec3frrrrrrf8", + "api_key_secret": "ppp0d6be348arrrre0ffb" } diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/mailjet_mail.yaml b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/mailjet_mail.yaml index 02750b9c0074..68833d35f41a 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/mailjet_mail.yaml +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/mailjet_mail.yaml @@ -9,15 +9,15 @@ definitions: http_method: "GET" authenticator: type: BasicHttpAuthenticator - username: "{{ config['username'] }}" - password: "{{ config['password'] }}" + username: "{{ config['api_key'] }}" + password: "{{ config['api_key_secret'] }}" offset_paginator: type: DefaultPaginator $options: url_base: "*ref(definitions.requester.url_base)" pagination_strategy: type: "OffsetIncrement" - page_size: 50 + page_size: 100 page_token_option: field_name: "offset" inject_into: "request_parameter" @@ -36,10 +36,17 @@ definitions: $ref: "*ref(definitions.retriever)" contactslist_stream: $ref: "*ref(definitions.base_stream)" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + $ref: "*ref(definitions.offset_paginator)" + requester: + $ref: "*ref(definitions.requester)" $options: name: "contactslist" primary_key: "ID" - path: "/contactslist?limit=100" + path: "/contactslist" contacts_stream: $ref: "*ref(definitions.base_stream)" retriever: @@ -53,10 +60,10 @@ definitions: name: "contacts" primary_key: "ID" path: "/contact" - stats_stream: + stats_api_lifetime_message_stream: $ref: "*ref(definitions.base_stream)" $options: - name: "stats" + name: "stats_api_lifetime_message" primary_key: "APIKeyID" path: "/statcounters?CounterSource=APIKey&CounterResolution=Lifetime&CounterTiming=Message" campaign_stream: @@ -64,7 +71,7 @@ definitions: $options: name: "campaign" primary_key: "ID" - path: "/campaign?FromTS=972314998" + path: "/campaign" message_stream: $ref: "*ref(definitions.base_stream)" retriever: @@ -82,7 +89,7 @@ definitions: streams: - "*ref(definitions.contactslist_stream)" - "*ref(definitions.contacts_stream)" - - "*ref(definitions.stats_stream)" + - "*ref(definitions.stats_api_lifetime_message_stream)" - "*ref(definitions.campaign_stream)" - "*ref(definitions.message_stream)" diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/TODO.md b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/TODO.md deleted file mode 100644 index a36641189a0a..000000000000 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/TODO.md +++ /dev/null @@ -1,16 +0,0 @@ -# TODO: Define your stream schemas -Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). - -You can describe the schema of your streams using one `.json` file per stream. - -## Static schemas -From the `mailjet_mail.yaml` configuration file, you read the `.json` files in the `schemas/` directory. You can refer to a schema in your configuration file using the `schema_loader` component's `file_path` field. For example: -``` -schema_loader: - type: JsonSchema - file_path: "./source_mailjet_mail/schemas/customers.json" -``` -Every stream specified in the configuration file should have a corresponding `.json` schema file. - -Delete this file once you're done. Or don't. Up to you :) - diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats.json deleted file mode 100644 index c2fe4a0a8f20..000000000000 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "APIKeyID": { - "type": "integer" - }, - "EventClickDelay": { - "type": "integer" - }, - "EventClickedCount": { - "type": "integer" - }, - "EventOpenDelay": { - "type": "integer" - }, - "EventOpenedCount": { - "type": "integer" - }, - "EventUnsubscribedCount": { - "type": "integer" - }, - "EventWorkflowExitedCount": { - "type": "integer" - }, - "MessageBlockedCount": { - "type": "integer" - }, - "MessageClickedCount": { - "type": "integer" - }, - "MessageDeferredCount": { - "type": "integer" - }, - "MessageHardBouncedCount": { - "type": "integer" - }, - "MessageOpenedCount": { - "type": "integer" - }, - "MessageQueuedCount": { - "type": "integer" - }, - "MessageSentCount": { - "type": "integer" - }, - "MessageSoftBouncedCount": { - "type": "integer" - }, - "MessageSpamCount": { - "type": "integer" - }, - "MessageUnsubscribedCount": { - "type": "integer" - }, - "MessageWorkFlowExitedCount": { - "type": "integer" - }, - "SourceID": { - "type": "integer" - }, - "Timeslice": { - "type": "string" - }, - "Total": { - "type": "integer" - } - } - } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml index f46b553c473d..a8c73c71b031 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml @@ -4,18 +4,18 @@ connectionSpecification: title: Mailjet Mail Spec type: object required: - - username - - password + - api_key + - api_key_secret additionalProperties: true properties: - username: + api_key: type: string description: >- - The username is your API Key. See here. - password: + api_key_secret: type: string description: >- - The password is your API Secret Key. See here. airbyte_secret: true diff --git a/docs/integrations/sources/mailjet-mail.md b/docs/integrations/sources/mailjet-mail.md index d17f66395f1c..18b881fe9b08 100644 --- a/docs/integrations/sources/mailjet-mail.md +++ b/docs/integrations/sources/mailjet-mail.md @@ -27,5 +27,11 @@ Mailjet APIs are under rate limits for the number of API calls allowed per API k ### Requirements -* Mailjet Mail API_KEY (username) -* Mailjet Mail SECRET_KEY (password) \ No newline at end of file +* Mailjet Mail API_KEY +* Mailjet Mail SECRET_KEY + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :-------------------------------------------------------- | :----------------------------------------- | +| 0.1.0 | 2022-10-26 | [#18332](https://github.com/airbytehq/airbyte/pull/18332) | 🎉 New Source: Mailjet Mail API [low-code CDK] | \ No newline at end of file From 1ba492cfb6c8155c8764a8d37c1da65ac8127c97 Mon Sep 17 00:00:00 2001 From: Haithem SOUALA Date: Tue, 25 Oct 2022 10:59:55 +0200 Subject: [PATCH 07/19] restore vscode config --- .../schemas/stats_api_lifetime_message.json | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats_api_lifetime_message.json diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats_api_lifetime_message.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats_api_lifetime_message.json new file mode 100644 index 000000000000..c2fe4a0a8f20 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats_api_lifetime_message.json @@ -0,0 +1,69 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "APIKeyID": { + "type": "integer" + }, + "EventClickDelay": { + "type": "integer" + }, + "EventClickedCount": { + "type": "integer" + }, + "EventOpenDelay": { + "type": "integer" + }, + "EventOpenedCount": { + "type": "integer" + }, + "EventUnsubscribedCount": { + "type": "integer" + }, + "EventWorkflowExitedCount": { + "type": "integer" + }, + "MessageBlockedCount": { + "type": "integer" + }, + "MessageClickedCount": { + "type": "integer" + }, + "MessageDeferredCount": { + "type": "integer" + }, + "MessageHardBouncedCount": { + "type": "integer" + }, + "MessageOpenedCount": { + "type": "integer" + }, + "MessageQueuedCount": { + "type": "integer" + }, + "MessageSentCount": { + "type": "integer" + }, + "MessageSoftBouncedCount": { + "type": "integer" + }, + "MessageSpamCount": { + "type": "integer" + }, + "MessageUnsubscribedCount": { + "type": "integer" + }, + "MessageWorkFlowExitedCount": { + "type": "integer" + }, + "SourceID": { + "type": "integer" + }, + "Timeslice": { + "type": "string" + }, + "Total": { + "type": "integer" + } + } + } \ No newline at end of file From f2e89891ab4c9043765279bf9458dd587481eeb3 Mon Sep 17 00:00:00 2001 From: Haithem SOUALA Date: Tue, 25 Oct 2022 11:00:14 +0200 Subject: [PATCH 08/19] restore vscode config --- frontend.code-workspace | 71 +++++++++++++++++++++++++++++++++++++++++ settings.json | 45 ++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 frontend.code-workspace create mode 100644 settings.json diff --git a/frontend.code-workspace b/frontend.code-workspace new file mode 100644 index 000000000000..c55ade0551d9 --- /dev/null +++ b/frontend.code-workspace @@ -0,0 +1,71 @@ +{ + "folders": [ + { + "path": "../airbyte-webapp" + }, + { + "path": "../airbyte-webapp-e2e-tests" + }, + { + "name": "docs/integrations/destinations", + "path": "../docs/integrations/destinations" + }, + { + "name": "docs/integrations/sources", + "path": "../docs/integrations/sources" + } + ], + "extensions": { + "recommendations": [ + "dbaeumer.vscode-eslint", + "stylelint.vscode-stylelint", + "esbenp.prettier-vscode", + "eamodio.gitlens", + ] + }, + "settings": { + "javascript.preferences.quoteStyle": "double", + "typescript.preferences.quoteStyle": "double", + "javascript.preferences.importModuleSpecifier": "shortest", + "typescript.preferences.importModuleSpecifier": "shortest", + "javascript.updateImportsOnFileMove.enabled": "always", + "typescript.updateImportsOnFileMove.enabled": "always", + "editor.detectIndentation": true, + "eslint.format.enable": true, + "eslint.run": "onType", + "stylelint.enable": true, + "stylelint.validate": ["css", "scss"], + "[javascript]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "dbaeumer.vscode-eslint", + "editor.codeActionsOnSave": { + "source.organizeImports": false + } + }, + "[typescript]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "dbaeumer.vscode-eslint", + "editor.codeActionsOnSave": { + "source.organizeImports": false + } + }, + "[typescriptreact]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "dbaeumer.vscode-eslint", + "editor.codeActionsOnSave": { + "source.organizeImports": false + } + }, + "[json]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[scss]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.codeActionsOnSave": { + "source.fixAll.stylelint": true + } + } + } +} diff --git a/settings.json b/settings.json new file mode 100644 index 000000000000..a328b59d4c58 --- /dev/null +++ b/settings.json @@ -0,0 +1,45 @@ +{ + "javascript.preferences.quoteStyle": "double", + "typescript.preferences.quoteStyle": "double", + "javascript.preferences.importModuleSpecifier": "shortest", + "typescript.preferences.importModuleSpecifier": "shortest", + "javascript.updateImportsOnFileMove.enabled": "always", + "typescript.updateImportsOnFileMove.enabled": "always", + "editor.detectIndentation": true, + "eslint.format.enable": true, + "eslint.run": "onType", + "stylelint.enable": true, + "stylelint.validate": ["css", "scss"], + "[javascript]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "dbaeumer.vscode-eslint", + "editor.codeActionsOnSave": { + "source.organizeImports": false + } + }, + "[typescript]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "dbaeumer.vscode-eslint", + "editor.codeActionsOnSave": { + "source.organizeImports": false + } + }, + "[typescriptreact]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "dbaeumer.vscode-eslint", + "editor.codeActionsOnSave": { + "source.organizeImports": false + } + }, + "[json]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "vscode.json-language-features" + }, + "[scss]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.codeActionsOnSave": { + "source.fixAll.stylelint": true + } + } +} From 6f8074180b0e6c3d6e5d5040405b9ca10abdee65 Mon Sep 17 00:00:00 2001 From: Haithem SOUALA Date: Tue, 25 Oct 2022 15:42:47 +0200 Subject: [PATCH 09/19] clean --- .../integration_tests/catalog.json | 39 ----- .../source_mailjet_mail/schemas/campaign.json | 139 +++++++++--------- .../source_mailjet_mail/schemas/contacts.json | 79 +++++----- .../source_mailjet_mail/schemas/message.json | 139 +++++++++--------- .../schemas/stats_api_lifetime_message.json | 134 ++++++++--------- .../source_mailjet_mail/spec.yaml | 2 +- 6 files changed, 245 insertions(+), 287 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-mailjet-mail/integration_tests/catalog.json diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/catalog.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/catalog.json deleted file mode 100644 index 439ddd3d700d..000000000000 --- a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/catalog.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "streams": [ - { - "name": "TODO fix this file", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": "column1", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } - }, - { - "name": "table1", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } - } - ] -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/campaign.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/campaign.json index c25780e63a66..a5b8b6045a0c 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/campaign.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/campaign.json @@ -1,73 +1,72 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "CampaignType": { - "type": "integer" - }, - "ClickTracked": { - "type": "integer" - }, - "CreatedAt": { - "type": "string" - }, - "CustomValue": { - "type": "integer" - }, - "FirstMessageID": { - "type": "integer" - }, - "FromEmail": { - "type": "boolean" - }, - "FromID": { - "type": "string" - }, - "FromName": { - "type": "string" - }, - "HasHtmlCount": { - "type": "string" - }, - "HasTxtCount": { - "type": "string" - }, - "ID": { - "type": "integer" - }, - "IsDeleted": { - "type": "boolean" - }, - "IsStarred": { - "type": "boolean" - }, - "ListID": { - "type": "integer" - }, - "NewsLetterID": { - "type": "integer" - }, - "OpenTracked": { - "type": "string" - }, - "SendEndAt": { - "type": "string" - }, - "SendStartAt": { - "type": "boolean" - }, - "SpamassScore": { - "type": "integer" - }, - "Status": { - "type": "integer" - }, - "Subject": { - "type": "string" - }, - "UnsubscribeTrackedCount": { - "type": "integer" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "CampaignType": { + "type": "integer" + }, + "ClickTracked": { + "type": "integer" + }, + "CreatedAt": { + "type": "string" + }, + "CustomValue": { + "type": "integer" + }, + "FirstMessageID": { + "type": "integer" + }, + "FromEmail": { + "type": "boolean" + }, + "FromID": { + "type": "string" + }, + "FromName": { + "type": "string" + }, + "HasHtmlCount": { + "type": "string" + }, + "HasTxtCount": { + "type": "string" + }, + "ID": { + "type": "integer" + }, + "IsDeleted": { + "type": "boolean" + }, + "IsStarred": { + "type": "boolean" + }, + "ListID": { + "type": "integer" + }, + "NewsLetterID": { + "type": "integer" + }, + "OpenTracked": { + "type": "string" + }, + "SendEndAt": { + "type": "string" + }, + "SendStartAt": { + "type": "boolean" + }, + "SpamassScore": { + "type": "integer" + }, + "Status": { + "type": "integer" + }, + "Subject": { + "type": "string" + }, + "UnsubscribeTrackedCount": { + "type": "integer" } } - \ No newline at end of file +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contacts.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contacts.json index 9bc816265663..58874e9f6187 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contacts.json @@ -1,43 +1,42 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "CreatedAt": { - "type": "string" - }, - "DeliveredCount": { - "type": "integer" - }, - "Email": { - "type": "string" - }, - "ID": { - "type": "integer" - }, - "IsOptInPending": { - "type": "boolean" - }, - "IsSpamComplaining": { - "type": "boolean" - }, - "LastActivityAt": { - "type": "string" - }, - "LastUpdateAt": { - "type": "string" - }, - "Name": { - "type": "string" - }, - "UnsubscribedAt": { - "type": "string" - }, - "UnsubscribedBy": { - "type": "string" - }, - "IsExcludedFromCampaigns": { - "type": "boolean" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "CreatedAt": { + "type": "string" + }, + "DeliveredCount": { + "type": "integer" + }, + "Email": { + "type": "string" + }, + "ID": { + "type": "integer" + }, + "IsOptInPending": { + "type": "boolean" + }, + "IsSpamComplaining": { + "type": "boolean" + }, + "LastActivityAt": { + "type": "string" + }, + "LastUpdateAt": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "UnsubscribedAt": { + "type": "string" + }, + "UnsubscribedBy": { + "type": "string" + }, + "IsExcludedFromCampaigns": { + "type": "boolean" } } - \ No newline at end of file +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/message.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/message.json index c6a472f813a4..d8016dae0a5e 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/message.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/message.json @@ -1,73 +1,72 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "ArrivedAt": { - "type": "string" - }, - "AttachmentCount": { - "type": "integer" - }, - "AttemptCount": { - "type": "integer" - }, - "ContactAlt": { - "type": "string" - }, - "ContactID": { - "type": "integer" - }, - "Delay": { - "type": "integer" - }, - "DestinationID": { - "type": "integer" - }, - "FilterTime": { - "type": "integer" - }, - "ID": { - "type": "integer" - }, - "IsClickTracked": { - "type": "boolean" - }, - "IsHTMLPartIncluded": { - "type": "boolean" - }, - "IsOpenTracked": { - "type": "boolean" - }, - "IsTextPartIncluded": { - "type": "boolean" - }, - "IsUnsubTracked": { - "type": "string" - }, - "MessageSize": { - "type": "integer" - }, - "SenderID": { - "type": "integer" - }, - "SpamassassinScore": { - "type": "integer" - }, - "SpamassRules": { - "type": "string" - }, - "StatePermanent": { - "type": "string" - }, - "Status": { - "type": "string" - }, - "Subject": { - "type": "string" - }, - "UUID": { - "type": "string" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ArrivedAt": { + "type": "string" + }, + "AttachmentCount": { + "type": "integer" + }, + "AttemptCount": { + "type": "integer" + }, + "ContactAlt": { + "type": "string" + }, + "ContactID": { + "type": "integer" + }, + "Delay": { + "type": "integer" + }, + "DestinationID": { + "type": "integer" + }, + "FilterTime": { + "type": "integer" + }, + "ID": { + "type": "integer" + }, + "IsClickTracked": { + "type": "boolean" + }, + "IsHTMLPartIncluded": { + "type": "boolean" + }, + "IsOpenTracked": { + "type": "boolean" + }, + "IsTextPartIncluded": { + "type": "boolean" + }, + "IsUnsubTracked": { + "type": "string" + }, + "MessageSize": { + "type": "integer" + }, + "SenderID": { + "type": "integer" + }, + "SpamassassinScore": { + "type": "integer" + }, + "SpamassRules": { + "type": "string" + }, + "StatePermanent": { + "type": "string" + }, + "Status": { + "type": "string" + }, + "Subject": { + "type": "string" + }, + "UUID": { + "type": "string" } } - \ No newline at end of file +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats_api_lifetime_message.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats_api_lifetime_message.json index c2fe4a0a8f20..173f933e39a5 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats_api_lifetime_message.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats_api_lifetime_message.json @@ -1,69 +1,69 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "APIKeyID": { - "type": "integer" - }, - "EventClickDelay": { - "type": "integer" - }, - "EventClickedCount": { - "type": "integer" - }, - "EventOpenDelay": { - "type": "integer" - }, - "EventOpenedCount": { - "type": "integer" - }, - "EventUnsubscribedCount": { - "type": "integer" - }, - "EventWorkflowExitedCount": { - "type": "integer" - }, - "MessageBlockedCount": { - "type": "integer" - }, - "MessageClickedCount": { - "type": "integer" - }, - "MessageDeferredCount": { - "type": "integer" - }, - "MessageHardBouncedCount": { - "type": "integer" - }, - "MessageOpenedCount": { - "type": "integer" - }, - "MessageQueuedCount": { - "type": "integer" - }, - "MessageSentCount": { - "type": "integer" - }, - "MessageSoftBouncedCount": { - "type": "integer" - }, - "MessageSpamCount": { - "type": "integer" - }, - "MessageUnsubscribedCount": { - "type": "integer" - }, - "MessageWorkFlowExitedCount": { - "type": "integer" - }, - "SourceID": { - "type": "integer" - }, - "Timeslice": { - "type": "string" - }, - "Total": { - "type": "integer" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "APIKeyID": { + "type": "integer" + }, + "EventClickDelay": { + "type": "integer" + }, + "EventClickedCount": { + "type": "integer" + }, + "EventOpenDelay": { + "type": "integer" + }, + "EventOpenedCount": { + "type": "integer" + }, + "EventUnsubscribedCount": { + "type": "integer" + }, + "EventWorkflowExitedCount": { + "type": "integer" + }, + "MessageBlockedCount": { + "type": "integer" + }, + "MessageClickedCount": { + "type": "integer" + }, + "MessageDeferredCount": { + "type": "integer" + }, + "MessageHardBouncedCount": { + "type": "integer" + }, + "MessageOpenedCount": { + "type": "integer" + }, + "MessageQueuedCount": { + "type": "integer" + }, + "MessageSentCount": { + "type": "integer" + }, + "MessageSoftBouncedCount": { + "type": "integer" + }, + "MessageSpamCount": { + "type": "integer" + }, + "MessageUnsubscribedCount": { + "type": "integer" + }, + "MessageWorkFlowExitedCount": { + "type": "integer" + }, + "SourceID": { + "type": "integer" + }, + "Timeslice": { + "type": "string" + }, + "Total": { + "type": "integer" } - } \ No newline at end of file + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml index a8c73c71b031..14a819f952ff 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml @@ -1,4 +1,4 @@ -documentationUrl: https://docsurl.com +documentationUrl: https://dev.mailjet.com/email/guides/ connectionSpecification: $schema: http://json-schema.org/draft-07/schema# title: Mailjet Mail Spec From ebb3f6ce1a57def2495f7768c10cc3e5de21bea8 Mon Sep 17 00:00:00 2001 From: Haithem SOUALA Date: Tue, 25 Oct 2022 15:45:17 +0200 Subject: [PATCH 10/19] lint --- frontend.code-workspace | 71 ----------------------------------------- settings.json | 45 -------------------------- 2 files changed, 116 deletions(-) delete mode 100644 frontend.code-workspace delete mode 100644 settings.json diff --git a/frontend.code-workspace b/frontend.code-workspace deleted file mode 100644 index c55ade0551d9..000000000000 --- a/frontend.code-workspace +++ /dev/null @@ -1,71 +0,0 @@ -{ - "folders": [ - { - "path": "../airbyte-webapp" - }, - { - "path": "../airbyte-webapp-e2e-tests" - }, - { - "name": "docs/integrations/destinations", - "path": "../docs/integrations/destinations" - }, - { - "name": "docs/integrations/sources", - "path": "../docs/integrations/sources" - } - ], - "extensions": { - "recommendations": [ - "dbaeumer.vscode-eslint", - "stylelint.vscode-stylelint", - "esbenp.prettier-vscode", - "eamodio.gitlens", - ] - }, - "settings": { - "javascript.preferences.quoteStyle": "double", - "typescript.preferences.quoteStyle": "double", - "javascript.preferences.importModuleSpecifier": "shortest", - "typescript.preferences.importModuleSpecifier": "shortest", - "javascript.updateImportsOnFileMove.enabled": "always", - "typescript.updateImportsOnFileMove.enabled": "always", - "editor.detectIndentation": true, - "eslint.format.enable": true, - "eslint.run": "onType", - "stylelint.enable": true, - "stylelint.validate": ["css", "scss"], - "[javascript]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "dbaeumer.vscode-eslint", - "editor.codeActionsOnSave": { - "source.organizeImports": false - } - }, - "[typescript]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "dbaeumer.vscode-eslint", - "editor.codeActionsOnSave": { - "source.organizeImports": false - } - }, - "[typescriptreact]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "dbaeumer.vscode-eslint", - "editor.codeActionsOnSave": { - "source.organizeImports": false - } - }, - "[json]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "esbenp.prettier-vscode" - }, - "[scss]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "esbenp.prettier-vscode", - "editor.codeActionsOnSave": { - "source.fixAll.stylelint": true - } - } - } -} diff --git a/settings.json b/settings.json deleted file mode 100644 index a328b59d4c58..000000000000 --- a/settings.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "javascript.preferences.quoteStyle": "double", - "typescript.preferences.quoteStyle": "double", - "javascript.preferences.importModuleSpecifier": "shortest", - "typescript.preferences.importModuleSpecifier": "shortest", - "javascript.updateImportsOnFileMove.enabled": "always", - "typescript.updateImportsOnFileMove.enabled": "always", - "editor.detectIndentation": true, - "eslint.format.enable": true, - "eslint.run": "onType", - "stylelint.enable": true, - "stylelint.validate": ["css", "scss"], - "[javascript]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "dbaeumer.vscode-eslint", - "editor.codeActionsOnSave": { - "source.organizeImports": false - } - }, - "[typescript]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "dbaeumer.vscode-eslint", - "editor.codeActionsOnSave": { - "source.organizeImports": false - } - }, - "[typescriptreact]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "dbaeumer.vscode-eslint", - "editor.codeActionsOnSave": { - "source.organizeImports": false - } - }, - "[json]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "vscode.json-language-features" - }, - "[scss]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "esbenp.prettier-vscode", - "editor.codeActionsOnSave": { - "source.fixAll.stylelint": true - } - } -} From b535245fc80361d6753d3c6a33e1f3c22c189303 Mon Sep 17 00:00:00 2001 From: Haithem SOUALA Date: Tue, 25 Oct 2022 15:46:21 +0200 Subject: [PATCH 11/19] lint --- .vscode/frontend.code-workspace | 71 +++++++++++++++++++++++++++++++++ .vscode/settings.json | 45 +++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 .vscode/frontend.code-workspace create mode 100644 .vscode/settings.json diff --git a/.vscode/frontend.code-workspace b/.vscode/frontend.code-workspace new file mode 100644 index 000000000000..c55ade0551d9 --- /dev/null +++ b/.vscode/frontend.code-workspace @@ -0,0 +1,71 @@ +{ + "folders": [ + { + "path": "../airbyte-webapp" + }, + { + "path": "../airbyte-webapp-e2e-tests" + }, + { + "name": "docs/integrations/destinations", + "path": "../docs/integrations/destinations" + }, + { + "name": "docs/integrations/sources", + "path": "../docs/integrations/sources" + } + ], + "extensions": { + "recommendations": [ + "dbaeumer.vscode-eslint", + "stylelint.vscode-stylelint", + "esbenp.prettier-vscode", + "eamodio.gitlens", + ] + }, + "settings": { + "javascript.preferences.quoteStyle": "double", + "typescript.preferences.quoteStyle": "double", + "javascript.preferences.importModuleSpecifier": "shortest", + "typescript.preferences.importModuleSpecifier": "shortest", + "javascript.updateImportsOnFileMove.enabled": "always", + "typescript.updateImportsOnFileMove.enabled": "always", + "editor.detectIndentation": true, + "eslint.format.enable": true, + "eslint.run": "onType", + "stylelint.enable": true, + "stylelint.validate": ["css", "scss"], + "[javascript]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "dbaeumer.vscode-eslint", + "editor.codeActionsOnSave": { + "source.organizeImports": false + } + }, + "[typescript]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "dbaeumer.vscode-eslint", + "editor.codeActionsOnSave": { + "source.organizeImports": false + } + }, + "[typescriptreact]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "dbaeumer.vscode-eslint", + "editor.codeActionsOnSave": { + "source.organizeImports": false + } + }, + "[json]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[scss]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.codeActionsOnSave": { + "source.fixAll.stylelint": true + } + } + } +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000000..a328b59d4c58 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,45 @@ +{ + "javascript.preferences.quoteStyle": "double", + "typescript.preferences.quoteStyle": "double", + "javascript.preferences.importModuleSpecifier": "shortest", + "typescript.preferences.importModuleSpecifier": "shortest", + "javascript.updateImportsOnFileMove.enabled": "always", + "typescript.updateImportsOnFileMove.enabled": "always", + "editor.detectIndentation": true, + "eslint.format.enable": true, + "eslint.run": "onType", + "stylelint.enable": true, + "stylelint.validate": ["css", "scss"], + "[javascript]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "dbaeumer.vscode-eslint", + "editor.codeActionsOnSave": { + "source.organizeImports": false + } + }, + "[typescript]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "dbaeumer.vscode-eslint", + "editor.codeActionsOnSave": { + "source.organizeImports": false + } + }, + "[typescriptreact]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "dbaeumer.vscode-eslint", + "editor.codeActionsOnSave": { + "source.organizeImports": false + } + }, + "[json]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "vscode.json-language-features" + }, + "[scss]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.codeActionsOnSave": { + "source.fixAll.stylelint": true + } + } +} From adb68dfeaa82263adaec5ff9245c2247c11e7118 Mon Sep 17 00:00:00 2001 From: Haithem SOUALA Date: Tue, 25 Oct 2022 15:47:29 +0200 Subject: [PATCH 12/19] lint --- .vscode/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index a328b59d4c58..e52044ed1069 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -33,7 +33,7 @@ }, "[json]": { "editor.formatOnSave": true, - "editor.defaultFormatter": "vscode.json-language-features" + "editor.defaultFormatter": "esbenp.prettier-vscode" }, "[scss]": { "editor.formatOnSave": true, From 71851802889a8ef3e44f1c3caf3f78e96c07c3ec Mon Sep 17 00:00:00 2001 From: Haithem SOUALA Date: Tue, 25 Oct 2022 15:49:34 +0200 Subject: [PATCH 13/19] lint --- .../integration_tests/configured_catalog.json | 2 +- .../source-mailjet-mail/integration_tests/sample_config.json | 2 +- .../source_mailjet_mail/schemas/campaign.json | 2 +- .../source_mailjet_mail/schemas/contacts.json | 2 +- .../source_mailjet_mail/schemas/message.json | 2 +- .../source_mailjet_mail/schemas/stats_api_lifetime_message.json | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json index b7fdde85fbee..c4bc59d0a5f7 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json @@ -56,4 +56,4 @@ "destination_sync_mode": "overwrite" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json index b9a1d25d959d..3669694a327f 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json @@ -1,3 +1,3 @@ { "fix-me": "TODO" -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/campaign.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/campaign.json index a5b8b6045a0c..d75dcd9d447e 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/campaign.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/campaign.json @@ -69,4 +69,4 @@ "type": "integer" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contacts.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contacts.json index 58874e9f6187..aff93f074c24 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/contacts.json @@ -39,4 +39,4 @@ "type": "boolean" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/message.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/message.json index d8016dae0a5e..fed3d0d3f56d 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/message.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/message.json @@ -69,4 +69,4 @@ "type": "string" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats_api_lifetime_message.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats_api_lifetime_message.json index 173f933e39a5..fdb92709ba89 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats_api_lifetime_message.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/stats_api_lifetime_message.json @@ -66,4 +66,4 @@ "type": "integer" } } -} \ No newline at end of file +} From 09a51260060af23e698708b60145e66d6ff095ba Mon Sep 17 00:00:00 2001 From: Haithem SOUALA Date: Wed, 26 Oct 2022 10:46:42 +0200 Subject: [PATCH 14/19] fix comments --- .../source-mailjet-mail/integration_tests/sample_config.json | 3 ++- .../source-mailjet-mail/source_mailjet_mail/spec.yaml | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json index 3669694a327f..219cc2f5e0cf 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json @@ -1,3 +1,4 @@ { - "fix-me": "TODO" + "api_key": "2e2eec3e6ab140cf8fb42181773d75da", + "api_key_secret": "0d6be348aaefb42181773d75da3c7608b" } diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml index 14a819f952ff..00043e9034df 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml @@ -1,4 +1,4 @@ -documentationUrl: https://dev.mailjet.com/email/guides/ +documentationUrl: https://docs.airbyte.com/integrations/sources/mailjet-mail connectionSpecification: $schema: http://json-schema.org/draft-07/schema# title: Mailjet Mail Spec @@ -9,11 +9,13 @@ connectionSpecification: additionalProperties: true properties: api_key: + title: API Key type: string description: >- Your API Key. See here. api_key_secret: + title: API Secret Key type: string description: >- Your API Secret Key. See Date: Thu, 27 Oct 2022 23:28:41 -0300 Subject: [PATCH 15/19] run format --- .../resources/seed/source_definitions.yaml | 7 +++++++ .../integration_tests/configured_catalog.json | 20 +++++-------------- .../integration_tests/sample_config.json | 4 ++-- .../source_mailjet_mail/spec.yaml | 2 +- 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 8e3e9b1b18d3..d216fa4c3319 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -674,6 +674,13 @@ icon: mailchimp.svg sourceType: api releaseStage: generally_available +- name: Mailjet Mail + sourceDefinitionId: 56582331-5de2-476b-b913-5798de77bbdf + dockerRepository: airbyte/source-mailjet-mail + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/mailjet-mail + sourceType: api + releaseStage: alpha - name: MailerLite sourceDefinitionId: dc3b9003-2432-4e93-a7f4-4620b0f14674 dockerRepository: airbyte/source-mailerlite diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json index c4bc59d0a5f7..78d2e30219ac 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/configured_catalog.json @@ -4,9 +4,7 @@ "stream": { "name": "contactslist", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -15,9 +13,7 @@ "stream": { "name": "stats_api_lifetime_message", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -26,9 +22,7 @@ "stream": { "name": "contacts", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -37,9 +31,7 @@ "stream": { "name": "campaign", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -48,9 +40,7 @@ "stream": { "name": "message", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json index 219cc2f5e0cf..b8356ca5774f 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/sample_config.json @@ -1,4 +1,4 @@ { - "api_key": "2e2eec3e6ab140cf8fb42181773d75da", - "api_key_secret": "0d6be348aaefb42181773d75da3c7608b" + "api_key": "2e2eec3e6ab140cf8fb42181773d75da", + "api_key_secret": "0d6be348aaefb42181773d75da3c7608b" } diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml index 00043e9034df..585b8c30c43a 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/spec.yaml @@ -17,7 +17,7 @@ connectionSpecification: api_key_secret: title: API Secret Key type: string - description: >- + description: >- Your API Secret Key. See here. airbyte_secret: true From 75bf7002413e47a681bc777e76c625b14164d0d7 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 27 Oct 2022 23:40:49 -0300 Subject: [PATCH 16/19] update acceptance test --- .../connectors/source-mailjet-mail/acceptance-test-config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml index ad650b63af16..b599202d3b9a 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml @@ -14,7 +14,6 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["contactslist", "contacts", "stats_api_lifetime_message", "campaign", "message"] # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file # expect_records: # path: "integration_tests/expected_records.txt" From 549f1a2e28947edeefe2eddcfb7adf65cf9589b5 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Fri, 28 Oct 2022 00:00:02 -0300 Subject: [PATCH 17/19] update schema --- .../acceptance-test-config.yml | 24 +++++++++---------- .../source_mailjet_mail/schemas/campaign.json | 14 +++++------ .../source_mailjet_mail/schemas/message.json | 4 ++-- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml index b599202d3b9a..3222c8759c6e 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml @@ -2,15 +2,15 @@ # for more information about how to configure these tests connector_image: airbyte/source-mailjet-mail:dev tests: - spec: - - spec_path: "source_mailjet_mail/spec.yaml" - connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - discovery: - - config_path: "secrets/config.json" + # spec: + # - spec_path: "source_mailjet_mail/spec.yaml" + # connection: + # - config_path: "secrets/config.json" + # status: "succeed" + # - config_path: "integration_tests/invalid_config.json" + # status: "failed" + # discovery: + # - config_path: "secrets/config.json" basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" @@ -20,6 +20,6 @@ tests: # extra_fields: no # exact_order: no # extra_records: yes - full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + # full_refresh: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/campaign.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/campaign.json index d75dcd9d447e..f722bb731bb8 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/campaign.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/campaign.json @@ -12,25 +12,25 @@ "type": "string" }, "CustomValue": { - "type": "integer" + "type": "string" }, "FirstMessageID": { "type": "integer" }, "FromEmail": { - "type": "boolean" + "type": "string" }, "FromID": { - "type": "string" + "type": "number" }, "FromName": { "type": "string" }, "HasHtmlCount": { - "type": "string" + "type": "number" }, "HasTxtCount": { - "type": "string" + "type": "number" }, "ID": { "type": "integer" @@ -48,13 +48,13 @@ "type": "integer" }, "OpenTracked": { - "type": "string" + "type": "number" }, "SendEndAt": { "type": "string" }, "SendStartAt": { - "type": "boolean" + "type": "string" }, "SpamassScore": { "type": "integer" diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/message.json b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/message.json index fed3d0d3f56d..4f70b7c38d55 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/message.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/schemas/message.json @@ -42,7 +42,7 @@ "type": "boolean" }, "IsUnsubTracked": { - "type": "string" + "type": "boolean" }, "MessageSize": { "type": "integer" @@ -57,7 +57,7 @@ "type": "string" }, "StatePermanent": { - "type": "string" + "type": "boolean" }, "Status": { "type": "string" From c7772463669468046d5bd214fc0a4b4bd6be9074 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Fri, 28 Oct 2022 00:15:45 -0300 Subject: [PATCH 18/19] update acceptance test --- .../acceptance-test-config.yml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml index 3222c8759c6e..b599202d3b9a 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-config.yml @@ -2,15 +2,15 @@ # for more information about how to configure these tests connector_image: airbyte/source-mailjet-mail:dev tests: - # spec: - # - spec_path: "source_mailjet_mail/spec.yaml" - # connection: - # - config_path: "secrets/config.json" - # status: "succeed" - # - config_path: "integration_tests/invalid_config.json" - # status: "failed" - # discovery: - # - config_path: "secrets/config.json" + spec: + - spec_path: "source_mailjet_mail/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" @@ -20,6 +20,6 @@ tests: # extra_fields: no # exact_order: no # extra_records: yes - # full_refresh: - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" From 7f507b6fa7b0b511d1c62bd0f75057fefdb03c57 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 28 Oct 2022 03:35:43 +0000 Subject: [PATCH 19/19] auto-bump connector version --- .../src/main/resources/seed/source_specs.yaml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 60795ac457e2..e6aec0e59c85 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -6466,6 +6466,32 @@ path_in_connector_config: - "credentials" - "client_secret" +- dockerImage: "airbyte/source-mailjet-mail:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/mailjet-mail" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Mailjet Mail Spec" + type: "object" + required: + - "api_key" + - "api_key_secret" + additionalProperties: true + properties: + api_key: + title: "API Key" + type: "string" + description: "Your API Key. See here." + api_key_secret: + title: "API Secret Key" + type: "string" + description: "Your API Secret Key. See here." + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-mailerlite:0.1.0" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/mailerlite"