From bc56f02c41795b69b623ae2fecbd42e92c4ddd7a Mon Sep 17 00:00:00 2001 From: Michel Tricot Date: Fri, 16 Oct 2020 10:56:08 -0700 Subject: [PATCH] Python sources refactoring (#592) --- .gitignore | 4 + .python-version | 1 + .../base-python/.dockerignore | 5 +- airbyte-integrations/base-python/.gitignore | 1 + airbyte-integrations/base-python/Dockerfile | 21 +-- .../base-python/airbyte_protocol/__init__.py | 12 ++ .../airbyte_protocol/.gitignore | 1 - .../airbyte_protocol/__init__.py | 125 ------------------ .../entrypoint.py} | 52 ++++---- .../airbyte_protocol/integration.py | 75 +++++++++++ .../base-python/airbyte_protocol/logger.py | 40 ++++++ .../airbyte_protocol/models/__init__.py | 25 ++++ .../base-python/airbyte_protocol/setup.py | 12 -- airbyte-integrations/base-python/build.gradle | 7 +- airbyte-integrations/base-python/main_dev.py | 4 + .../base-python/requirements.txt | 1 + airbyte-integrations/base-python/setup.py | 24 ++++ .../singer/base-singer/.dockerignore | 6 +- .../singer/base-singer/Dockerfile | 7 +- .../base-singer/base_singer/singer_helpers.py | 18 +-- .../singer/base-singer/build.gradle | 1 + .../singer/base-singer/requirements.txt | 2 + .../singer/base-singer/setup.py | 10 +- .../exchangeratesapi/source/.dockerignore | 8 +- .../singer/exchangeratesapi/source/Dockerfile | 24 ++-- .../exchangeratesapi/source/build.gradle | 2 +- .../exchangeratesapi/source/main_dev.py | 8 ++ .../exchangeratesapi/source/requirements.txt | 3 + .../singer/exchangeratesapi/source/setup.py | 17 ++- .../__init__.py | 2 +- ...e_exchangeratesapi_singer.py => source.py} | 18 ++- .../spec.json | 0 ...erExchangeRatesApiSourceDataModelTest.java | 2 +- .../stripe_abprotocol/source/.dockerignore | 8 +- .../stripe_abprotocol/source/Dockerfile | 24 ++-- .../stripe_abprotocol/source/main_dev.py | 8 ++ .../stripe_abprotocol/source/requirements.txt | 3 + .../singer/stripe_abprotocol/source/setup.py | 13 +- .../source/source_stripe_singer/__init__.py | 2 +- .../{source_stripe_singer.py => source.py} | 16 +-- .../{ => source_stripe_singer}/spec.json | 0 .../template/python-source/.dockerignore | 1 + .../template/python-source/Dockerfile | 15 +++ .../template/python-source/README.md | 31 +++++ .../template/python-source/build.gradle | 3 + .../template/python-source/main_dev.py | 9 ++ .../template/python-source/requirements.txt | 2 + .../sample_files/test_catalog.json | 16 +++ .../sample_files/test_config.json | 3 + .../template/python-source/setup.py | 15 +++ .../template_python_source/__init__.py | 1 + .../template_python_source/catalog.json | 16 +++ .../template_python_source/source.py | 36 +++++ .../template_python_source/spec.json | 17 +++ .../template/singer-source/.dockerignore | 1 + .../template/singer-source/Dockerfile | 19 +++ .../template/singer-source/README.md | 1 + .../template/singer-source/build.gradle | 5 + .../template/singer-source/main_dev.py | 8 ++ .../template/singer-source/requirements.txt | 3 + .../template/singer-source/setup.py | 19 +++ .../template_singer_source/__init__.py | 1 + .../template_singer_source/source.py | 28 ++++ .../template_singer_source/spec.json | 21 +++ build.gradle | 2 +- 65 files changed, 606 insertions(+), 279 deletions(-) create mode 100644 .python-version create mode 100644 airbyte-integrations/base-python/.gitignore create mode 100644 airbyte-integrations/base-python/airbyte_protocol/__init__.py delete mode 100644 airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/.gitignore delete mode 100644 airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py rename airbyte-integrations/base-python/{base.py => airbyte_protocol/entrypoint.py} (74%) create mode 100644 airbyte-integrations/base-python/airbyte_protocol/integration.py create mode 100644 airbyte-integrations/base-python/airbyte_protocol/logger.py create mode 100644 airbyte-integrations/base-python/airbyte_protocol/models/__init__.py delete mode 100644 airbyte-integrations/base-python/airbyte_protocol/setup.py create mode 100644 airbyte-integrations/base-python/main_dev.py create mode 100644 airbyte-integrations/base-python/requirements.txt create mode 100644 airbyte-integrations/base-python/setup.py create mode 100644 airbyte-integrations/singer/base-singer/requirements.txt create mode 100644 airbyte-integrations/singer/exchangeratesapi/source/main_dev.py create mode 100644 airbyte-integrations/singer/exchangeratesapi/source/requirements.txt rename airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/{source_exchangeratesapi_singer.py => source.py} (73%) rename airbyte-integrations/singer/exchangeratesapi/source/{ => source_exchangeratesapi_singer}/spec.json (100%) create mode 100644 airbyte-integrations/singer/stripe_abprotocol/source/main_dev.py create mode 100644 airbyte-integrations/singer/stripe_abprotocol/source/requirements.txt rename airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/{source_stripe_singer.py => source.py} (86%) rename airbyte-integrations/singer/stripe_abprotocol/source/{ => source_stripe_singer}/spec.json (100%) create mode 100644 airbyte-integrations/template/python-source/.dockerignore create mode 100644 airbyte-integrations/template/python-source/Dockerfile create mode 100644 airbyte-integrations/template/python-source/README.md create mode 100644 airbyte-integrations/template/python-source/build.gradle create mode 100644 airbyte-integrations/template/python-source/main_dev.py create mode 100644 airbyte-integrations/template/python-source/requirements.txt create mode 100644 airbyte-integrations/template/python-source/sample_files/test_catalog.json create mode 100644 airbyte-integrations/template/python-source/sample_files/test_config.json create mode 100644 airbyte-integrations/template/python-source/setup.py create mode 100644 airbyte-integrations/template/python-source/template_python_source/__init__.py create mode 100644 airbyte-integrations/template/python-source/template_python_source/catalog.json create mode 100644 airbyte-integrations/template/python-source/template_python_source/source.py create mode 100644 airbyte-integrations/template/python-source/template_python_source/spec.json create mode 100644 airbyte-integrations/template/singer-source/.dockerignore create mode 100644 airbyte-integrations/template/singer-source/Dockerfile create mode 100644 airbyte-integrations/template/singer-source/README.md create mode 100644 airbyte-integrations/template/singer-source/build.gradle create mode 100644 airbyte-integrations/template/singer-source/main_dev.py create mode 100644 airbyte-integrations/template/singer-source/requirements.txt create mode 100644 airbyte-integrations/template/singer-source/setup.py create mode 100644 airbyte-integrations/template/singer-source/template_singer_source/__init__.py create mode 100644 airbyte-integrations/template/singer-source/template_singer_source/source.py create mode 100644 airbyte-integrations/template/singer-source/template_singer_source/spec.json diff --git a/.gitignore b/.gitignore index ab0b9b93ef0fb..3b729f527b65d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,7 @@ build !tools/build .DS_Store data + +# Python +*.egg-info +__pycache__ diff --git a/.python-version b/.python-version new file mode 100644 index 0000000000000..c77a7de85cc88 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.7.9 diff --git a/airbyte-integrations/base-python/.dockerignore b/airbyte-integrations/base-python/.dockerignore index d73922a6fabd3..378eac25d3117 100644 --- a/airbyte-integrations/base-python/.dockerignore +++ b/airbyte-integrations/base-python/.dockerignore @@ -1,4 +1 @@ -* -!Dockerfile -!base.py -!airbyte_protocol +build diff --git a/airbyte-integrations/base-python/.gitignore b/airbyte-integrations/base-python/.gitignore new file mode 100644 index 0000000000000..f332debf15a65 --- /dev/null +++ b/airbyte-integrations/base-python/.gitignore @@ -0,0 +1 @@ +airbyte_protocol/models/yaml diff --git a/airbyte-integrations/base-python/Dockerfile b/airbyte-integrations/base-python/Dockerfile index 149006e89309c..80958e492762e 100644 --- a/airbyte-integrations/base-python/Dockerfile +++ b/airbyte-integrations/base-python/Dockerfile @@ -1,22 +1,15 @@ FROM python:3.7-slim COPY --from=airbyte/integration-base:dev /airbyte /airbyte -WORKDIR /airbyte -ENV VIRTUAL_ENV=/airbyte/env -RUN python -m venv $VIRTUAL_ENV -ENV PATH="$VIRTUAL_ENV/bin:$PATH" - -WORKDIR /airbyte/airbyte_protocol -COPY airbyte_protocol . +WORKDIR /airbyte/base_python_code +COPY airbyte_protocol ./airbyte_protocol +COPY setup.py ./ RUN pip install . -WORKDIR /airbyte/base-python -COPY base.py . - -ENV AIRBYTE_SPEC_CMD "python3 /airbyte/base-python/base.py spec" -ENV AIRBYTE_CHECK_CMD "python3 /airbyte/base-python/base.py check" -ENV AIRBYTE_DISCOVER_CMD "python3 /airbyte/base-python/base.py discover" -ENV AIRBYTE_READ_CMD "python3 /airbyte/base-python/base.py read" +ENV AIRBYTE_SPEC_CMD "base-python spec" +ENV AIRBYTE_CHECK_CMD "base-python check" +ENV AIRBYTE_DISCOVER_CMD "base-python discover" +ENV AIRBYTE_READ_CMD "base-python read" ENTRYPOINT ["/airbyte/base.sh"] diff --git a/airbyte-integrations/base-python/airbyte_protocol/__init__.py b/airbyte-integrations/base-python/airbyte_protocol/__init__.py new file mode 100644 index 0000000000000..41b47cf10a89a --- /dev/null +++ b/airbyte-integrations/base-python/airbyte_protocol/__init__.py @@ -0,0 +1,12 @@ +from .integration import * +from .logger import AirbyteLogger +from .models import AirbyteCatalog +from .models import AirbyteLogMessage +from .models import AirbyteMessage +from .models import AirbyteRecordMessage +from .models import AirbyteStateMessage +from .models import AirbyteStream + +# Must be the last one because the way we load the connector module creates a circular +# dependency and models might not have been loaded yet +from .entrypoint import AirbyteEntrypoint diff --git a/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/.gitignore b/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/.gitignore deleted file mode 100644 index 64dba6c418546..0000000000000 --- a/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/.gitignore +++ /dev/null @@ -1 +0,0 @@ -types diff --git a/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py b/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py deleted file mode 100644 index 816db9eb18944..0000000000000 --- a/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py +++ /dev/null @@ -1,125 +0,0 @@ -from typing import Generator -import yaml -import json -import pkgutil -import warnings -import python_jsonschema_objects as pjs -from dataclasses import dataclass - - -def _load_classes(yaml_path: str): - data = yaml.load(pkgutil.get_data(__name__, yaml_path), Loader=yaml.FullLoader) - builder = pjs.ObjectBuilder(data) - return builder.build_classes(standardize_names=False) - - -# hide json schema version warnings -with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=UserWarning) - message_classes = _load_classes("types/airbyte_message.yaml") - AirbyteMessage = message_classes.AirbyteMessage - AirbyteLogMessage = message_classes.AirbyteLogMessage - AirbyteRecordMessage = message_classes.AirbyteRecordMessage - AirbyteStateMessage = message_classes.AirbyteStateMessage - - catalog_classes = _load_classes("types/airbyte_catalog.yaml") - AirbyteCatalog = catalog_classes.AirbyteCatalog - AirbyteStream = catalog_classes.AirbyteStream - - -class AirbyteSpec(object): - def __init__(self, spec_string): - self.spec_string = spec_string - - -class AirbyteCheckResponse(object): - def __init__(self, successful, field_to_error): - self.successful = successful - self.field_to_error = field_to_error - - -class Integration(object): - def __init__(self): - pass - - def spec(self) -> AirbyteSpec: - raise Exception("Not Implemented") - - def read_config(self, config_path): - with open(config_path, 'r') as file: - contents = file.read() - return json.loads(contents) - - # can be overridden to change an input file config - def transform_config(self, raw_config): - return raw_config - - def write_config(self, config_object, path): - with open(path, 'w') as fh: - fh.write(json.dumps(config_object)) - - def check(self, logger, config_container) -> AirbyteCheckResponse: - raise Exception("Not Implemented") - - def discover(self, logger, config_container) -> AirbyteCatalog: - raise Exception("Not Implemented") - - -class Source(Integration): - def __init__(self): - pass - - # Iterator - def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: - raise Exception("Not Implemented") - - -class Destination(Integration): - def __init__(self): - pass - - -class AirbyteLogger: - def __init__(self): - self.valid_log_types = ["FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE"] - - def log_by_prefix(self, message, default_level): - split_line = message.split() - first_word = next(iter(split_line), None) - if first_word in self.valid_log_types: - log_level = first_word - rendered_message = " ".join(split_line[1:]) - else: - log_level = default_level - rendered_message = message - self.log(log_level, rendered_message) - - def log(self, level, message): - log_record = AirbyteLogMessage(level=level, message=message) - log_message = AirbyteMessage(type="LOG", log=log_record) - print(log_message.serialize()) - - def fatal(self, message): - self.log("FATAL", message) - - def error(self, message): - self.log("ERROR", message) - - def warn(self, message): - self.log("WARN", message) - - def info(self, message): - self.log("INFO", message) - - def debug(self, message): - self.log("DEBUG", message) - - def trace(self, message): - self.log("TRACE", message) - -@dataclass -class ConfigContainer: - raw_config: object - rendered_config: object - raw_config_path: str - rendered_config_path: str diff --git a/airbyte-integrations/base-python/base.py b/airbyte-integrations/base-python/airbyte_protocol/entrypoint.py similarity index 74% rename from airbyte-integrations/base-python/base.py rename to airbyte-integrations/base-python/airbyte_protocol/entrypoint.py index 86e5f32a53ad6..84ae91df93500 100644 --- a/airbyte-integrations/base-python/base.py +++ b/airbyte-integrations/base-python/airbyte_protocol/entrypoint.py @@ -1,28 +1,25 @@ import argparse +import importlib +import os.path import sys import tempfile -import os.path -import importlib -from airbyte_protocol import ConfigContainer -from airbyte_protocol import Source -from airbyte_protocol import AirbyteLogger -from airbyte_protocol import AirbyteLogMessage -from airbyte_protocol import AirbyteMessage - -impl_module = os.environ['AIRBYTE_IMPL_MODULE'] -impl_class = os.environ['AIRBYTE_IMPL_PATH'] +from .integration import ConfigContainer, Source +from .logger import AirbyteLogger +impl_module = os.environ.get('AIRBYTE_IMPL_MODULE', Source.__module__) +impl_class = os.environ.get('AIRBYTE_IMPL_PATH', Source.__name__) module = importlib.import_module(impl_module) impl = getattr(module, impl_class) logger = AirbyteLogger() + class AirbyteEntrypoint(object): def __init__(self, source): self.source = source - def start(self): + def start(self, args): # set up parent parsers parent_parser = argparse.ArgumentParser(add_help=False) main_parser = argparse.ArgumentParser() @@ -57,23 +54,25 @@ def start(self): help='path to the catalog used to determine which data to read') # parse the args - parsed_args = main_parser.parse_args() + parsed_args = main_parser.parse_args(args) # execute cmd = parsed_args.command + if not cmd: + raise Exception("No command passed") # todo: add try catch for exceptions with different exit codes with tempfile.TemporaryDirectory() as temp_dir: if cmd == "spec": # todo: output this as a JSON formatted message - print(source.spec().spec_string) + print(self.source.spec(logger).spec_string) sys.exit(0) rendered_config_path = os.path.join(temp_dir, 'config.json') - raw_config = source.read_config(parsed_args.config) - rendered_config = source.transform_config(raw_config) - source.write_config(rendered_config, rendered_config_path) + raw_config = self.source.read_config(parsed_args.config) + rendered_config = self.source.transform_config(raw_config) + self.source.write_config(rendered_config, rendered_config_path) config_container = ConfigContainer( raw_config=raw_config, @@ -82,7 +81,7 @@ def start(self): rendered_config_path=rendered_config_path) if cmd == "check": - check_result = source.check(logger, config_container) + check_result = self.source.check(logger, config_container) if check_result.successful: logger.info("Check succeeded") sys.exit(0) @@ -90,11 +89,11 @@ def start(self): logger.error("Check failed") sys.exit(1) elif cmd == "discover": - catalog = source.discover(logger, config_container) + catalog = self.source.discover(logger, config_container) print(catalog.serialize()) sys.exit(0) elif cmd == "read": - generator = source.read(logger, config_container, parsed_args.catalog, parsed_args.state) + generator = self.source.read(logger, config_container, parsed_args.catalog, parsed_args.state) for message in generator: print(message.serialize()) sys.exit(0) @@ -102,10 +101,15 @@ def start(self): raise Exception("Unexpected command " + cmd) -# set up and run entrypoint -source = impl() +def launch(source, args): + AirbyteEntrypoint(source).start(args) + + +def main(): + # set up and run entrypoint + source = impl() -if not isinstance(source, Source): - raise Exception("Source implementation provided does not implement Source class!") + if not isinstance(source, Source): + raise Exception("Source implementation provided does not implement Source class!") -AirbyteEntrypoint(source).start() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/base-python/airbyte_protocol/integration.py b/airbyte-integrations/base-python/airbyte_protocol/integration.py new file mode 100644 index 0000000000000..be12354581061 --- /dev/null +++ b/airbyte-integrations/base-python/airbyte_protocol/integration.py @@ -0,0 +1,75 @@ +import json +import pkgutil +from dataclasses import dataclass +from typing import Generator + +from .models import AirbyteCatalog, AirbyteMessage + + +class AirbyteSpec(object): + @staticmethod + def from_file(file): + with open(file) as file: + spec_text = file.read() + return AirbyteSpec(spec_text) + + def __init__(self, spec_string): + self.spec_string = spec_string + + +class AirbyteCheckResponse(object): + def __init__(self, successful, field_to_error): + self.successful = successful + self.field_to_error = field_to_error + + +@dataclass +class ConfigContainer: + raw_config: object + rendered_config: object + raw_config_path: str + rendered_config_path: str + + +class Integration(object): + def __init__(self): + pass + + def spec(self, logger) -> AirbyteSpec: + raw_spec = pkgutil.get_data(self.__class__.__module__.split('.')[0], 'spec.json') + # we need to output a spec on a single line + flattened_json = json.dumps(json.loads(raw_spec)) + return AirbyteSpec(flattened_json) + + def read_config(self, config_path): + with open(config_path, 'r') as file: + contents = file.read() + return json.loads(contents) + + # can be overridden to change an input file config + def transform_config(self, raw_config): + return raw_config + + def write_config(self, config_object, path): + with open(path, 'w') as fh: + fh.write(json.dumps(config_object)) + + def check(self, logger, config_container) -> AirbyteCheckResponse: + raise Exception("Not Implemented") + + def discover(self, logger, config_container) -> AirbyteCatalog: + raise Exception("Not Implemented") + + +class Source(Integration): + def __init__(self): + super().__init__() + + # Iterator + def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: + raise Exception("Not Implemented") + + +class Destination(Integration): + def __init__(self): + super().__init__() diff --git a/airbyte-integrations/base-python/airbyte_protocol/logger.py b/airbyte-integrations/base-python/airbyte_protocol/logger.py new file mode 100644 index 0000000000000..4c713eedb2e8b --- /dev/null +++ b/airbyte-integrations/base-python/airbyte_protocol/logger.py @@ -0,0 +1,40 @@ +from .models import AirbyteLogMessage, AirbyteMessage + + +class AirbyteLogger: + def __init__(self): + self.valid_log_types = ["FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE"] + + def log_by_prefix(self, message, default_level): + split_line = message.split() + first_word = next(iter(split_line), None) + if first_word in self.valid_log_types: + log_level = first_word + rendered_message = " ".join(split_line[1:]) + else: + log_level = default_level + rendered_message = message + self.log(log_level, rendered_message) + + def log(self, level, message): + log_record = AirbyteLogMessage(level=level, message=message) + log_message = AirbyteMessage(type="LOG", log=log_record) + print(log_message.serialize()) + + def fatal(self, message): + self.log("FATAL", message) + + def error(self, message): + self.log("ERROR", message) + + def warn(self, message): + self.log("WARN", message) + + def info(self, message): + self.log("INFO", message) + + def debug(self, message): + self.log("DEBUG", message) + + def trace(self, message): + self.log("TRACE", message) diff --git a/airbyte-integrations/base-python/airbyte_protocol/models/__init__.py b/airbyte-integrations/base-python/airbyte_protocol/models/__init__.py new file mode 100644 index 0000000000000..627f7b0aae9b3 --- /dev/null +++ b/airbyte-integrations/base-python/airbyte_protocol/models/__init__.py @@ -0,0 +1,25 @@ +import pkgutil +import warnings + +import python_jsonschema_objects as pjs +import yaml + + +def _load_classes(yaml_path: str): + data = yaml.load(pkgutil.get_data(__name__, yaml_path), Loader=yaml.FullLoader) + builder = pjs.ObjectBuilder(data) + return builder.build_classes(standardize_names=False) + + +# hide json schema version warnings +with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning) + message_classes = _load_classes("yaml/airbyte_message.yaml") + AirbyteMessage = message_classes.AirbyteMessage + AirbyteLogMessage = message_classes.AirbyteLogMessage + AirbyteRecordMessage = message_classes.AirbyteRecordMessage + AirbyteStateMessage = message_classes.AirbyteStateMessage + + catalog_classes = _load_classes("yaml/airbyte_catalog.yaml") + AirbyteCatalog = catalog_classes.AirbyteCatalog + AirbyteStream = catalog_classes.AirbyteStream diff --git a/airbyte-integrations/base-python/airbyte_protocol/setup.py b/airbyte-integrations/base-python/airbyte_protocol/setup.py deleted file mode 100644 index c591c46461b06..0000000000000 --- a/airbyte-integrations/base-python/airbyte_protocol/setup.py +++ /dev/null @@ -1,12 +0,0 @@ -from setuptools import setup - -setup( - name='airbyte_protocol', - description='Contains classes representing the schema of the Airbyte protocol.', - author='Airbyte', - author_email='contact@airbyte.io', - packages=['airbyte_protocol'], - install_requires=['PyYAML==5.3.1', 'python-jsonschema-objects==0.3.13'], - package_data={'': ['types/*.yaml']}, - include_package_data=True -) diff --git a/airbyte-integrations/base-python/build.gradle b/airbyte-integrations/base-python/build.gradle index 4ad23f3d3e791..3abb915df4a94 100644 --- a/airbyte-integrations/base-python/build.gradle +++ b/airbyte-integrations/base-python/build.gradle @@ -1,15 +1,16 @@ apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') +def typesDir = 'airbyte_protocol/models/yaml' task deleteProtocolDefinitions(type: Delete) { - delete 'airbyte_protocol/airbyte_protocol/types' + delete typesDir } task copyProtocolDefinitions(type: Copy) { from file("$rootDir/airbyte-protocol/models/src/main/resources/airbyte_protocol").absolutePath - into "airbyte_protocol/airbyte_protocol/types" + into typesDir dependsOn deleteProtocolDefinitions } +assemble.dependsOn copyProtocolDefinitions clean.dependsOn deleteProtocolDefinitions -buildImage.dependsOn copyProtocolDefinitions buildImage.dependsOn ':airbyte-integrations:base:buildImage' diff --git a/airbyte-integrations/base-python/main_dev.py b/airbyte-integrations/base-python/main_dev.py new file mode 100644 index 0000000000000..6d2c0c60d6e9d --- /dev/null +++ b/airbyte-integrations/base-python/main_dev.py @@ -0,0 +1,4 @@ +from airbyte_protocol.entrypoint import main + +if __name__ == "__main__": + main() diff --git a/airbyte-integrations/base-python/requirements.txt b/airbyte-integrations/base-python/requirements.txt new file mode 100644 index 0000000000000..d6e1198b1ab1f --- /dev/null +++ b/airbyte-integrations/base-python/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/airbyte-integrations/base-python/setup.py b/airbyte-integrations/base-python/setup.py new file mode 100644 index 0000000000000..9c8bc93632bf3 --- /dev/null +++ b/airbyte-integrations/base-python/setup.py @@ -0,0 +1,24 @@ +import setuptools + +setuptools.setup( + name='airbyte-protocol', + description='Contains classes representing the schema of the Airbyte protocol.', + author='Airbyte', + author_email='contact@airbyte.io', + url='https://github.com/airbytehq/airbyte', + + packages=setuptools.find_packages(), + package_data={ + '': ['models/yaml/*.yaml'] + }, + + install_requires=[ + 'PyYAML==5.3.1', + 'python-jsonschema-objects==0.3.13' + ], + entry_points={ + 'console_scripts': [ + 'base-python=airbyte_protocol.entrypoint:main' + ], + } +) diff --git a/airbyte-integrations/singer/base-singer/.dockerignore b/airbyte-integrations/singer/base-singer/.dockerignore index a00d2e2805833..378eac25d3117 100644 --- a/airbyte-integrations/singer/base-singer/.dockerignore +++ b/airbyte-integrations/singer/base-singer/.dockerignore @@ -1,5 +1 @@ -* -!Dockerfile -!base_singer/__init__.py -!base_singer/singer_helpers.py -!setup.py +build diff --git a/airbyte-integrations/singer/base-singer/Dockerfile b/airbyte-integrations/singer/base-singer/Dockerfile index 63fbf556c78c0..c9918cbf156a6 100644 --- a/airbyte-integrations/singer/base-singer/Dockerfile +++ b/airbyte-integrations/singer/base-singer/Dockerfile @@ -1,9 +1,8 @@ FROM airbyte/integration-base-python:dev -WORKDIR /airbyte/base_singer -COPY base_singer/__init__.py base_singer/__init__.py -COPY base_singer/singer_helpers.py base_singer/singer_helpers.py -COPY setup.py . +WORKDIR /airbyte/base_singer_code +COPY base_singer ./base_singer +COPY setup.py ./ RUN pip install . LABEL io.airbyte.version=0.1.0 diff --git a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py index df665fd33807d..5df307ca8348c 100644 --- a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py +++ b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py @@ -3,16 +3,16 @@ import selectors import subprocess import tempfile -from airbyte_protocol import AirbyteSpec +from dataclasses import dataclass +from datetime import datetime +from typing import Generator + from airbyte_protocol import AirbyteCatalog from airbyte_protocol import AirbyteMessage -from airbyte_protocol import AirbyteLogMessage from airbyte_protocol import AirbyteRecordMessage +from airbyte_protocol import AirbyteSpec from airbyte_protocol import AirbyteStateMessage from airbyte_protocol import AirbyteStream -from typing import Generator -from datetime import datetime -from dataclasses import dataclass def to_json(string): @@ -36,14 +36,6 @@ class Catalogs: class SingerHelper: - @staticmethod - def spec_from_file(spec_path) -> AirbyteSpec: - with open(spec_path) as file: - spec_text = file.read() - # we need to output a spec on a single line - flattened_json = json.dumps(json.loads(spec_text)) - return AirbyteSpec(flattened_json) - @staticmethod def get_catalogs(logger, shell_command, singer_transform=(lambda catalog: catalog), airbyte_transform=(lambda catalog: catalog)) -> Catalogs: completed_process = subprocess.run(shell_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/airbyte-integrations/singer/base-singer/build.gradle b/airbyte-integrations/singer/base-singer/build.gradle index f2766bdd3d791..ed1a036fcb429 100644 --- a/airbyte-integrations/singer/base-singer/build.gradle +++ b/airbyte-integrations/singer/base-singer/build.gradle @@ -1,3 +1,4 @@ apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') + buildImage.dependsOn ':airbyte-integrations:base-python:buildImage' diff --git a/airbyte-integrations/singer/base-singer/requirements.txt b/airbyte-integrations/singer/base-singer/requirements.txt new file mode 100644 index 0000000000000..d92984a47ae4b --- /dev/null +++ b/airbyte-integrations/singer/base-singer/requirements.txt @@ -0,0 +1,2 @@ +-e ../../base-python +-e . diff --git a/airbyte-integrations/singer/base-singer/setup.py b/airbyte-integrations/singer/base-singer/setup.py index 7873c5e9ca3ec..9b047a0917181 100644 --- a/airbyte-integrations/singer/base-singer/setup.py +++ b/airbyte-integrations/singer/base-singer/setup.py @@ -1,10 +1,12 @@ -from setuptools import setup +from setuptools import setup, find_packages setup( - name='base_singer', + name='base-singer', description='Contains helpers for handling Singer sources and destinations.', author='Airbyte', author_email='contact@airbyte.io', - packages=['base_singer'], - install_requires=['airbyte_protocol'] + + packages=find_packages(), + + install_requires=['airbyte-protocol'] ) diff --git a/airbyte-integrations/singer/exchangeratesapi/source/.dockerignore b/airbyte-integrations/singer/exchangeratesapi/source/.dockerignore index fa354df6ca00d..378eac25d3117 100644 --- a/airbyte-integrations/singer/exchangeratesapi/source/.dockerignore +++ b/airbyte-integrations/singer/exchangeratesapi/source/.dockerignore @@ -1,7 +1 @@ -* -!Dockerfile -!entrypoint.sh -!source_exchangeratesapi_singer/source_exchangeratesapi_singer.py -!source_exchangeratesapi_singer/__init__.py -!spec.json -!setup.py +build diff --git a/airbyte-integrations/singer/exchangeratesapi/source/Dockerfile b/airbyte-integrations/singer/exchangeratesapi/source/Dockerfile index b3ad6e957af3d..6ef0ee0330fd0 100644 --- a/airbyte-integrations/singer/exchangeratesapi/source/Dockerfile +++ b/airbyte-integrations/singer/exchangeratesapi/source/Dockerfile @@ -1,21 +1,19 @@ FROM airbyte/integration-base-singer:dev -RUN apt-get update && apt-get install -y jq - -COPY spec.json /airbyte/exchangeratesapi-files/spec.json - -WORKDIR /airbyte/source_exchangeratesapi_singer - -COPY source_exchangeratesapi_singer/__init__.py ./source_exchangeratesapi_singer/__init__.py -COPY source_exchangeratesapi_singer/source_exchangeratesapi_singer.py ./source_exchangeratesapi_singer/source_exchangeratesapi_singer.py - -COPY setup.py . -RUN pip install . - -WORKDIR /airbyte +RUN apt-get update && apt-get install -y \ + jq \ + && rm -rf /var/lib/apt/lists/* +ENV CODE_PATH="source_exchangeratesapi_singer" ENV AIRBYTE_IMPL_MODULE="source_exchangeratesapi_singer" ENV AIRBYTE_IMPL_PATH="SourceExchangeRatesApiSinger" LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-exchangeratesapi-singer + +WORKDIR /airbyte/integration_code +COPY $CODE_PATH ./$CODE_PATH +COPY setup.py ./ +RUN pip install . + +WORKDIR /airbyte diff --git a/airbyte-integrations/singer/exchangeratesapi/source/build.gradle b/airbyte-integrations/singer/exchangeratesapi/source/build.gradle index d5050ea3a9144..2f74493429e3d 100644 --- a/airbyte-integrations/singer/exchangeratesapi/source/build.gradle +++ b/airbyte-integrations/singer/exchangeratesapi/source/build.gradle @@ -3,7 +3,6 @@ plugins { } apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') -buildImage.dependsOn ":airbyte-integrations:singer:base-singer:buildImage" apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') dependencies { @@ -13,3 +12,4 @@ dependencies { } integrationTest.dependsOn(buildImage) +buildImage.dependsOn ":airbyte-integrations:singer:base-singer:buildImage" diff --git a/airbyte-integrations/singer/exchangeratesapi/source/main_dev.py b/airbyte-integrations/singer/exchangeratesapi/source/main_dev.py new file mode 100644 index 0000000000000..16bf5aaf7d909 --- /dev/null +++ b/airbyte-integrations/singer/exchangeratesapi/source/main_dev.py @@ -0,0 +1,8 @@ +import sys +from airbyte_protocol.entrypoint import launch + +from source_exchangeratesapi_singer import SourceExchangeRatesApiSinger + +if __name__ == "__main__": + source = SourceExchangeRatesApiSinger() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/singer/exchangeratesapi/source/requirements.txt b/airbyte-integrations/singer/exchangeratesapi/source/requirements.txt new file mode 100644 index 0000000000000..c0c300b759506 --- /dev/null +++ b/airbyte-integrations/singer/exchangeratesapi/source/requirements.txt @@ -0,0 +1,3 @@ +-e ../../../base-python +-e ../../base-singer +-e . diff --git a/airbyte-integrations/singer/exchangeratesapi/source/setup.py b/airbyte-integrations/singer/exchangeratesapi/source/setup.py index b2a68c2fa01fc..2aca1715013a8 100644 --- a/airbyte-integrations/singer/exchangeratesapi/source/setup.py +++ b/airbyte-integrations/singer/exchangeratesapi/source/setup.py @@ -1,10 +1,19 @@ -from setuptools import setup +from setuptools import setup, find_packages setup( - name='source_exchangeratesapi_singer', + name='source-exchangeratesapi-singer', description='Source implementation for the exchange rates API.', author='Airbyte', author_email='contact@airbyte.io', - packages=['source_exchangeratesapi_singer'], - install_requires=['tap-exchangeratesapi==0.1.1', 'base_singer', 'airbyte_protocol'] + + packages=find_packages(), + package_data={ + '': ['*.json'] + }, + + install_requires=[ + 'tap-exchangeratesapi==0.1.1', + 'base_singer', + 'airbyte_protocol' + ] ) diff --git a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/__init__.py b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/__init__.py index d9296067923e8..3caf4212a67ab 100644 --- a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/__init__.py +++ b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/__init__.py @@ -1 +1 @@ -from .source_exchangeratesapi_singer import * +from .source import * diff --git a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source.py similarity index 73% rename from airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py rename to airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source.py index 4b507b193a9c6..d6bb135c8ab0c 100644 --- a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py +++ b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source.py @@ -1,27 +1,25 @@ -from airbyte_protocol import Source -from airbyte_protocol import AirbyteSpec -from airbyte_protocol import AirbyteCheckResponse -from airbyte_protocol import AirbyteCatalog -from airbyte_protocol import AirbyteMessage import urllib.request from typing import Generator + +from airbyte_protocol import AirbyteCatalog +from airbyte_protocol import AirbyteCheckResponse +from airbyte_protocol import AirbyteMessage +from airbyte_protocol import Source from base_singer import SingerHelper -from base_singer import Catalogs + class SourceExchangeRatesApiSinger(Source): def __init__(self): pass - def spec(self) -> AirbyteSpec: - return SingerHelper.spec_from_file("/airbyte/exchangeratesapi-files/spec.json") - def check(self, logger, config_container) -> AirbyteCheckResponse: code = urllib.request.urlopen("https://api.exchangeratesapi.io/").getcode() logger.info(f"Ping response code: {code}") return AirbyteCheckResponse(code == 200, {}) def discover(self, logger, config_container) -> AirbyteCatalog: - catalogs = SingerHelper.get_catalogs(logger, "tap-exchangeratesapi | grep '\"type\": \"SCHEMA\"' | head -1 | jq -c '{\"streams\":[{\"stream\": .stream, \"schema\": .schema}]}'") + cmd = "tap-exchangeratesapi | grep '\"type\": \"SCHEMA\"' | head -1 | jq -c '{\"streams\":[{\"stream\": .stream, \"schema\": .schema}]}'" + catalogs = SingerHelper.get_catalogs(logger, cmd) return catalogs.airbyte_catalog def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: diff --git a/airbyte-integrations/singer/exchangeratesapi/source/spec.json b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/spec.json similarity index 100% rename from airbyte-integrations/singer/exchangeratesapi/source/spec.json rename to airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/spec.json diff --git a/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceDataModelTest.java b/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceDataModelTest.java index 88a5c09ca7310..d385290c9cbfc 100644 --- a/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceDataModelTest.java +++ b/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceDataModelTest.java @@ -70,7 +70,7 @@ void testDeserializeCatalog() throws IOException { } @Test - void stripeSchemaMessageIsValid() throws IOException { + void schemaMessageIsValid() throws IOException { final String input = MoreResources.readResource("schema_message.json"); assertTrue(new SingerProtocolPredicate().test(Jsons.deserialize(input))); } diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/.dockerignore b/airbyte-integrations/singer/stripe_abprotocol/source/.dockerignore index 37171c8176965..378eac25d3117 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/.dockerignore +++ b/airbyte-integrations/singer/stripe_abprotocol/source/.dockerignore @@ -1,7 +1 @@ -* -!Dockerfile -!entrypoint.sh -!source_stripe_singer/*.py -!spec.json -!setup.py - +build diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/Dockerfile b/airbyte-integrations/singer/stripe_abprotocol/source/Dockerfile index 560915e93f935..d325ed1cfe7a7 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/Dockerfile +++ b/airbyte-integrations/singer/stripe_abprotocol/source/Dockerfile @@ -1,20 +1,22 @@ FROM airbyte/integration-base-singer:dev -RUN apt-get update && apt-get install -y jq curl bash +RUN apt-get update && apt-get install -y \ + jq \ + curl \ + bash \ + && rm -rf /var/lib/apt/lists/* -COPY spec.json /airbyte/stripe-files/spec.json - -WORKDIR /airbyte/source_stripe_singer +ENV CODE_PATH="source_stripe_singer" +ENV AIRBYTE_IMPL_MODULE="source_stripe_singer" +ENV AIRBYTE_IMPL_PATH="SourceStripeSinger" -COPY source_stripe_singer/*.py ./source_stripe_singer/ +LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.name=airbyte/source-stripe-abprotocol-singer -COPY setup.py . +WORKDIR /airbyte/integration_code +COPY $CODE_PATH ./$CODE_PATH +COPY setup.py ./ RUN pip install . WORKDIR /airbyte -ENV AIRBYTE_IMPL_MODULE="source_stripe_singer" -ENV AIRBYTE_IMPL_PATH="SourceStripeSinger" - -LABEL io.airbyte.version=0.1.3 -LABEL io.airbyte.name=airbyte/source-stripe-abprotocol-singer diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/main_dev.py b/airbyte-integrations/singer/stripe_abprotocol/source/main_dev.py new file mode 100644 index 0000000000000..41ed85215a3a0 --- /dev/null +++ b/airbyte-integrations/singer/stripe_abprotocol/source/main_dev.py @@ -0,0 +1,8 @@ +import sys +from airbyte_protocol.entrypoint import launch + +from source_stripe_singer import SourceStripeSinger + +if __name__ == "__main__": + source = SourceStripeSinger() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/requirements.txt b/airbyte-integrations/singer/stripe_abprotocol/source/requirements.txt new file mode 100644 index 0000000000000..c0c300b759506 --- /dev/null +++ b/airbyte-integrations/singer/stripe_abprotocol/source/requirements.txt @@ -0,0 +1,3 @@ +-e ../../../base-python +-e ../../base-singer +-e . diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/setup.py b/airbyte-integrations/singer/stripe_abprotocol/source/setup.py index 6c32f14290919..6dd6dd476cf59 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/setup.py +++ b/airbyte-integrations/singer/stripe_abprotocol/source/setup.py @@ -1,15 +1,20 @@ -from setuptools import setup +from setuptools import setup, find_packages setup( name='source_stripe_singer', description='Source implementation for Stripe.', author='Airbyte', author_email='contact@airbyte.io', - packages=['source_stripe_singer'], + + packages=find_packages(), + package_data={ + '': ['*.json'] + }, + install_requires=[ 'tap-stripe==1.4.4', 'requests', 'base_singer', - 'airbyte_protocol'] + 'airbyte_protocol' + ] ) - diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/__init__.py b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/__init__.py index 3e9669349d030..a825b9ffbe148 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/__init__.py +++ b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/__init__.py @@ -1,2 +1,2 @@ -from .source_stripe_singer import * +from .source import * diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source.py similarity index 86% rename from airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py rename to airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source.py index e5d409aa33140..a7117c3add787 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py +++ b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source.py @@ -1,20 +1,16 @@ -from airbyte_protocol import Source -from airbyte_protocol import AirbyteSpec -from airbyte_protocol import AirbyteCheckResponse +import requests from airbyte_protocol import AirbyteCatalog +from airbyte_protocol import AirbyteCheckResponse from airbyte_protocol import AirbyteMessage -import requests -from typing import Generator +from airbyte_protocol import Source from base_singer import SingerHelper +from typing import Generator class SourceStripeSinger(Source): def __init__(self): pass - def spec(self) -> AirbyteSpec: - return SingerHelper.spec_from_file('/airbyte/stripe-files/spec.json') - def check(self, logger, config_container) -> AirbyteCheckResponse: json_config = config_container.rendered_config r = requests.get('https://api.stripe.com/v1/customers', auth=(json_config['client_secret'], '')) @@ -26,8 +22,10 @@ def discover(self, logger, config_container) -> AirbyteCatalog: return catalogs.airbyte_catalog def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: + discover_cmd = f"tap-stripe --config {config_container.rendered_config_path} --discover" + discovered_singer_catalog = SingerHelper.get_catalogs(logger, discover_cmd).singer_catalog + masked_airbyte_catalog = self.read_config(catalog_path) - discovered_singer_catalog = SingerHelper.get_catalogs(logger, f"tap-stripe --config {config_container.rendered_config_path} --discover").singer_catalog selected_singer_catalog = SingerHelper.create_singer_catalog_with_selection(masked_airbyte_catalog, discovered_singer_catalog) config_option = f"--config {config_container.rendered_config_path}" diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/spec.json b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/spec.json similarity index 100% rename from airbyte-integrations/singer/stripe_abprotocol/source/spec.json rename to airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/spec.json diff --git a/airbyte-integrations/template/python-source/.dockerignore b/airbyte-integrations/template/python-source/.dockerignore new file mode 100644 index 0000000000000..378eac25d3117 --- /dev/null +++ b/airbyte-integrations/template/python-source/.dockerignore @@ -0,0 +1 @@ +build diff --git a/airbyte-integrations/template/python-source/Dockerfile b/airbyte-integrations/template/python-source/Dockerfile new file mode 100644 index 0000000000000..27ae85357db23 --- /dev/null +++ b/airbyte-integrations/template/python-source/Dockerfile @@ -0,0 +1,15 @@ +FROM airbyte/integration-base-python:dev + +ENV CODE_PATH="template_python_source" +ENV AIRBYTE_IMPL_MODULE="template_python_source" +ENV AIRBYTE_IMPL_PATH="TemplatePythonSource" + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-template-python + +WORKDIR /airbyte/integration_code +COPY $CODE_PATH ./$CODE_PATH +COPY setup.py ./ +RUN pip install . + +WORKDIR /airbyte diff --git a/airbyte-integrations/template/python-source/README.md b/airbyte-integrations/template/python-source/README.md new file mode 100644 index 0000000000000..b4c660484d8a2 --- /dev/null +++ b/airbyte-integrations/template/python-source/README.md @@ -0,0 +1,31 @@ +# Python Airbyte Source Development + +Prepare development environment: +``` +cd airbyte-integrations/template/python-source + +# create & activate virtualenv +virtualenv build/venv +source build/venv/bin/activate + +# install necessary dependencies +pip install -r requirements.txt +``` + +Test locally: +``` +python main_dev.py spec +python main_dev.py check --config sample_files/test_config.json +python main_dev.py discover --config sample_files/test_config.json +python main_dev.py read --config sample_files/test_config.json --catalog sample_files/test_catalog.json +``` + +Test image: +``` +# in airbyte root directory +./gradlew :airbyte-integrations:template:python-source:buildImage +docker run --rm -v $(pwd)/airbyte-integrations/template/python-source/sample_files:/sample_files airbyte/source-template-python:dev spec +docker run --rm -v $(pwd)/airbyte-integrations/template/python-source/sample_files:/sample_files airbyte/source-template-python:dev check --config /sample_files/test_config.json +docker run --rm -v $(pwd)/airbyte-integrations/template/python-source/sample_files:/sample_files airbyte/source-template-python:dev discover --config /sample_files/test_config.json +docker run --rm -v $(pwd)/airbyte-integrations/template/python-source/sample_files:/sample_files airbyte/source-template-python:dev read --config /sample_files/test_config.json --catalog /sample_files/test_catalog.json +``` diff --git a/airbyte-integrations/template/python-source/build.gradle b/airbyte-integrations/template/python-source/build.gradle new file mode 100644 index 0000000000000..f5858412095ec --- /dev/null +++ b/airbyte-integrations/template/python-source/build.gradle @@ -0,0 +1,3 @@ +apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') + +buildImage.dependsOn ":airbyte-integrations:base-python:buildImage" diff --git a/airbyte-integrations/template/python-source/main_dev.py b/airbyte-integrations/template/python-source/main_dev.py new file mode 100644 index 0000000000000..8a5db84d9be55 --- /dev/null +++ b/airbyte-integrations/template/python-source/main_dev.py @@ -0,0 +1,9 @@ +import sys + +from airbyte_protocol.entrypoint import launch + +from template_python_source import TemplatePythonSource + +if __name__ == "__main__": + source = TemplatePythonSource() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/template/python-source/requirements.txt b/airbyte-integrations/template/python-source/requirements.txt new file mode 100644 index 0000000000000..d92984a47ae4b --- /dev/null +++ b/airbyte-integrations/template/python-source/requirements.txt @@ -0,0 +1,2 @@ +-e ../../base-python +-e . diff --git a/airbyte-integrations/template/python-source/sample_files/test_catalog.json b/airbyte-integrations/template/python-source/sample_files/test_catalog.json new file mode 100644 index 0000000000000..b7ee0c9c09bcd --- /dev/null +++ b/airbyte-integrations/template/python-source/sample_files/test_catalog.json @@ -0,0 +1,16 @@ +{ + "streams": [ + { + "name": "love_airbyte", + "schema": { + "type": "object", + "required": ["love"], + "properties": { + "love": { + "type": "boolean" + } + } + } + } + ] +} diff --git a/airbyte-integrations/template/python-source/sample_files/test_config.json b/airbyte-integrations/template/python-source/sample_files/test_config.json new file mode 100644 index 0000000000000..ccc5fb07ee693 --- /dev/null +++ b/airbyte-integrations/template/python-source/sample_files/test_config.json @@ -0,0 +1,3 @@ +{ + "love_airbyte": true +} diff --git a/airbyte-integrations/template/python-source/setup.py b/airbyte-integrations/template/python-source/setup.py new file mode 100644 index 0000000000000..afff28006bc56 --- /dev/null +++ b/airbyte-integrations/template/python-source/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup, find_packages + +setup( + name='template-python-source', + description='Source implementation template', + author='Airbyte', + author_email='contact@airbyte.io', + + packages=find_packages(), + package_data={ + '': ['*.json'] + }, + + install_requires=['airbyte-protocol'] +) diff --git a/airbyte-integrations/template/python-source/template_python_source/__init__.py b/airbyte-integrations/template/python-source/template_python_source/__init__.py new file mode 100644 index 0000000000000..3caf4212a67ab --- /dev/null +++ b/airbyte-integrations/template/python-source/template_python_source/__init__.py @@ -0,0 +1 @@ +from .source import * diff --git a/airbyte-integrations/template/python-source/template_python_source/catalog.json b/airbyte-integrations/template/python-source/template_python_source/catalog.json new file mode 100644 index 0000000000000..b7ee0c9c09bcd --- /dev/null +++ b/airbyte-integrations/template/python-source/template_python_source/catalog.json @@ -0,0 +1,16 @@ +{ + "streams": [ + { + "name": "love_airbyte", + "schema": { + "type": "object", + "required": ["love"], + "properties": { + "love": { + "type": "boolean" + } + } + } + } + ] +} diff --git a/airbyte-integrations/template/python-source/template_python_source/source.py b/airbyte-integrations/template/python-source/template_python_source/source.py new file mode 100644 index 0000000000000..dbf9d6191c611 --- /dev/null +++ b/airbyte-integrations/template/python-source/template_python_source/source.py @@ -0,0 +1,36 @@ +import pkgutil +import time +from typing import Generator + +from airbyte_protocol import AirbyteCatalog +from airbyte_protocol import AirbyteCheckResponse +from airbyte_protocol import AirbyteMessage +from airbyte_protocol import AirbyteRecordMessage +from airbyte_protocol import AirbyteSpec +from airbyte_protocol import AirbyteStateMessage +from airbyte_protocol import Source + + +class TemplatePythonSource(Source): + def __init__(self): + pass + + def check(self, logger, config_container) -> AirbyteCheckResponse: + logger.info(f'Checking configuration ({config_container.rendered_config_path})...') + return AirbyteCheckResponse(True, {}) + + def discover(self, logger, config_container) -> AirbyteCatalog: + logger.info(f'Discovering ({config_container.rendered_config_path})...') + return AirbyteCatalog.from_json(pkgutil.get_data(__name__, 'catalog.json')) + + def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: + logger.info(f'Reading ({config_container.rendered_config_path}, {catalog_path}, {state})...') + + message = AirbyteRecordMessage( + stream='love_airbyte', + data={'love': True}, + emitted_at=int(time.time() * 1000)) + yield AirbyteMessage(type='RECORD', record=message) + + state = AirbyteStateMessage(data={'love_cursor': 'next_version'}) + yield AirbyteMessage(type='STATE', state=state) diff --git a/airbyte-integrations/template/python-source/template_python_source/spec.json b/airbyte-integrations/template/python-source/template_python_source/spec.json new file mode 100644 index 0000000000000..10b8b4372f7b7 --- /dev/null +++ b/airbyte-integrations/template/python-source/template_python_source/spec.json @@ -0,0 +1,17 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/template-python-source", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Template python source Spec", + "type": "object", + "required": ["love_airbyte"], + "additionalProperties": false, + "properties": { + "love_airbyte": { + "type": "boolean", + "description": "Do you love Airbyte", + "examples": ["true"] + } + } + } +} diff --git a/airbyte-integrations/template/singer-source/.dockerignore b/airbyte-integrations/template/singer-source/.dockerignore new file mode 100644 index 0000000000000..378eac25d3117 --- /dev/null +++ b/airbyte-integrations/template/singer-source/.dockerignore @@ -0,0 +1 @@ +build diff --git a/airbyte-integrations/template/singer-source/Dockerfile b/airbyte-integrations/template/singer-source/Dockerfile new file mode 100644 index 0000000000000..37199a053cce8 --- /dev/null +++ b/airbyte-integrations/template/singer-source/Dockerfile @@ -0,0 +1,19 @@ +FROM airbyte/integration-base-singer:dev + +RUN apt-get update && apt-get install -y \ + jq \ + && rm -rf /var/lib/apt/lists/* + +ENV CODE_PATH="template_singer_source" +ENV AIRBYTE_IMPL_MODULE="template_singer_source" +ENV AIRBYTE_IMPL_PATH="TemplateSingerSource" + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-template-singer + +WORKDIR /airbyte/integration_code +COPY $CODE_PATH ./$CODE_PATH +COPY setup.py ./ +RUN pip install . + +WORKDIR /airbyte diff --git a/airbyte-integrations/template/singer-source/README.md b/airbyte-integrations/template/singer-source/README.md new file mode 100644 index 0000000000000..6fe76e55c02a5 --- /dev/null +++ b/airbyte-integrations/template/singer-source/README.md @@ -0,0 +1 @@ +# This is a demo source diff --git a/airbyte-integrations/template/singer-source/build.gradle b/airbyte-integrations/template/singer-source/build.gradle new file mode 100644 index 0000000000000..2fb951e69fdd1 --- /dev/null +++ b/airbyte-integrations/template/singer-source/build.gradle @@ -0,0 +1,5 @@ +apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') +apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') + +integrationTest.dependsOn(buildImage) +buildImage.dependsOn ":airbyte-integrations:singer:base-singer:buildImage" diff --git a/airbyte-integrations/template/singer-source/main_dev.py b/airbyte-integrations/template/singer-source/main_dev.py new file mode 100644 index 0000000000000..b6e250e541156 --- /dev/null +++ b/airbyte-integrations/template/singer-source/main_dev.py @@ -0,0 +1,8 @@ +import sys +from airbyte_protocol.entrypoint import launch + +from template_singer_source import TemplateSingerSource + +if __name__ == "__main__": + source = TemplateSingerSource() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/template/singer-source/requirements.txt b/airbyte-integrations/template/singer-source/requirements.txt new file mode 100644 index 0000000000000..d1efae608b88f --- /dev/null +++ b/airbyte-integrations/template/singer-source/requirements.txt @@ -0,0 +1,3 @@ +-e ../../base-python +-e ../../singer/base-singer +-e . diff --git a/airbyte-integrations/template/singer-source/setup.py b/airbyte-integrations/template/singer-source/setup.py new file mode 100644 index 0000000000000..3dee3337bc3ad --- /dev/null +++ b/airbyte-integrations/template/singer-source/setup.py @@ -0,0 +1,19 @@ +from setuptools import setup, find_packages + +setup( + name='template-singer-source', + description='Singer source implementation template', + author='Airbyte', + author_email='contact@airbyte.io', + + packages=find_packages(), + package_data={ + '': ['*.json'] + }, + + install_requires=[ + 'tap-exchangeratesapi==0.1.1', + 'base_singer', + 'airbyte_protocol' + ] +) diff --git a/airbyte-integrations/template/singer-source/template_singer_source/__init__.py b/airbyte-integrations/template/singer-source/template_singer_source/__init__.py new file mode 100644 index 0000000000000..3caf4212a67ab --- /dev/null +++ b/airbyte-integrations/template/singer-source/template_singer_source/__init__.py @@ -0,0 +1 @@ +from .source import * diff --git a/airbyte-integrations/template/singer-source/template_singer_source/source.py b/airbyte-integrations/template/singer-source/template_singer_source/source.py new file mode 100644 index 0000000000000..eef142757f30a --- /dev/null +++ b/airbyte-integrations/template/singer-source/template_singer_source/source.py @@ -0,0 +1,28 @@ +import urllib.request +from typing import Generator + +from airbyte_protocol import AirbyteCatalog +from airbyte_protocol import AirbyteCheckResponse +from airbyte_protocol import AirbyteMessage +from airbyte_protocol import Source +from base_singer import SingerHelper + + +class TemplateSingerSource(Source): + def __init__(self): + pass + + def check(self, logger, config_container) -> AirbyteCheckResponse: + code = urllib.request.urlopen("https://api.exchangeratesapi.io/").getcode() + logger.info(f"Ping response code: {code}") + return AirbyteCheckResponse(code == 200, {}) + + def discover(self, logger, config_container) -> AirbyteCatalog: + cmd = "tap-exchangeratesapi | grep '\"type\": \"SCHEMA\"' | head -1 | jq -c '{\"streams\":[{\"stream\": .stream, \"schema\": .schema}]}'" + catalogs = SingerHelper.get_catalogs(logger, cmd) + return catalogs.airbyte_catalog + + def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: + config_option = f"--config {config_container.rendered_config_path}" + state_option = f"--state {state}" if state else "" + return SingerHelper.read(logger, f"tap-exchangeratesapi {config_option} {state_option}") diff --git a/airbyte-integrations/template/singer-source/template_singer_source/spec.json b/airbyte-integrations/template/singer-source/template_singer_source/spec.json new file mode 100644 index 0000000000000..4f260cecd1a48 --- /dev/null +++ b/airbyte-integrations/template/singer-source/template_singer_source/spec.json @@ -0,0 +1,21 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/exchangeratesapi-io", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Template Singer Source Spec", + "type": "object", + "required": ["start_date", "base"], + "additionalProperties": false, + "properties": { + "start_date": { + "type": "string", + "description": "Start getting data from that date.", + "examples": ["YYYY-MM-DD"] + }, + "base": { + "type": "string", + "description": "ISO reference currency. See here." + } + } + } +} diff --git a/build.gradle b/build.gradle index e87a524c405a8..ad0484b9a1a08 100644 --- a/build.gradle +++ b/build.gradle @@ -2,7 +2,7 @@ plugins { id 'base' id 'java' id 'pmd' - id 'com.diffplug.spotless' version '5.4.0' + id 'com.diffplug.spotless' version '5.6.1' // id 'de.aaschmid.cpd' version '3.1' }