From 5f8f78747367ce7725838a50642d4f463f2949bf Mon Sep 17 00:00:00 2001 From: jrhizor Date: Fri, 9 Oct 2020 09:51:37 -0700 Subject: [PATCH 01/31] use log messages for check and add todos --- airbyte-integrations/base-python/base.py | 15 ++++++++++++--- .../base-singer/base_singer/singer_helpers.py | 1 + 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/base-python/base.py b/airbyte-integrations/base-python/base.py index 5b7f5c3d5dc1e..68dc6f6450228 100644 --- a/airbyte-integrations/base-python/base.py +++ b/airbyte-integrations/base-python/base.py @@ -6,6 +6,8 @@ import importlib from airbyte_protocol import Source +from airbyte_protocol import AirbyteLogMessage +from airbyte_protocol import AirbyteMessage impl_module = os.environ['AIRBYTE_IMPL_MODULE'] impl_class = os.environ['AIRBYTE_IMPL_PATH'] @@ -13,6 +15,13 @@ module = importlib.import_module(impl_module) impl = getattr(module, impl_class) + +def log(level, text): + log_message = AirbyteLogMessage(level=level, message=text) + message = AirbyteMessage(type="LOG", log=log_message) + print(message.serialize) + + class AirbyteEntrypoint(object): def __init__(self, source): self.source = source @@ -73,14 +82,14 @@ def start(self): if cmd == "check": check_result = source.check(logging, rendered_config_path) if check_result.successful: - print("Check succeeded") + log("INFO", "Check succeeded") sys.exit(0) else: - print("Check failed") + log("ERROR", "Check failed") sys.exit(1) elif cmd == "discover": schema = source.discover(logging, rendered_config_path) - print(schema.schema) + print(schema.schema) # todo: print as serialized catalog message sys.exit(0) elif cmd == "read": # todo: pass in state diff --git a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py index 6eae7c1026c04..1f2eec973149b 100644 --- a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py +++ b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py @@ -55,6 +55,7 @@ def discover(shell_command, transform=(lambda x: AirbyteSchema(x))) -> AirbyteSc def read(shell_command, is_message=(lambda x: True), transform=(lambda x: x)) -> Generator[AirbyteMessage, None, None]: with subprocess.Popen(shell_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1, universal_newlines=True) as p: + # todo: generate combined schema message from discovery and reading the catalog for tuple in zip(p.stdout, p.stderr): out_line = tuple[0] err_line = tuple[1] From 13eea10a8cc91a610a6e0e198be0e944c631b047 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Fri, 9 Oct 2020 11:06:06 -0700 Subject: [PATCH 02/31] working state --- .../airbyte_protocol/airbyte_protocol/__init__.py | 7 +------ airbyte-integrations/base-python/base.py | 11 +++++------ airbyte-integrations/base/base.sh | 7 +++++-- .../singer/base-singer/base_singer/singer_helpers.py | 5 +++-- .../source_exchangeratesapi_singer.py | 9 ++++++--- 5 files changed, 20 insertions(+), 19 deletions(-) diff --git a/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py b/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py index cca238df75170..643a48effc0e0 100644 --- a/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py +++ b/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py @@ -36,11 +36,6 @@ def __init__(self, successful, field_to_error): self.field_to_error = field_to_error -class AirbyteSchema(object): - def __init__(self, schema): - self.schema = schema - - class AirbyteConfig(object): def __init__(self, config_string): self.config_string = config_string @@ -67,7 +62,7 @@ def render_config(self, config_object, rendered_config_path): def check(self, config_object, rendered_config_path) -> AirbyteCheckResponse: raise Exception("Not Implemented") - def discover(self, config_object, rendered_config_path) -> AirbyteSchema: + def discover(self, config_object, rendered_config_path) -> AirbyteCatalog: raise Exception("Not Implemented") diff --git a/airbyte-integrations/base-python/base.py b/airbyte-integrations/base-python/base.py index 68dc6f6450228..f68ed0a76af67 100644 --- a/airbyte-integrations/base-python/base.py +++ b/airbyte-integrations/base-python/base.py @@ -52,8 +52,8 @@ def start(self): # read read_parser = subparsers.add_parser("read", help="reads the source and outputs messages to STDOUT", parents=[parent_parser]) - # todo: re-add state handling - # read_parser.add_argument('--state', type=str, required=False, help='path to the json-encoded state file') + + read_parser.add_argument('--state', type=str, required=False, help='path to the json-encoded state file') required_read_parser = read_parser.add_argument_group('required named arguments') required_read_parser.add_argument('--config', type=str, required=True, help='path to the json configuration file') @@ -88,12 +88,11 @@ def start(self): log("ERROR", "Check failed") sys.exit(1) elif cmd == "discover": - schema = source.discover(logging, rendered_config_path) - print(schema.schema) # todo: print as serialized catalog message + catalog = source.discover(logging, rendered_config_path) + print(catalog.serialize()) sys.exit(0) elif cmd == "read": - # todo: pass in state - generator = source.read(logging, rendered_config_path) + generator = source.read(logging, rendered_config_path, parsed_args.state) for message in generator: print(message.serialize()) sys.exit(0) diff --git a/airbyte-integrations/base/base.sh b/airbyte-integrations/base/base.sh index 869ca581f33e0..6588424b43818 100755 --- a/airbyte-integrations/base/base.sh +++ b/airbyte-integrations/base/base.sh @@ -48,8 +48,11 @@ function main() { eval "$AIRBYTE_DISCOVER_CMD" --config "$CONFIG_FILE" ;; read) - # todo: state should be optional: --state "$STATE_FILE" - eval "$AIRBYTE_READ_CMD" --config "$CONFIG_FILE" --catalog "$CATALOG_FILE" + if [[ -z "$STATE_FILE" ]]; then + eval "$AIRBYTE_READ_CMD" --config "$CONFIG_FILE" --catalog "$CATALOG_FILE" + else + eval "$AIRBYTE_READ_CMD" --config "$CONFIG_FILE" --catalog "$CATALOG_FILE" --state "$STATE_FILE" + fi ;; write) eval "$AIRBYTE_WRITE_CMD" --config "$CONFIG_FILE" --catalog "$CATALOG_FILE" diff --git a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py index 05e2c47f110f4..d46f432697999 100644 --- a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py +++ b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py @@ -1,7 +1,7 @@ import json import subprocess from airbyte_protocol import AirbyteSpec -from airbyte_protocol import AirbyteSchema +from airbyte_protocol import AirbyteCatalog from airbyte_protocol import AirbyteMessage from airbyte_protocol import AirbyteLogMessage from airbyte_protocol import AirbyteRecordMessage @@ -46,9 +46,10 @@ def spec_from_file(spec_path) -> AirbyteSpec: # todo: support stderr in the discover process @staticmethod - def discover(shell_command, transform=(lambda x: AirbyteSchema(x))) -> AirbyteSchema: + def discover(shell_command, transform=(lambda x: AirbyteCatalog(x))) -> AirbyteCatalog: completed_process = subprocess.run(shell_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + # todo: handle this return transform(completed_process.stdout) @staticmethod diff --git a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py index 7660de89b1563..c891898e83f86 100644 --- a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py +++ b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py @@ -1,7 +1,7 @@ from airbyte_protocol import Source from airbyte_protocol import AirbyteSpec from airbyte_protocol import AirbyteCheckResponse -from airbyte_protocol import AirbyteSchema +from airbyte_protocol import AirbyteCatalog from airbyte_protocol import AirbyteMessage import urllib.request from typing import Generator @@ -19,9 +19,12 @@ def check(self, config_object, rendered_config_path) -> AirbyteCheckResponse: code = urllib.request.urlopen("https://api.exchangeratesapi.io/").getcode() return AirbyteCheckResponse(code == 200, {}) - def discover(self, config_object, rendered_config_path) -> AirbyteSchema: + def discover(self, config_object, rendered_config_path) -> AirbyteCatalog: return SingerHelper.discover("tap-exchangeratesapi | grep '\"type\": \"SCHEMA\"' | head -1 | jq -c '{\"streams\":[{\"stream\": .stream, \"schema\": .schema}]}'") # todo: handle state def read(self, config_object, rendered_config_path, state=None) -> Generator[AirbyteMessage, None, None]: - return SingerHelper.read(f"tap-exchangeratesapi --config {rendered_config_path}") + if state: + return SingerHelper.read(f"tap-exchangeratesapi --config {rendered_config_path} --state {state}") + else: + return SingerHelper.read(f"tap-exchangeratesapi --config {rendered_config_path}") From 92b3bd7df33f41498a5bcc7654ea1acbd2a1aaf3 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 13 Oct 2020 06:28:04 -0700 Subject: [PATCH 03/31] handle stderr in discoer and use airbyte catalog as output --- .../base-singer/base_singer/singer_helpers.py | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py index d46f432697999..5f09aef909ad3 100644 --- a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py +++ b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py @@ -6,6 +6,7 @@ from airbyte_protocol import AirbyteLogMessage from airbyte_protocol import AirbyteRecordMessage from airbyte_protocol import AirbyteStateMessage +from airbyte_protocol import AirbyteStream from typing import Generator from datetime import datetime @@ -44,13 +45,26 @@ def spec_from_file(spec_path) -> AirbyteSpec: spec_text = file.read() return AirbyteSpec(spec_text) - # todo: support stderr in the discover process + @staticmethod - def discover(shell_command, transform=(lambda x: AirbyteCatalog(x))) -> AirbyteCatalog: + def discover(shell_command, transform=(lambda catalog: catalog)) -> AirbyteCatalog: completed_process = subprocess.run(shell_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) - # todo: handle this - return transform(completed_process.stdout) + + for line in completed_process.stderr.splitlines(): + log_line(line, "ERROR") + + airbyte_streams = [] + obj = json.loads(completed_process.stdout) + + for stream in obj.get("streams"): + name = stream.get("stream") + schema = stream.get("schema").get("properties") + airbyte_streams += [AirbyteStream(name=name, schema=schema)] + + catalog = AirbyteCatalog(streams=airbyte_streams) + + return transform(catalog) @staticmethod def read(shell_command, is_message=(lambda x: True), transform=(lambda x: x)) -> Generator[AirbyteMessage, None, None]: @@ -73,8 +87,6 @@ def read(shell_command, is_message=(lambda x: True), transform=(lambda x: x)) -> out_message = AirbyteMessage(type="STATE", state=out_record) yield transform(out_message) else: - # todo: remove type from record - # todo: handle stream designation # todo: check that messages match the discovered schema stream_name = transformed_json["stream"] out_record = AirbyteRecordMessage(stream=stream_name, data=transformed_json["record"], emitted_at=str(datetime.now())) From f701fa4f7bf9d1c4e307266492cf8bd2428db00a Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 13 Oct 2020 07:10:25 -0700 Subject: [PATCH 04/31] misc restructure --- .../airbyte_protocol/__init__.py | 2 +- airbyte-integrations/base-python/base.py | 2 +- .../base-singer/base_singer/singer_helpers.py | 25 +++++++++++-------- .../source_exchangeratesapi_singer.py | 22 ++++++++++++---- 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py b/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py index 643a48effc0e0..5568d32ff6765 100644 --- a/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py +++ b/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py @@ -71,7 +71,7 @@ def __init__(self): pass # Iterator - def read(self, config_object, rendered_config_path, state=None) -> Generator[AirbyteMessage, None, None]: + def read(self, config_object, rendered_config_path, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: raise Exception("Not Implemented") diff --git a/airbyte-integrations/base-python/base.py b/airbyte-integrations/base-python/base.py index f68ed0a76af67..bfbc19dcb32cc 100644 --- a/airbyte-integrations/base-python/base.py +++ b/airbyte-integrations/base-python/base.py @@ -92,7 +92,7 @@ def start(self): print(catalog.serialize()) sys.exit(0) elif cmd == "read": - generator = source.read(logging, rendered_config_path, parsed_args.state) + generator = source.read(logging, rendered_config_path, parsed_args.catalog, parsed_args.state) for message in generator: print(message.serialize()) sys.exit(0) diff --git a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py index 8d5d0a38b0ee2..2643cb881f1cd 100644 --- a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py +++ b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py @@ -9,6 +9,7 @@ from airbyte_protocol import AirbyteStream from typing import Generator from datetime import datetime +from dataclasses import dataclass # helper to delegate input and output to a piped command @@ -38,6 +39,12 @@ def to_json(string): return False +@dataclass +class Catalogs: + singer_catalog: object + airbyte_catalog: AirbyteCatalog + + class SingerHelper: @staticmethod def spec_from_file(spec_path) -> AirbyteSpec: @@ -45,9 +52,8 @@ def spec_from_file(spec_path) -> AirbyteSpec: spec_text = file.read() return AirbyteSpec(spec_text) - @staticmethod - def discover(shell_command, transform=(lambda catalog: catalog)) -> AirbyteCatalog: + def discover(shell_command, singer_transform=(lambda catalog: catalog), airbyte_transform=(lambda catalog: catalog)) -> Catalogs: completed_process = subprocess.run(shell_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) @@ -55,25 +61,24 @@ def discover(shell_command, transform=(lambda catalog: catalog)) -> AirbyteCatal log_line(line, "ERROR") airbyte_streams = [] - obj = json.loads(completed_process.stdout) + singer_catalog = singer_transform(json.loads(completed_process.stdout)) - for stream in obj.get("streams"): + for stream in singer_catalog.get("streams"): name = stream.get("stream") schema = stream.get("schema").get("properties") airbyte_streams += [AirbyteStream(name=name, schema=schema)] - catalog = AirbyteCatalog(streams=airbyte_streams) + airbyte_catalog = airbyte_transform(AirbyteCatalog(streams=airbyte_streams)) - return transform(catalog) + return Catalogs(singer_catalog=singer_catalog, airbyte_catalog=airbyte_catalog) @staticmethod def read(shell_command, is_message=(lambda x: True), transform=(lambda x: x)) -> Generator[AirbyteMessage, None, None]: with subprocess.Popen(shell_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1, universal_newlines=True) as p: - # todo: generate combined schema message from discovery and reading the catalog - for tuple in zip(p.stdout, p.stderr): - out_line = tuple[0] - err_line = tuple[1] + for line_tuple in zip(p.stdout, p.stderr): + out_line = line_tuple[0] + err_line = line_tuple[1] if out_line: out_json = to_json(out_line) diff --git a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py index c891898e83f86..35d0e31a2ea40 100644 --- a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py +++ b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py @@ -6,6 +6,11 @@ import urllib.request from typing import Generator from base_singer import SingerHelper +from base_singer import Catalogs + + +def get_catalogs() -> Catalogs: + return SingerHelper.discover("tap-exchangeratesapi | grep '\"type\": \"SCHEMA\"' | head -1 | jq -c '{\"streams\":[{\"stream\": .stream, \"schema\": .schema}]}'") class SourceExchangeRatesApiSinger(Source): @@ -20,11 +25,18 @@ def check(self, config_object, rendered_config_path) -> AirbyteCheckResponse: return AirbyteCheckResponse(code == 200, {}) def discover(self, config_object, rendered_config_path) -> AirbyteCatalog: - return SingerHelper.discover("tap-exchangeratesapi | grep '\"type\": \"SCHEMA\"' | head -1 | jq -c '{\"streams\":[{\"stream\": .stream, \"schema\": .schema}]}'") + return get_catalogs().airbyte_catalog + + def read(self, config_object, rendered_config_path, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: + airbyte_catalog = catalog_path # todo: read + + # call discover + singer_catalog = get_catalogs().singer_catalog + + # todo: combine discovered singer catalog with the generated airbyte catalog - # todo: handle state - def read(self, config_object, rendered_config_path, state=None) -> Generator[AirbyteMessage, None, None]: + sync_prefix = f"tap-exchangeratesapi --config {rendered_config_path}" if state: - return SingerHelper.read(f"tap-exchangeratesapi --config {rendered_config_path} --state {state}") + return SingerHelper.read(sync_prefix + f"--state {state}") else: - return SingerHelper.read(f"tap-exchangeratesapi --config {rendered_config_path}") + return SingerHelper.read(sync_prefix) From 7cfb17ca856949f7090b18c43267a08ae2458652 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 13 Oct 2020 07:50:00 -0700 Subject: [PATCH 05/31] inject logger again --- .../airbyte_protocol/__init__.py | 23 ++++++++++++-- airbyte-integrations/base-python/base.py | 7 +++-- .../base-singer/base_singer/singer_helpers.py | 31 +++---------------- .../source_exchangeratesapi_singer.py | 24 +++++--------- 4 files changed, 37 insertions(+), 48 deletions(-) diff --git a/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py b/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py index 5568d32ff6765..008d9184ff9dc 100644 --- a/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py +++ b/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py @@ -59,10 +59,10 @@ def render_config(self, config_object, rendered_config_path): with open(rendered_config_path, 'w') as fh: fh.write(config_object.config_string) - def check(self, config_object, rendered_config_path) -> AirbyteCheckResponse: + def check(self, logger, rendered_config_path) -> AirbyteCheckResponse: raise Exception("Not Implemented") - def discover(self, config_object, rendered_config_path) -> AirbyteCatalog: + def discover(self, logger, rendered_config_path) -> AirbyteCatalog: raise Exception("Not Implemented") @@ -71,10 +71,27 @@ def __init__(self): pass # Iterator - def read(self, config_object, rendered_config_path, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: + def read(self, logger, rendered_config_path, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: raise Exception("Not Implemented") class Destination(Integration): def __init__(self): pass + + +valid_log_types = ["FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE"] + + +def log_line(line, default_level): + split_line = line.split() + first_word = next(iter(split_line), None) + if first_word in valid_log_types: + log_level = first_word + rendered_line = " ".join(split_line[1:]) + else: + log_level = default_level + rendered_line = line + log_record = AirbyteLogMessage(level=log_level, message=rendered_line) + log_message = AirbyteMessage(type="LOG", log=log_record) + print(log_message.serialize()) diff --git a/airbyte-integrations/base-python/base.py b/airbyte-integrations/base-python/base.py index bfbc19dcb32cc..5b73d079b3817 100644 --- a/airbyte-integrations/base-python/base.py +++ b/airbyte-integrations/base-python/base.py @@ -8,6 +8,7 @@ from airbyte_protocol import Source from airbyte_protocol import AirbyteLogMessage from airbyte_protocol import AirbyteMessage +from airbyte_protocol import log_line impl_module = os.environ['AIRBYTE_IMPL_MODULE'] impl_class = os.environ['AIRBYTE_IMPL_PATH'] @@ -80,7 +81,7 @@ def start(self): # todo: output message for check if cmd == "check": - check_result = source.check(logging, rendered_config_path) + check_result = source.check(log_line, rendered_config_path) if check_result.successful: log("INFO", "Check succeeded") sys.exit(0) @@ -88,11 +89,11 @@ def start(self): log("ERROR", "Check failed") sys.exit(1) elif cmd == "discover": - catalog = source.discover(logging, rendered_config_path) + catalog = source.discover(log_line, rendered_config_path) print(catalog.serialize()) sys.exit(0) elif cmd == "read": - generator = source.read(logging, rendered_config_path, parsed_args.catalog, parsed_args.state) + generator = source.read(log_line, rendered_config_path, parsed_args.catalog, parsed_args.state) for message in generator: print(message.serialize()) sys.exit(0) diff --git a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py index 2643cb881f1cd..64b3e52cb4627 100644 --- a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py +++ b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py @@ -11,27 +11,6 @@ from datetime import datetime from dataclasses import dataclass - -# helper to delegate input and output to a piped command -# todo: add error handling (make sure the overall tap fails if there's a failure in here) - -valid_log_types = ["FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE"] - - -def log_line(line, default_level): - split_line = line.split() - first_word = next(iter(split_line), None) - if first_word in valid_log_types: - log_level = first_word - rendered_line = " ".join(split_line[1:]) - else: - log_level = default_level - rendered_line = line - log_record = AirbyteLogMessage(level=log_level, message=rendered_line) - log_message = AirbyteMessage(type="LOG", log=log_record) - print(log_message.serialize()) - - def to_json(string): try: return json.loads(string) @@ -53,12 +32,12 @@ def spec_from_file(spec_path) -> AirbyteSpec: return AirbyteSpec(spec_text) @staticmethod - def discover(shell_command, singer_transform=(lambda catalog: catalog), airbyte_transform=(lambda catalog: catalog)) -> Catalogs: + def discover(logger, shell_command, singer_transform=(lambda catalog: catalog), airbyte_transform=(lambda catalog: catalog)) -> Catalogs: completed_process = subprocess.run(shell_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) for line in completed_process.stderr.splitlines(): - log_line(line, "ERROR") + logger(line, "ERROR") airbyte_streams = [] singer_catalog = singer_transform(json.loads(completed_process.stdout)) @@ -73,7 +52,7 @@ def discover(shell_command, singer_transform=(lambda catalog: catalog), airbyte_ return Catalogs(singer_catalog=singer_catalog, airbyte_catalog=airbyte_catalog) @staticmethod - def read(shell_command, is_message=(lambda x: True), transform=(lambda x: x)) -> Generator[AirbyteMessage, None, None]: + def read(logger, shell_command, is_message=(lambda x: True), transform=(lambda x: x)) -> Generator[AirbyteMessage, None, None]: with subprocess.Popen(shell_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1, universal_newlines=True) as p: for line_tuple in zip(p.stdout, p.stderr): @@ -101,7 +80,7 @@ def read(shell_command, is_message=(lambda x: True), transform=(lambda x: x)) -> out_message = AirbyteMessage(type="RECORD", record=out_record) yield transform(out_message) elif out_line: - log_line(out_line, "INFO") + logger(out_line, "INFO") if err_line: - log_line(err_line, "ERROR") + logger(err_line, "ERROR") diff --git a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py index 35d0e31a2ea40..ae5853e98741c 100644 --- a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py +++ b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py @@ -9,8 +9,8 @@ from base_singer import Catalogs -def get_catalogs() -> Catalogs: - return SingerHelper.discover("tap-exchangeratesapi | grep '\"type\": \"SCHEMA\"' | head -1 | jq -c '{\"streams\":[{\"stream\": .stream, \"schema\": .schema}]}'") +def get_catalogs(logger) -> Catalogs: + return SingerHelper.discover(logger, "tap-exchangeratesapi | grep '\"type\": \"SCHEMA\"' | head -1 | jq -c '{\"streams\":[{\"stream\": .stream, \"schema\": .schema}]}'") class SourceExchangeRatesApiSinger(Source): @@ -20,23 +20,15 @@ def __init__(self): def spec(self) -> AirbyteSpec: return SingerHelper.spec_from_file("/airbyte/exchangeratesapi-files/spec.json") - def check(self, config_object, rendered_config_path) -> AirbyteCheckResponse: + def check(self, logger, rendered_config_path) -> AirbyteCheckResponse: code = urllib.request.urlopen("https://api.exchangeratesapi.io/").getcode() return AirbyteCheckResponse(code == 200, {}) - def discover(self, config_object, rendered_config_path) -> AirbyteCatalog: - return get_catalogs().airbyte_catalog + def discover(self, logger, rendered_config_path) -> AirbyteCatalog: + return get_catalogs(logger).airbyte_catalog - def read(self, config_object, rendered_config_path, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: - airbyte_catalog = catalog_path # todo: read - - # call discover - singer_catalog = get_catalogs().singer_catalog - - # todo: combine discovered singer catalog with the generated airbyte catalog - - sync_prefix = f"tap-exchangeratesapi --config {rendered_config_path}" + def read(self, logger, rendered_config_path, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: if state: - return SingerHelper.read(sync_prefix + f"--state {state}") + return SingerHelper.read(logger, f"tap-exchangeratesapi --config {rendered_config_path} --state {state}") else: - return SingerHelper.read(sync_prefix) + return SingerHelper.read(logger, f"tap-exchangeratesapi --config {rendered_config_path}") From 07e7e4d5b5d874206122bab7636a071a98754543 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 13 Oct 2020 08:03:30 -0700 Subject: [PATCH 06/31] use config container (still at the base level) --- .../airbyte_protocol/__init__.py | 36 +++++++++++-------- airbyte-integrations/base-python/base.py | 19 ++++++---- .../source_exchangeratesapi_singer.py | 10 +++--- 3 files changed, 39 insertions(+), 26 deletions(-) diff --git a/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py b/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py index 008d9184ff9dc..47659f3517a90 100644 --- a/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py +++ b/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py @@ -1,8 +1,10 @@ from typing import Generator import yaml +import json import pkgutil import warnings import python_jsonschema_objects as pjs +from dataclasses import dataclass def _load_classes(yaml_path: str): @@ -36,11 +38,6 @@ def __init__(self, successful, field_to_error): self.field_to_error = field_to_error -class AirbyteConfig(object): - def __init__(self, config_string): - self.config_string = config_string - - class Integration(object): def __init__(self): pass @@ -48,21 +45,23 @@ def __init__(self): def spec(self) -> AirbyteSpec: raise Exception("Not Implemented") - # default version reads the config_path to a string - # this will often be overwritten to add fields for easy consumption or to modify the string for delegating to singer - def read_config(self, config_path) -> AirbyteConfig: + def read_config(self, config_path): with open(config_path, 'r') as file: contents = file.read() - return AirbyteConfig(contents) + return json.loads(contents) - def render_config(self, config_object, rendered_config_path): - with open(rendered_config_path, 'w') as fh: - fh.write(config_object.config_string) + # can be overridden to change an input file config + def transform_config(self, raw_config): + return raw_config - def check(self, logger, rendered_config_path) -> AirbyteCheckResponse: + def write_config(self, config_object, path): + with open(path, 'w') as fh: + fh.write(json.dumps(config_object)) + + def check(self, logger, config_container) -> AirbyteCheckResponse: raise Exception("Not Implemented") - def discover(self, logger, rendered_config_path) -> AirbyteCatalog: + def discover(self, logger, config_container) -> AirbyteCatalog: raise Exception("Not Implemented") @@ -71,7 +70,7 @@ def __init__(self): pass # Iterator - def read(self, logger, rendered_config_path, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: + def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: raise Exception("Not Implemented") @@ -95,3 +94,10 @@ def log_line(line, default_level): log_record = AirbyteLogMessage(level=log_level, message=rendered_line) log_message = AirbyteMessage(type="LOG", log=log_record) print(log_message.serialize()) + +@dataclass +class ConfigContainer: + raw_config: object + rendered_config: object + raw_config_path: str + rendered_config_path: str diff --git a/airbyte-integrations/base-python/base.py b/airbyte-integrations/base-python/base.py index 5b73d079b3817..248403c52c9de 100644 --- a/airbyte-integrations/base-python/base.py +++ b/airbyte-integrations/base-python/base.py @@ -5,6 +5,7 @@ import os.path import importlib +from airbyte_protocol import ConfigContainer from airbyte_protocol import Source from airbyte_protocol import AirbyteLogMessage from airbyte_protocol import AirbyteMessage @@ -76,12 +77,18 @@ def start(self): sys.exit(0) rendered_config_path = os.path.join(temp_dir, 'config.json') - config_object = source.read_config(parsed_args.config) - source.render_config(config_object, rendered_config_path) + raw_config = source.read_config(parsed_args.config) + rendered_config = source.transform_config(raw_config) + source.write_config(rendered_config, rendered_config_path) + + config_container = ConfigContainer( + raw_config=raw_config, + rendered_config=rendered_config, + raw_config_path=parsed_args.config, + rendered_config_path=rendered_config_path) - # todo: output message for check if cmd == "check": - check_result = source.check(log_line, rendered_config_path) + check_result = source.check(log_line, config_container) if check_result.successful: log("INFO", "Check succeeded") sys.exit(0) @@ -89,11 +96,11 @@ def start(self): log("ERROR", "Check failed") sys.exit(1) elif cmd == "discover": - catalog = source.discover(log_line, rendered_config_path) + catalog = source.discover(log_line, config_container) print(catalog.serialize()) sys.exit(0) elif cmd == "read": - generator = source.read(log_line, rendered_config_path, parsed_args.catalog, parsed_args.state) + generator = source.read(log_line, config_container, parsed_args.catalog, parsed_args.state) for message in generator: print(message.serialize()) sys.exit(0) diff --git a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py index ae5853e98741c..d48bc399f4fdf 100644 --- a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py +++ b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py @@ -20,15 +20,15 @@ def __init__(self): def spec(self) -> AirbyteSpec: return SingerHelper.spec_from_file("/airbyte/exchangeratesapi-files/spec.json") - def check(self, logger, rendered_config_path) -> AirbyteCheckResponse: + def check(self, logger, config_container) -> AirbyteCheckResponse: code = urllib.request.urlopen("https://api.exchangeratesapi.io/").getcode() return AirbyteCheckResponse(code == 200, {}) - def discover(self, logger, rendered_config_path) -> AirbyteCatalog: + def discover(self, logger, config_container) -> AirbyteCatalog: return get_catalogs(logger).airbyte_catalog - def read(self, logger, rendered_config_path, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: + def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: if state: - return SingerHelper.read(logger, f"tap-exchangeratesapi --config {rendered_config_path} --state {state}") + return SingerHelper.read(logger, f"tap-exchangeratesapi --config {config_container.rendered_config_path} --state {state}") else: - return SingerHelper.read(logger, f"tap-exchangeratesapi --config {rendered_config_path}") + return SingerHelper.read(logger, f"tap-exchangeratesapi --config {config_container.rendered_config_path}") From d39476671ba016d3d55fe7754f17397fa387c4f4 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 13 Oct 2020 08:20:49 -0700 Subject: [PATCH 07/31] add tests and fix logging for check command --- airbyte-integrations/base-python/base.py | 5 +- .../source_exchangeratesapi_singer.py | 1 + .../SingerExchangeRatesApiSourceTest.java | 53 +++++++++++++++++-- 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/base-python/base.py b/airbyte-integrations/base-python/base.py index 248403c52c9de..5fbd56e7ab4de 100644 --- a/airbyte-integrations/base-python/base.py +++ b/airbyte-integrations/base-python/base.py @@ -1,5 +1,4 @@ import argparse -import logging import sys import tempfile import os.path @@ -90,10 +89,10 @@ def start(self): if cmd == "check": check_result = source.check(log_line, config_container) if check_result.successful: - log("INFO", "Check succeeded") + log_line("Check succeeded", "INFO") sys.exit(0) else: - log("ERROR", "Check failed") + log_line("Check failed", "ERROR") sys.exit(1) elif cmd == "discover": catalog = source.discover(log_line, config_container) diff --git a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py index d48bc399f4fdf..e955bdfa36fee 100644 --- a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py +++ b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py @@ -22,6 +22,7 @@ def spec(self) -> AirbyteSpec: def check(self, logger, config_container) -> AirbyteCheckResponse: code = urllib.request.urlopen("https://api.exchangeratesapi.io/").getcode() + logger(f"Ping response code: {code}", "INFO") return AirbyteCheckResponse(code == 200, {}) def discover(self, logger, config_container) -> AirbyteCatalog: diff --git a/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceTest.java b/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceTest.java index 3ae95c83b28dc..caeb159715e8f 100644 --- a/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceTest.java +++ b/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceTest.java @@ -24,14 +24,14 @@ package io.airbyte.integration_tests.sources; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.workers.WorkerException; import io.airbyte.workers.process.DockerProcessBuilderFactory; import io.airbyte.workers.process.ProcessBuilderFactory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -40,8 +40,9 @@ import java.time.temporal.ChronoUnit; import java.util.Date; import java.util.Optional; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class SingerExchangeRatesApiSourceTest { @@ -73,6 +74,26 @@ public void setUp() throws IOException { "host"); } + @Test + public void testSpec() throws InterruptedException, IOException, WorkerException { + Process process = createSpecProcess(); + process.waitFor(); + + assertEquals(0, process.exitValue()); + + // todo: add assertion to ensure the spec matches the schema / is serializable + } + + @Test + public void testCheck() throws IOException, WorkerException, InterruptedException { + IOs.writeFile(jobRoot, CONFIG, "{}"); + + Process process = createCheckProcess(CONFIG); + process.waitFor(); + + assertEquals(0, process.exitValue()); + } + @Test public void testSuccessfulDiscover() throws IOException, InterruptedException, WorkerException { IOs.writeFile(jobRoot, CONFIG, "{}"); @@ -110,6 +131,28 @@ public void testSync() throws IOException, InterruptedException, WorkerException .get("CAD").asDouble() > 0); } + private Process createSpecProcess() throws IOException, WorkerException { + return pbf.create( + jobRoot, + IMAGE_NAME, + "spec") + .redirectOutput(ProcessBuilder.Redirect.INHERIT) + .redirectError(ProcessBuilder.Redirect.INHERIT) + .start(); + } + + private Process createCheckProcess(String configFileName) throws IOException, WorkerException { + return pbf.create( + jobRoot, + IMAGE_NAME, + "check", + "--config", + configFileName) + .redirectOutput(ProcessBuilder.Redirect.INHERIT) + .redirectError(ProcessBuilder.Redirect.INHERIT) + .start(); + } + private Process createDiscoveryProcess(String configFileName) throws IOException, WorkerException { return pbf.create( jobRoot, From cde03b9ac6588388fb0514fd9f67ef0478e3f0b0 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 13 Oct 2020 08:38:42 -0700 Subject: [PATCH 08/31] move michel's branch into this one --- .../stripe_abprotocol/source/.dockerignore | 7 + .../stripe_abprotocol/source/.gitignore | 2 + .../stripe_abprotocol/source/Dockerfile | 20 + .../singer/stripe_abprotocol/source/README.md | 21 + .../stripe_abprotocol/source/build.gradle | 62 + .../singer/stripe_abprotocol/source/setup.py | 15 + .../source/source_stripe_singer/__init__.py | 2 + .../source_stripe_singer.py | 31 + .../singer/stripe_abprotocol/source/spec.json | 29 + .../sources/SingerStripeSourceTest.java | 241 + .../test-integration/resources/catalog.json | 1003 +++ .../src/test-integration/resources/spec.json | 29 + .../resources/stripe_catalog.json | 5904 +++++++++++++++++ .../resources/stripe_schema_message.json | 862 +++ .../resources/sync_output_subset.txt | 4 + 15 files changed, 8232 insertions(+) create mode 100644 airbyte-integrations/singer/stripe_abprotocol/source/.dockerignore create mode 100644 airbyte-integrations/singer/stripe_abprotocol/source/.gitignore create mode 100644 airbyte-integrations/singer/stripe_abprotocol/source/Dockerfile create mode 100644 airbyte-integrations/singer/stripe_abprotocol/source/README.md create mode 100644 airbyte-integrations/singer/stripe_abprotocol/source/build.gradle create mode 100644 airbyte-integrations/singer/stripe_abprotocol/source/setup.py create mode 100644 airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/__init__.py create mode 100644 airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py create mode 100644 airbyte-integrations/singer/stripe_abprotocol/source/spec.json create mode 100644 airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java create mode 100644 airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/catalog.json create mode 100644 airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/spec.json create mode 100644 airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/stripe_catalog.json create mode 100644 airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/stripe_schema_message.json create mode 100644 airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/sync_output_subset.txt diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/.dockerignore b/airbyte-integrations/singer/stripe_abprotocol/source/.dockerignore new file mode 100644 index 0000000000000..37171c8176965 --- /dev/null +++ b/airbyte-integrations/singer/stripe_abprotocol/source/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!entrypoint.sh +!source_stripe_singer/*.py +!spec.json +!setup.py + diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/.gitignore b/airbyte-integrations/singer/stripe_abprotocol/source/.gitignore new file mode 100644 index 0000000000000..f3fb4d3ca286b --- /dev/null +++ b/airbyte-integrations/singer/stripe_abprotocol/source/.gitignore @@ -0,0 +1,2 @@ +config/* + diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/Dockerfile b/airbyte-integrations/singer/stripe_abprotocol/source/Dockerfile new file mode 100644 index 0000000000000..eecab6f64d067 --- /dev/null +++ b/airbyte-integrations/singer/stripe_abprotocol/source/Dockerfile @@ -0,0 +1,20 @@ +FROM airbyte/base-singer:dev + +RUN apt-get update && apt-get install -y jq curl bash + +COPY spec.json /airbyte/stripe-files/spec.json + +WORKDIR /airbyte/source_stripe_singer + +COPY source_stripe_singer/*.py ./source_stripe_singer/ + +COPY setup.py . +RUN pip install . + +WORKDIR /airbyte + +ENV AIRBYTE_IMPL_MODULE="source_stripe_singer" +ENV AIRBYTE_IMPL_PATH="SourceStripeSinger" + +LABEL io.airbyte.version=0.1.3 + diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/README.md b/airbyte-integrations/singer/stripe_abprotocol/source/README.md new file mode 100644 index 0000000000000..12e117b92a9bb --- /dev/null +++ b/airbyte-integrations/singer/stripe_abprotocol/source/README.md @@ -0,0 +1,21 @@ +# Stripe Test Configuration + +In order to test the Stripe source, you will need API credentials and the ability to create records within Stripe. + +## Community Contributor + +1. Create an empty account on Stripe. +1. Create a file at `config/config.json` with the following format using your client secret and account id: +``` +{ + "client_secret": "sk_XXXXXXXXXXX", + "account_id": "acct_XXXXXXXX", + "start_date": "2017-01-01T00:00:00Z" +} +``` + +## Airbyte Employee + +1. Access the `Stripe Integration Test Config` secret on Rippling under the `Engineering` folder +1. Create a file with the contents at `config/config.json` + diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/build.gradle b/airbyte-integrations/singer/stripe_abprotocol/source/build.gradle new file mode 100644 index 0000000000000..91fd14880e185 --- /dev/null +++ b/airbyte-integrations/singer/stripe_abprotocol/source/build.gradle @@ -0,0 +1,62 @@ +import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage +plugins { + id 'java' + id 'com.bmuschko.docker-remote-api' +} + +// TODO: rename without abprotocol when we can get rid of singer protocol +def image = 'airbyte/source-stripe-abprotocol-singer:dev' + +sourceSets { + integrationTest { + java { + srcDir 'src/test-integration/java' + } + resources { + srcDir 'src/test-integration/resources' + } + } +} +test.dependsOn('compileIntegrationTestJava') + +configurations { + integrationTestImplementation.extendsFrom testImplementation + integrationTestRuntimeOnly.extendsFrom testRuntimeOnly +} + +dependencies { + integrationTestImplementation 'org.apache.commons:commons-dbcp2:2.7.0' + integrationTestImplementation 'com.fasterxml.jackson.core:jackson-databind' + integrationTestImplementation 'org.apache.commons:commons-text:1.9' + integrationTestImplementation "com.stripe:stripe-java:20.6.0" + + integrationTestImplementation project(':airbyte-workers') + integrationTestImplementation project(':airbyte-config:models') +} + +task imageName { + doLast { + println "IMAGE $image" + } +} + +task buildImage(type: DockerBuildImage) { + inputDir = projectDir + images.add(image) + dependsOn ":airbyte-integrations:singer:base-singer:buildImage" +} + +task integrationTest(type: Test) { + testClassesDirs += sourceSets.integrationTest.output.classesDirs + classpath += sourceSets.integrationTest.runtimeClasspath + + useJUnitPlatform() + testLogging() { + events "passed", "failed" + exceptionFormat "full" + } + + mustRunAfter test + dependsOn(buildImage) +} + diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/setup.py b/airbyte-integrations/singer/stripe_abprotocol/source/setup.py new file mode 100644 index 0000000000000..6c32f14290919 --- /dev/null +++ b/airbyte-integrations/singer/stripe_abprotocol/source/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup + +setup( + name='source_stripe_singer', + description='Source implementation for Stripe.', + author='Airbyte', + author_email='contact@airbyte.io', + packages=['source_stripe_singer'], + install_requires=[ + 'tap-stripe==1.4.4', + 'requests', + 'base_singer', + 'airbyte_protocol'] +) + diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/__init__.py b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/__init__.py new file mode 100644 index 0000000000000..3e9669349d030 --- /dev/null +++ b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/__init__.py @@ -0,0 +1,2 @@ +from .source_stripe_singer import * + diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py new file mode 100644 index 0000000000000..e43b8a1d41a36 --- /dev/null +++ b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py @@ -0,0 +1,31 @@ +from airbyte_protocol import Source +from airbyte_protocol import AirbyteSpec +from airbyte_protocol import AirbyteCheckResponse +from airbyte_protocol import AirbyteCatalog +from airbyte_protocol import AirbyteMessage +import requests +from typing import Generator +from base_singer import SingerHelper +import sys + +class SourceStripeSinger(Source): + def __init__(self): + pass + + def spec(self) -> AirbyteSpec: + return SingerHelper.spec_from_file('/airbyte/stripe-files/spec.json') + + def check(self, logger, config_container) -> AirbyteCheckResponse: + json_config = config_container.rendered_config + r = requests.get('https://api.stripe.com/v1/customers', auth=(json_config['client_secret'], '')) + + return AirbyteCheckResponse(r.status_code == 200, {}) + + def discover(self, logger, config_container) -> AirbyteCatalog: + return SingerHelper.discover(f"tap-stripe --config {config_container.rendered_config_path} --discover") + + def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: + if state: + return SingerHelper.read(logger, f"tap-stripe --config {config_container.rendered_config_path} --state {state}") + else: + return SingerHelper.read(logger, f"tap-stripe --config {config_container.rendered_config_path}") diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/spec.json b/airbyte-integrations/singer/stripe_abprotocol/source/spec.json new file mode 100644 index 0000000000000..35d10e8f5128a --- /dev/null +++ b/airbyte-integrations/singer/stripe_abprotocol/source/spec.json @@ -0,0 +1,29 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/stripe", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Stripe Source Spec", + "type": "object", + "required": ["client_secret", "account_id", "start_date"], + "additionalProperties": false, + "properties": { + "client_secret": { + "type": "string", + "pattern": "^(s|r)k_(live|test)_[a-zA-Z0-9]+$", + "description": "Stripe API key (usually starts with 'sk_live_'; find yours here)." + }, + "account_id": { + "type": "string", + "pattern": "^acct_[a-zA-Z0-9]+$", + "description": "Your Stripe account ID (starts with 'acct_', find yours here)." + }, + "start_date": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", + "examples": ["2017-01-25T00:00:00Z"] + } + } + } +} + diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java new file mode 100644 index 0000000000000..037e8da645da0 --- /dev/null +++ b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java @@ -0,0 +1,241 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integration_tests.sources; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.stripe.exception.StripeException; +import com.stripe.model.Customer; +import com.stripe.net.RequestOptions; +import com.stripe.param.CustomerCreateParams; +import com.stripe.param.CustomerListParams; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.workers.WorkerException; +import io.airbyte.workers.process.AirbyteIntegrationLauncher; +import io.airbyte.workers.process.DockerProcessBuilderFactory; +import io.airbyte.workers.process.IntegrationLauncher; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class SingerStripeSourceTest { + + private static final Path TESTS_PATH = Path.of("/tmp/airbyte_integration_tests"); + private static final String IMAGE_NAME = "airbyte/source-stripe-abprotocol-singer:dev"; + + private static final String CATALOG = "catalog.json"; + private static final String CONFIG = "config.json"; + private static final String CONFIG_PATH = "config/config.json"; + private static final String INVALID_CONFIG = "invalid_config.json"; + + protected Path jobRoot; + protected Path workspaceRoot; + protected IntegrationLauncher launcher; + protected Path catalogPath; + + @BeforeEach + public void setUp() throws IOException, StripeException { + createTestRecordsIfNonExistent(); + + Files.createDirectories(TESTS_PATH); + workspaceRoot = Files.createTempDirectory(TESTS_PATH, "stripe"); + jobRoot = Path.of(workspaceRoot.toString(), "job"); + Files.createDirectories(jobRoot); + + catalogPath = jobRoot.resolve(CATALOG); + + writeConfigFilesToJobRoot(); + + launcher = new AirbyteIntegrationLauncher( + IMAGE_NAME, + new DockerProcessBuilderFactory(workspaceRoot, workspaceRoot.toString(), "", "host")); + } + + private static String getEmail(int number) { + return "customer" + number + "@test.com"; + } + + private void createTestRecordsIfNonExistent() throws IOException, StripeException { + String credentialsJsonString = new String(Files.readAllBytes(Paths.get(CONFIG_PATH))); + JsonNode credentials = Jsons.deserialize(credentialsJsonString); + String stripeApiKey = credentials.get("client_secret").textValue(); + + RequestOptions requestOptions = RequestOptions.builder() + .setApiKey(stripeApiKey) + .build(); + + for (int i = 1; i <= 4; i++) { + String email = getEmail(i); + String phone = "" + i + i + i + "-" + i + i + i + "-" + i + i + i + i; + String description = "Customer " + i; + + List customers = Customer.list(CustomerListParams.builder().setEmail(email).build(), requestOptions).getData(); + + if (customers.isEmpty()) { + Customer.create( + CustomerCreateParams.builder() + .setEmail(email) + .setDescription(description) + .setPhone(phone) + .build(), + requestOptions); + } + } + } + + @Test + public void testGetSpec() throws WorkerException, IOException, InterruptedException { + Process process = launcher.spec(jobRoot).start(); + process.waitFor(); + + InputStream expectedSpecInputStream = Objects.requireNonNull(getClass().getClassLoader().getResourceAsStream("spec.json")); + + assertEquals( + Jsons.deserialize(new String(expectedSpecInputStream.readAllBytes())), + Jsons.deserialize(new String(process.getInputStream().readAllBytes()))); + } + + @Test + public void testInvalidCredentialsDiscover() throws IOException, InterruptedException, WorkerException { + Process process = createDiscoveryProcess(INVALID_CONFIG); + process.waitFor(); + + assertEquals(2, process.exitValue()); + } + + @Test + public void testSuccessfulDiscover() throws IOException, InterruptedException, WorkerException { + Process process = createDiscoveryProcess(CONFIG); + process.waitFor(); + + assertEquals(0, process.exitValue()); + + final String catalog = IOs.readFile(jobRoot, catalogPath.toString()); + + assertTrue(catalog.lines().count() > 10000); + assertTrue(catalog.contains("customer")); + assertTrue(catalog.contains("address_zip_check")); + } + + @Test + public void testSync() throws IOException, InterruptedException, WorkerException { + String catalog = MoreResources.readResource(CATALOG); + IOs.writeFile(catalogPath.getParent(), catalogPath.getFileName().toString(), catalog); + + // run syn process + Path syncOutputPath = jobRoot.resolve("sync_output.txt"); + Process process = createSyncProcess(syncOutputPath); + process.waitFor(); + + assertEquals(0, process.exitValue()); + + final Set actualSyncOutput = IOs.readFile(jobRoot, syncOutputPath.toString()).lines() + .map(Jsons::deserialize) + .map(SingerStripeSourceTest::normalize) + .collect(Collectors.toSet()); + + MoreResources.readResource("sync_output_subset.txt").lines() + .map(Jsons::deserialize) + .map(SingerStripeSourceTest::normalize) + .forEach(record -> assertTrue(actualSyncOutput.contains(record))); + } + + private static JsonNode normalize(JsonNode node) { + ObjectNode normalized = node.deepCopy(); + + if (normalized.get("type").textValue().equals("RECORD")) { + ObjectNode record = (ObjectNode) normalized.get("record"); + record.put("id", "id"); + record.put("created", "created"); + record.put("invoice_prefix", "invoice_prefix"); + record.put("updated", "updated"); + + normalized.replace("record", record); + } + + normalized.put("time_extracted", "time_extracted"); + + return normalized; + } + + private void writeConfigFilesToJobRoot() throws IOException { + writeValidConfigFile(); + writeInvalidConfigFile(); + } + + private void writeValidConfigFile() throws IOException { + String credentialsJsonString = new String(Files.readAllBytes(Paths.get(CONFIG_PATH))); + JsonNode credentials = Jsons.deserialize(credentialsJsonString); + + assertTrue(credentials.get("client_secret").textValue().startsWith("sk_test_")); + assertTrue(credentials.get("account_id").textValue().startsWith("acct_")); + assertEquals("2017-01-01T00:00:00Z", credentials.get("start_date").textValue()); + + Files.writeString( + Path.of(jobRoot.toString(), "config.json"), credentialsJsonString); + } + + private void writeInvalidConfigFile() throws IOException { + Map fullConfig = new HashMap<>(); + + fullConfig.put("client_secret", "sk_test_" + RandomStringUtils.randomAlphanumeric(20)); + fullConfig.put("account_id", "acct_" + RandomStringUtils.randomAlphanumeric(20)); + fullConfig.put("start_date", "2017-01-01T00:00:00Z"); + + Files.writeString(Path.of(jobRoot.toString(), INVALID_CONFIG), Jsons.serialize(fullConfig)); + } + + private Process createDiscoveryProcess(String configFileName) throws IOException, WorkerException { + return launcher.discover(jobRoot, configFileName) + .redirectOutput(catalogPath.toFile()) + .redirectError(ProcessBuilder.Redirect.INHERIT) + .start(); + } + + private Process createSyncProcess(Path syncOutputPath) throws IOException, WorkerException { + return launcher.read(jobRoot, CONFIG, CATALOG) + .redirectOutput(syncOutputPath.toFile()) + .redirectError(ProcessBuilder.Redirect.INHERIT) + .start(); + } + +} + diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/catalog.json b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/catalog.json new file mode 100644 index 0000000000000..09de4fe599824 --- /dev/null +++ b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/catalog.json @@ -0,0 +1,1003 @@ +{ + "streams": [ + { + "stream": "customers", + "tap_stream_id": "customers", + "schema": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "shipping": { + "type": ["null", "object"], + "properties": { + "address": { + "type": ["null", "object"], + "properties": { + "line2": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + } + } + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + } + }, + "sources": { + "anyOf": [ + { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "type": { + "type": ["null", "string"] + }, + "address_zip": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "card": { + "type": ["null", "object"], + "properties": { + "fingerprint": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "three_d_secure": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "brand": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "address_country": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "last4": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "brand": { + "type": ["null", "string"] + }, + "address_line2": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "usage": { + "type": ["null", "string"] + }, + "address_line1": { + "type": ["null", "string"] + }, + "owner": { + "type": ["null", "object"], + "properties": { + "verified_address": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "object"], + "properties": { + "line2": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + } + } + }, + "verified_email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "verified_name": { + "type": ["null", "string"] + }, + "verified_phone": { + "type": ["null", "string"] + } + } + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "client_secret": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "address_city": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "receiver": { + "type": ["null", "object"], + "properties": { + "refund_attributes_method": { + "type": ["null", "string"] + }, + "amount_returned": { + "type": ["null", "integer"] + }, + "amount_received": { + "type": ["null", "integer"] + }, + "refund_attributes_status": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "amount_charged": { + "type": ["null", "integer"] + } + } + }, + "flow": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "ach_credit_transfer": { + "type": ["null", "object"], + "properties": { + "bank_name": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "routing_number": { + "type": ["null", "string"] + }, + "swift_code": { + "type": ["null", "string"] + }, + "refund_account_holder_type": { + "type": ["null", "string"] + }, + "refund_account_holder_name": { + "type": ["null", "string"] + }, + "refund_account_number": { + "type": ["null", "string"] + }, + "refund_routing_number": { + "type": ["null", "string"] + }, + "account_number": { + "type": ["null", "string"] + } + } + }, + "customer": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "address_state": { + "type": ["null", "string"] + }, + "alipay": { + "type": ["null", "object"], + "properties": {} + }, + "bancontact": { + "type": ["null", "object"], + "properties": {} + }, + "eps": { + "type": ["null", "object"], + "properties": {} + }, + "ideal": { + "type": ["null", "object"], + "properties": {} + }, + "multibanco": { + "type": ["null", "object"], + "properties": {} + }, + "redirect": { + "type": ["null", "object"], + "properties": { + "failure_reason": { + "type": ["null", "string"] + }, + "return_url": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + } + } + } + }, + { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "type": { + "type": ["null", "string"] + }, + "address_zip": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "card": { + "type": ["null", "object"], + "properties": { + "fingerprint": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "three_d_secure": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "brand": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "address_country": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "last4": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "brand": { + "type": ["null", "string"] + }, + "address_line2": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "usage": { + "type": ["null", "string"] + }, + "address_line1": { + "type": ["null", "string"] + }, + "owner": { + "type": ["null", "object"], + "properties": { + "verified_address": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "object"], + "properties": { + "line2": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + } + } + }, + "verified_email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "verified_name": { + "type": ["null", "string"] + }, + "verified_phone": { + "type": ["null", "string"] + } + } + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "client_secret": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "address_city": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "receiver": { + "type": ["null", "object"], + "properties": { + "refund_attributes_method": { + "type": ["null", "string"] + }, + "amount_returned": { + "type": ["null", "integer"] + }, + "amount_received": { + "type": ["null", "integer"] + }, + "refund_attributes_status": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "amount_charged": { + "type": ["null", "integer"] + } + } + }, + "flow": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "ach_credit_transfer": { + "type": ["null", "object"], + "properties": { + "bank_name": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "routing_number": { + "type": ["null", "string"] + }, + "swift_code": { + "type": ["null", "string"] + }, + "refund_account_holder_type": { + "type": ["null", "string"] + }, + "refund_account_holder_name": { + "type": ["null", "string"] + }, + "refund_account_number": { + "type": ["null", "string"] + }, + "refund_routing_number": { + "type": ["null", "string"] + }, + "account_number": { + "type": ["null", "string"] + } + } + }, + "customer": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "address_state": { + "type": ["null", "string"] + }, + "alipay": { + "type": ["null", "object"], + "properties": {} + }, + "bancontact": { + "type": ["null", "object"], + "properties": {} + }, + "eps": { + "type": ["null", "object"], + "properties": {} + }, + "ideal": { + "type": ["null", "object"], + "properties": {} + }, + "multibanco": { + "type": ["null", "object"], + "properties": {} + }, + "redirect": { + "type": ["null", "object"], + "properties": { + "failure_reason": { + "type": ["null", "string"] + }, + "return_url": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + } + } + } + ] + }, + "delinquent": { + "type": ["null", "boolean"] + }, + "description": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "default_source": { + "type": ["null", "string"] + }, + "cards": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "object": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "last4": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "brand": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "customer": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "address_line2": { + "type": ["null", "string"] + }, + "address_line1": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "address_zip": { + "type": ["null", "string"] + }, + "address_city": { + "type": ["null", "string"] + }, + "address_country": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address_state": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + } + }, + "email": { + "type": ["null", "string"] + }, + "default_card": { + "type": ["null", "string"] + }, + "subscriptions": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "discount": { + "type": ["null", "object"], + "properties": { + "end": { + "type": ["null", "string"], + "format": "date-time" + }, + "coupon": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "valid": { + "type": ["null", "boolean"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "amount_off": { + "type": ["null", "integer"] + }, + "redeem_by": { + "type": ["null", "string"], + "format": "date-time" + }, + "duration_in_months": { + "type": ["null", "integer"] + }, + "percent_off_precise": { + "type": ["null", "number"] + }, + "max_redemptions": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "times_redeemed": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, + "duration": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "percent_off": { + "type": ["null", "integer"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "customer": { + "type": ["null", "string"] + }, + "start": { + "type": ["null", "string"], + "format": "date-time" + }, + "object": { + "type": ["null", "string"] + }, + "subscription": { + "type": ["null", "string"] + } + } + }, + "account_balance": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "invoice_prefix": { + "type": ["null", "string"] + }, + "tax_info_verification": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "tax_info": { + "type": ["null", "string"] + }, + "updated": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "metadata": [ + { + "breadcrumb": [], + "metadata": { + "table-key-properties": ["id"], + "forced-replication-method": "INCREMENTAL", + "selected": "true", + "valid-replication-keys": ["created"] + } + }, + { + "breadcrumb": ["properties", "metadata"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "shipping"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "sources"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "delinquent"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "description"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "livemode"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "default_source"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "cards"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "email"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "default_card"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "subscriptions"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "discount"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "account_balance"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "currency"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "id"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "invoice_prefix"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "tax_info_verification"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "object"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "created"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "tax_info"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "updated"], + "metadata": { + "inclusion": "automatic" + } + } + ], + "key_properties": ["id"] + } + ] +} + diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/spec.json b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/spec.json new file mode 100644 index 0000000000000..35d10e8f5128a --- /dev/null +++ b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/spec.json @@ -0,0 +1,29 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/stripe", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Stripe Source Spec", + "type": "object", + "required": ["client_secret", "account_id", "start_date"], + "additionalProperties": false, + "properties": { + "client_secret": { + "type": "string", + "pattern": "^(s|r)k_(live|test)_[a-zA-Z0-9]+$", + "description": "Stripe API key (usually starts with 'sk_live_'; find yours here)." + }, + "account_id": { + "type": "string", + "pattern": "^acct_[a-zA-Z0-9]+$", + "description": "Your Stripe account ID (starts with 'acct_', find yours here)." + }, + "start_date": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", + "examples": ["2017-01-25T00:00:00Z"] + } + } + } +} + diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/stripe_catalog.json b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/stripe_catalog.json new file mode 100644 index 0000000000000..b091044ce7f0d --- /dev/null +++ b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/stripe_catalog.json @@ -0,0 +1,5904 @@ +{ + "streams": [ + { + "stream": "charges", + "tap_stream_id": "charges", + "schema": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "fraud_details": { + "type": ["null", "object"], + "properties": { + "stripe_report": { + "type": ["null", "string"] + } + } + }, + "transfer_group": { + "type": ["null", "string"] + }, + "on_behalf_of": { + "type": ["null", "string"] + }, + "review": { + "type": ["null", "string"] + }, + "failure_message": { + "type": ["null", "string"] + }, + "receipt_email": { + "type": ["null", "string"] + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "type": { + "type": ["null", "string"] + }, + "address_zip": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "card": { + "type": ["null", "object"], + "properties": { + "fingerprint": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "three_d_secure": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "brand": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "address_country": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "last4": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "brand": { + "type": ["null", "string"] + }, + "address_line2": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "usage": { + "type": ["null", "string"] + }, + "address_line1": { + "type": ["null", "string"] + }, + "owner": { + "type": ["null", "object"], + "properties": { + "verified_address": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "object"], + "properties": { + "line2": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + } + } + }, + "verified_email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "verified_name": { + "type": ["null", "string"] + }, + "verified_phone": { + "type": ["null", "string"] + } + } + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "client_secret": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "address_city": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "receiver": { + "type": ["null", "object"], + "properties": { + "refund_attributes_method": { + "type": ["null", "string"] + }, + "amount_returned": { + "type": ["null", "integer"] + }, + "amount_received": { + "type": ["null", "integer"] + }, + "refund_attributes_status": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "amount_charged": { + "type": ["null", "integer"] + } + } + }, + "flow": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "ach_credit_transfer": { + "type": ["null", "object"], + "properties": { + "bank_name": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "routing_number": { + "type": ["null", "string"] + }, + "swift_code": { + "type": ["null", "string"] + }, + "refund_account_holder_type": { + "type": ["null", "string"] + }, + "refund_account_holder_name": { + "type": ["null", "string"] + }, + "refund_account_number": { + "type": ["null", "string"] + }, + "refund_routing_number": { + "type": ["null", "string"] + }, + "account_number": { + "type": ["null", "string"] + } + } + }, + "customer": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "address_state": { + "type": ["null", "string"] + }, + "alipay": { + "type": ["null", "object"], + "properties": {} + }, + "bancontact": { + "type": ["null", "object"], + "properties": {} + }, + "eps": { + "type": ["null", "object"], + "properties": {} + }, + "ideal": { + "type": ["null", "object"], + "properties": {} + }, + "multibanco": { + "type": ["null", "object"], + "properties": {} + }, + "redirect": { + "type": ["null", "object"], + "properties": { + "failure_reason": { + "type": ["null", "string"] + }, + "return_url": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + } + } + }, + "destination": { + "type": ["null", "string"] + }, + "id": { + "type": ["string"] + }, + "object": { + "type": ["null", "string"] + }, + "outcome": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "seller_message": { + "type": ["null", "string"] + }, + "reason": { + "type": ["null", "string"] + }, + "risk_level": { + "type": ["null", "string"] + }, + "network_status": { + "type": ["null", "string"] + }, + "risk_score": { + "type": ["null", "integer"] + } + } + }, + "status": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "order": { + "type": ["null", "string"] + }, + "application": { + "type": ["null", "string"] + }, + "refunded": { + "type": ["null", "boolean"] + }, + "receipt_number": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "captured": { + "type": ["null", "boolean"] + }, + "paid": { + "type": ["null", "boolean"] + }, + "shipping": { + "type": ["null", "object"], + "properties": {} + }, + "invoice": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "customer": { + "type": ["null", "string"] + }, + "payment_intent": { + "type": ["null", "string"] + }, + "source_transfer": { + "type": ["null", "string"] + }, + "statement_description": { + "type": ["null", "string"] + }, + "refunds": { + "type": ["null", "array"], + "items": {} + }, + "application_fee": { + "type": ["null", "string"] + }, + "card": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "exp_month": { + "type": ["null", "integer"] + }, + "address_state": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "address_line1": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "address_country": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "address_city": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "address_line2": { + "type": ["null", "string"] + }, + "customer": { + "type": ["null", "string"] + }, + "brand": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "address_zip": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + }, + "payment_method_details": { + "type": ["null", "object"], + "properties": { + "ach_credit_transfer": { + "type": ["null", "object"], + "properties": { + "account_number": { + "type": ["null", "string"] + }, + "bank_name": { + "type": ["null", "string"] + }, + "routing_number": { + "type": ["null", "string"] + }, + "swift_code": { + "type": ["null", "string"] + } + } + }, + "ach_debit": { + "type": ["null", "object"], + "properties": { + "account_holder_type": { + "type": ["null", "string"] + }, + "bank_name": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "routing_number": { + "type": ["null", "string"] + } + } + }, + "alipay": { + "type": ["null", "object"] + }, + "bancontact": { + "type": ["null", "object"], + "properties": { + "bank_code": { + "type": ["null", "string"] + }, + "bank_name": { + "type": ["null", "string"] + }, + "bic": { + "type": ["null", "string"] + }, + "iban_last4": { + "type": ["null", "string"] + }, + "preferred_language": { + "type": ["null", "string"] + }, + "verified_name": { + "type": ["null", "string"] + } + } + }, + "card": { + "type": ["null", "object"], + "properties": { + "brand": { + "type": ["null", "string"] + }, + "checks": { + "type": ["null", "object"], + "properties": { + "address_line1_check": { + "type": ["null", "string"] + }, + "address_postal_code_check": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + } + } + }, + "country": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "installments": { + "type": ["null", "object"], + "properties": { + "plan": { + "type": ["null", "object"], + "properties": { + "count": { + "type": ["null", "integer"] + }, + "interval": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + } + } + }, + "last4": { + "type": ["null", "string"] + }, + "network": { + "type": ["null", "string"] + }, + "three_d_secure": { + "type": ["null", "object"], + "properties": { + "authenticated": { + "type": ["null", "boolean"] + }, + "succeeded": { + "type": ["null", "boolean"] + }, + "version": { + "type": ["null", "string"] + } + } + }, + "wallet": { + "type": ["null", "object"], + "properties": { + "amex_express_checkout": { + "type": ["null", "object"], + "properties": {} + }, + "apple_pay": { + "type": ["null", "object"], + "properties": {} + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "google_pay": { + "type": ["null", "object"], + "properties": {} + }, + "masterpass": { + "type": ["null", "object"], + "properties": { + "billing_address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + }, + "line2": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + } + } + }, + "email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "shipping_address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + }, + "line2": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + } + } + } + } + }, + "samsung_pay": { + "type": ["null", "object"], + "properties": {} + }, + "type": { + "type": ["null", "string"] + }, + "visa_checkout": { + "type": ["null", "object"], + "properties": { + "billing_address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + }, + "line2": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + } + } + }, + "email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "shipping_address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + }, + "line2": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + } + } + } + } + } + } + }, + "card_present": { + "type": ["null", "object"], + "properties": { + "brand": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "emv_auth_data": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "generated_card": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "network": { + "type": ["null", "string"] + }, + "read_method": { + "type": ["null", "string"] + }, + "receipt": { + "type": ["null", "object"], + "properties": { + "application_cryptogram": { + "type": ["null", "string"] + }, + "application_preferred_name": { + "type": ["null", "string"] + }, + "authorization_code": { + "type": ["null", "string"] + }, + "authorization_response_code": { + "type": ["null", "string"] + }, + "cardholder_verification_method": { + "type": ["null", "string"] + }, + "dedicated_file_name": { + "type": ["null", "string"] + }, + "terminal_verification_results": { + "type": ["null", "string"] + }, + "transaction_status_information": { + "type": ["null", "string"] + } + } + } + } + }, + "eps": { + "type": ["null", "object"], + "properties": { + "verified_name": { + "type": ["null", "string"] + } + } + }, + "giropay": { + "type": ["null", "object"], + "properties": { + "bank_code": { + "type": ["null", "string"] + }, + "bank_name": { + "type": ["null", "string"] + }, + "bic": { + "type": ["null", "string"] + }, + "verified_name": { + "type": ["null", "string"] + } + } + }, + "ideal": { + "type": ["null", "object"], + "properties": { + "bank": { + "type": ["null", "string"] + }, + "bic": { + "type": ["null", "string"] + }, + "iban_last4": { + "type": ["null", "string"] + }, + "verified_name": { + "type": ["null", "string"] + } + } + }, + "klarna": { + "type": ["null", "object"], + "properties": {} + }, + "multibanco": { + "type": ["null", "object"], + "properties": { + "entity": { + "type": ["null", "string"] + }, + "reference": { + "type": ["null", "string"] + } + } + }, + "p24": { + "type": ["null", "object"], + "properties": { + "reference": { + "type": ["null", "string"] + }, + "verified_name": { + "type": ["null", "string"] + } + } + }, + "sepa_debit": { + "type": ["null", "object"], + "properties": { + "bank_code": { + "type": ["null", "string"] + }, + "branch_code": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "mandate": { + "type": ["null", "string"] + } + } + }, + "sofort": { + "type": ["null", "object"], + "properties": { + "bank_code": { + "type": ["null", "string"] + }, + "bank_name": { + "type": ["null", "string"] + }, + "bic": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "iban_last4": { + "type": ["null", "string"] + }, + "verified_name": { + "type": ["null", "string"] + } + } + }, + "stripe_account": { + "type": ["null", "object"], + "properties": {} + }, + "type": { + "type": ["null", "string"] + }, + "wechat": { + "type": ["null", "object"], + "properties": {} + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "address_state": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "address_line1": { + "type": ["null", "string"] + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "address_country": { + "type": ["null", "string"] + }, + "address_city": { + "type": ["null", "string"] + }, + "address_line2": { + "type": ["null", "string"] + }, + "customer": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address_zip": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + } + } + }, + "type": { + "type": ["null", "string"] + } + } + }, + "balance_transaction": { + "type": ["null", "string"] + }, + "amount_refunded": { + "type": ["null", "integer"] + }, + "failure_code": { + "type": ["null", "string"] + }, + "dispute": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "updated": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "metadata": [ + { + "breadcrumb": [], + "metadata": { + "table-key-properties": ["id"], + "forced-replication-method": "INCREMENTAL", + "valid-replication-keys": ["created"] + } + }, + { + "breadcrumb": ["properties", "metadata"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "fraud_details"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "transfer_group"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "on_behalf_of"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "review"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "failure_message"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "receipt_email"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "statement_descriptor"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "source"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "destination"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "id"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "object"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "outcome"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "status"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "currency"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "created"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "order"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "application"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "refunded"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "receipt_number"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "livemode"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "captured"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "paid"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "shipping"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "invoice"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "amount"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "customer"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "payment_intent"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "source_transfer"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "statement_description"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "refunds"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "application_fee"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "card"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "payment_method_details"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "balance_transaction"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "amount_refunded"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "failure_code"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "dispute"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "description"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "updated"], + "metadata": { + "inclusion": "automatic" + } + } + ], + "key_properties": ["id"] + }, + { + "stream": "events", + "tap_stream_id": "events", + "schema": { + "type": "object", + "properties": { + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "data": { + "type": ["null", "object"], + "properties": {} + }, + "id": { + "type": ["null", "string"] + }, + "api_version": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "pending_webhooks": { + "type": ["null", "integer"] + }, + "request": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "updated": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "metadata": [ + { + "breadcrumb": [], + "metadata": { + "table-key-properties": ["id"], + "forced-replication-method": "INCREMENTAL", + "valid-replication-keys": ["created"] + } + }, + { + "breadcrumb": ["properties", "created"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "data"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "id"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "api_version"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "object"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "livemode"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "pending_webhooks"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "request"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "type"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "updated"], + "metadata": { + "inclusion": "automatic" + } + } + ], + "key_properties": ["id"] + }, + { + "stream": "customers", + "tap_stream_id": "customers", + "schema": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "shipping": { + "type": ["null", "object"], + "properties": { + "address": { + "type": ["null", "object"], + "properties": { + "line2": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + } + } + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + } + }, + "sources": { + "anyOf": [ + { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "type": { + "type": ["null", "string"] + }, + "address_zip": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "card": { + "type": ["null", "object"], + "properties": { + "fingerprint": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "three_d_secure": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "brand": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "address_country": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "last4": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "brand": { + "type": ["null", "string"] + }, + "address_line2": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "usage": { + "type": ["null", "string"] + }, + "address_line1": { + "type": ["null", "string"] + }, + "owner": { + "type": ["null", "object"], + "properties": { + "verified_address": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "object"], + "properties": { + "line2": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + } + } + }, + "verified_email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "verified_name": { + "type": ["null", "string"] + }, + "verified_phone": { + "type": ["null", "string"] + } + } + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "client_secret": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "address_city": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "receiver": { + "type": ["null", "object"], + "properties": { + "refund_attributes_method": { + "type": ["null", "string"] + }, + "amount_returned": { + "type": ["null", "integer"] + }, + "amount_received": { + "type": ["null", "integer"] + }, + "refund_attributes_status": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "amount_charged": { + "type": ["null", "integer"] + } + } + }, + "flow": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "ach_credit_transfer": { + "type": ["null", "object"], + "properties": { + "bank_name": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "routing_number": { + "type": ["null", "string"] + }, + "swift_code": { + "type": ["null", "string"] + }, + "refund_account_holder_type": { + "type": ["null", "string"] + }, + "refund_account_holder_name": { + "type": ["null", "string"] + }, + "refund_account_number": { + "type": ["null", "string"] + }, + "refund_routing_number": { + "type": ["null", "string"] + }, + "account_number": { + "type": ["null", "string"] + } + } + }, + "customer": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "address_state": { + "type": ["null", "string"] + }, + "alipay": { + "type": ["null", "object"], + "properties": {} + }, + "bancontact": { + "type": ["null", "object"], + "properties": {} + }, + "eps": { + "type": ["null", "object"], + "properties": {} + }, + "ideal": { + "type": ["null", "object"], + "properties": {} + }, + "multibanco": { + "type": ["null", "object"], + "properties": {} + }, + "redirect": { + "type": ["null", "object"], + "properties": { + "failure_reason": { + "type": ["null", "string"] + }, + "return_url": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + } + } + } + }, + { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "type": { + "type": ["null", "string"] + }, + "address_zip": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "card": { + "type": ["null", "object"], + "properties": { + "fingerprint": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "three_d_secure": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "brand": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "address_country": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "last4": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "brand": { + "type": ["null", "string"] + }, + "address_line2": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "usage": { + "type": ["null", "string"] + }, + "address_line1": { + "type": ["null", "string"] + }, + "owner": { + "type": ["null", "object"], + "properties": { + "verified_address": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "object"], + "properties": { + "line2": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + } + } + }, + "verified_email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "verified_name": { + "type": ["null", "string"] + }, + "verified_phone": { + "type": ["null", "string"] + } + } + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "client_secret": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "address_city": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "receiver": { + "type": ["null", "object"], + "properties": { + "refund_attributes_method": { + "type": ["null", "string"] + }, + "amount_returned": { + "type": ["null", "integer"] + }, + "amount_received": { + "type": ["null", "integer"] + }, + "refund_attributes_status": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "amount_charged": { + "type": ["null", "integer"] + } + } + }, + "flow": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "ach_credit_transfer": { + "type": ["null", "object"], + "properties": { + "bank_name": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "routing_number": { + "type": ["null", "string"] + }, + "swift_code": { + "type": ["null", "string"] + }, + "refund_account_holder_type": { + "type": ["null", "string"] + }, + "refund_account_holder_name": { + "type": ["null", "string"] + }, + "refund_account_number": { + "type": ["null", "string"] + }, + "refund_routing_number": { + "type": ["null", "string"] + }, + "account_number": { + "type": ["null", "string"] + } + } + }, + "customer": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "address_state": { + "type": ["null", "string"] + }, + "alipay": { + "type": ["null", "object"], + "properties": {} + }, + "bancontact": { + "type": ["null", "object"], + "properties": {} + }, + "eps": { + "type": ["null", "object"], + "properties": {} + }, + "ideal": { + "type": ["null", "object"], + "properties": {} + }, + "multibanco": { + "type": ["null", "object"], + "properties": {} + }, + "redirect": { + "type": ["null", "object"], + "properties": { + "failure_reason": { + "type": ["null", "string"] + }, + "return_url": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + } + } + } + ] + }, + "delinquent": { + "type": ["null", "boolean"] + }, + "description": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "default_source": { + "type": ["null", "string"] + }, + "cards": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "object": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "last4": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "brand": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "customer": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "address_line2": { + "type": ["null", "string"] + }, + "address_line1": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "address_zip": { + "type": ["null", "string"] + }, + "address_city": { + "type": ["null", "string"] + }, + "address_country": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address_state": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + } + }, + "email": { + "type": ["null", "string"] + }, + "default_card": { + "type": ["null", "string"] + }, + "subscriptions": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "discount": { + "type": ["null", "object"], + "properties": { + "end": { + "type": ["null", "string"], + "format": "date-time" + }, + "coupon": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "valid": { + "type": ["null", "boolean"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "amount_off": { + "type": ["null", "integer"] + }, + "redeem_by": { + "type": ["null", "string"], + "format": "date-time" + }, + "duration_in_months": { + "type": ["null", "integer"] + }, + "percent_off_precise": { + "type": ["null", "number"] + }, + "max_redemptions": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "times_redeemed": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, + "duration": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "percent_off": { + "type": ["null", "integer"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "customer": { + "type": ["null", "string"] + }, + "start": { + "type": ["null", "string"], + "format": "date-time" + }, + "object": { + "type": ["null", "string"] + }, + "subscription": { + "type": ["null", "string"] + } + } + }, + "account_balance": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "invoice_prefix": { + "type": ["null", "string"] + }, + "tax_info_verification": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "tax_info": { + "type": ["null", "string"] + }, + "updated": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "metadata": [ + { + "breadcrumb": [], + "metadata": { + "table-key-properties": ["id"], + "forced-replication-method": "INCREMENTAL", + "valid-replication-keys": ["created"] + } + }, + { + "breadcrumb": ["properties", "metadata"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "shipping"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "sources"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "delinquent"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "description"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "livemode"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "default_source"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "cards"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "email"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "default_card"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "subscriptions"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "discount"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "account_balance"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "currency"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "id"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "invoice_prefix"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "tax_info_verification"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "object"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "created"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "tax_info"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "updated"], + "metadata": { + "inclusion": "automatic" + } + } + ], + "key_properties": ["id"] + }, + { + "stream": "plans", + "tap_stream_id": "plans", + "schema": { + "type": ["null", "object", "string"], + "properties": { + "nickname": { + "type": ["null", "string"] + }, + "tiers": { + "type": ["null", "array"], + "items": { + "type": ["null", "string", "object"], + "properties": { + "flat_amount": { + "type": ["null", "integer"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "up_to": { + "type": ["null", "integer"] + } + } + } + }, + "object": { + "type": ["null", "string"] + }, + "aggregate_usage": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "statement_description": { + "type": ["null", "string"] + }, + "product": { + "type": ["null", "string"] + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "interval_count": { + "type": ["null", "integer"] + }, + "transform_usage": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "interval": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "trial_period_days": { + "type": ["null", "integer"] + }, + "usage_type": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "tiers_mode": { + "type": ["null", "string"] + }, + "billing_scheme": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "currency": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "updated": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "metadata": [ + { + "breadcrumb": [], + "metadata": { + "table-key-properties": ["id"], + "forced-replication-method": "INCREMENTAL", + "valid-replication-keys": ["created"] + } + }, + { + "breadcrumb": ["properties", "nickname"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "tiers"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "object"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "aggregate_usage"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "created"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "statement_description"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "product"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "statement_descriptor"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "interval_count"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "transform_usage"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "name"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "amount"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "interval"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "id"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "trial_period_days"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "usage_type"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "active"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "tiers_mode"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "billing_scheme"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "livemode"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "currency"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "metadata"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "updated"], + "metadata": { + "inclusion": "automatic" + } + } + ], + "key_properties": ["id"] + }, + { + "stream": "invoices", + "tap_stream_id": "invoices", + "schema": { + "type": ["null", "object"], + "properties": { + "date": { + "type": ["null", "string"], + "format": "date-time" + }, + "next_payment_attempt": { + "type": ["null", "string"], + "format": "date-time" + }, + "tax": { + "type": ["null", "integer"] + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "charge": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "receipt_number": { + "type": ["null", "string"] + }, + "attempt_count": { + "type": ["null", "integer"] + }, + "payment": { + "type": ["null", "string"] + }, + "amount_paid": { + "type": ["null", "integer"] + }, + "due_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "string"] + }, + "webhooks_delivered_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "hosted_invoice_url": { + "type": ["null", "string"] + }, + "period_end": { + "type": ["null", "string"], + "format": "date-time" + }, + "amount_remaining": { + "type": ["null", "integer"] + }, + "tax_percent": { + "type": ["null", "number"] + }, + "billing": { + "type": ["null", "string"] + }, + "auto_advance": { + "type": ["null", "boolean"] + }, + "paid": { + "type": ["null", "boolean"] + }, + "discount": { + "type": ["null", "object"], + "properties": { + "end": { + "type": ["null", "string"], + "format": "date-time" + }, + "coupon": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "valid": { + "type": ["null", "boolean"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "amount_off": { + "type": ["null", "integer"] + }, + "redeem_by": { + "type": ["null", "string"], + "format": "date-time" + }, + "duration_in_months": { + "type": ["null", "integer"] + }, + "percent_off_precise": { + "type": ["null", "number"] + }, + "max_redemptions": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "times_redeemed": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, + "duration": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "percent_off": { + "type": ["null", "integer"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "customer": { + "type": ["null", "string"] + }, + "start": { + "type": ["null", "string"], + "format": "date-time" + }, + "object": { + "type": ["null", "string"] + }, + "subscription": { + "type": ["null", "string"] + } + } + }, + "number": { + "type": ["null", "string"] + }, + "billing_reason": { + "type": ["null", "string"] + }, + "ending_balance": { + "type": ["null", "integer"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "period_start": { + "type": ["null", "string"], + "format": "date-time" + }, + "attempted": { + "type": ["null", "boolean"] + }, + "closed": { + "type": ["null", "boolean"] + }, + "invoice_pdf": { + "type": ["null", "string"] + }, + "customer": { + "type": ["null", "string"] + }, + "subtotal": { + "type": ["null", "integer"] + }, + "application_fee": { + "type": ["null", "integer"] + }, + "lines": { + "type": ["null", "array", "object"], + "items": { + "type": ["null", "string"] + }, + "properties": {} + }, + "forgiven": { + "type": ["null", "boolean"] + }, + "object": { + "type": ["null", "string"] + }, + "starting_balance": { + "type": ["null", "integer"] + }, + "amount_due": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "total": { + "type": ["null", "integer"] + }, + "statement_description": { + "type": ["null", "string"] + }, + "subscription": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "updated": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "metadata": [ + { + "breadcrumb": [], + "metadata": { + "table-key-properties": ["id"], + "forced-replication-method": "INCREMENTAL", + "valid-replication-keys": ["date"] + } + }, + { + "breadcrumb": ["properties", "date"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "next_payment_attempt"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "tax"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "metadata"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "charge"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "description"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "receipt_number"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "attempt_count"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "payment"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "amount_paid"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "due_date"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "id"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "webhooks_delivered_at"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "statement_descriptor"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "hosted_invoice_url"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "period_end"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "amount_remaining"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "tax_percent"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "billing"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "auto_advance"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "paid"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "discount"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "number"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "billing_reason"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "ending_balance"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "livemode"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "period_start"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "attempted"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "closed"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "invoice_pdf"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "customer"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "subtotal"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "application_fee"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "lines"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "forgiven"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "object"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "starting_balance"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "amount_due"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "currency"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "total"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "statement_description"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "subscription"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "status"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "updated"], + "metadata": { + "inclusion": "automatic" + } + } + ], + "key_properties": ["id"] + }, + { + "stream": "invoice_items", + "tap_stream_id": "invoice_items", + "schema": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "plan": { + "type": ["null", "object", "string"], + "properties": { + "nickname": { + "type": ["null", "string"] + }, + "tiers": { + "type": ["null", "array"], + "items": { + "type": ["null", "string", "object"], + "properties": { + "flat_amount": { + "type": ["null", "integer"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "up_to": { + "type": ["null", "integer"] + } + } + } + }, + "object": { + "type": ["null", "string"] + }, + "aggregate_usage": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "statement_description": { + "type": ["null", "string"] + }, + "product": { + "type": ["null", "string"] + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "interval_count": { + "type": ["null", "integer"] + }, + "transform_usage": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "interval": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "trial_period_days": { + "type": ["null", "integer"] + }, + "usage_type": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "tiers_mode": { + "type": ["null", "string"] + }, + "billing_scheme": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "currency": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "updated": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "invoice": { + "type": ["null", "string"] + }, + "period": { + "type": ["null", "object"], + "properties": { + "end": { + "type": ["null", "string"], + "format": "date-time" + }, + "start": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "quantity": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"], + "format": "date-time" + }, + "object": { + "type": ["null", "string"] + }, + "subscription": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "discountable": { + "type": ["null", "boolean"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "customer": { + "type": ["null", "string"] + }, + "proration": { + "type": ["null", "boolean"] + }, + "subscription_item": { + "type": ["null", "string"] + }, + "updated": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "metadata": [ + { + "breadcrumb": [], + "metadata": { + "table-key-properties": ["id"], + "forced-replication-method": "INCREMENTAL", + "valid-replication-keys": ["date"] + } + }, + { + "breadcrumb": ["properties", "amount"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "metadata"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "plan"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "invoice"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "period"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "quantity"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "description"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "date"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "object"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "subscription"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "id"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "livemode"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "discountable"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "unit_amount"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "currency"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "customer"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "proration"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "subscription_item"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "updated"], + "metadata": { + "inclusion": "automatic" + } + } + ], + "key_properties": ["id"] + }, + { + "stream": "invoice_line_items", + "tap_stream_id": "invoice_line_items", + "schema": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "invoice": { + "type": ["null", "string"] + }, + "subscription_item": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "description": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "discountable": { + "type": ["null", "boolean"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "proration": { + "type": ["null", "boolean"] + }, + "period": { + "type": ["null", "object"], + "properties": { + "start": { + "type": ["null", "string"], + "format": "date-time" + }, + "end": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "subscription": { + "type": ["null", "string"] + }, + "plan": { + "type": ["null", "object", "string"], + "properties": { + "nickname": { + "type": ["null", "string"] + }, + "tiers": { + "type": ["null", "array"], + "items": { + "type": ["null", "string", "object"], + "properties": { + "flat_amount": { + "type": ["null", "integer"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "up_to": { + "type": ["null", "integer"] + } + } + } + }, + "object": { + "type": ["null", "string"] + }, + "aggregate_usage": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "statement_description": { + "type": ["null", "string"] + }, + "product": { + "type": ["null", "string"] + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "interval_count": { + "type": ["null", "integer"] + }, + "transform_usage": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "interval": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "trial_period_days": { + "type": ["null", "integer"] + }, + "usage_type": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "tiers_mode": { + "type": ["null", "string"] + }, + "billing_scheme": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "currency": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "updated": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "invoice_item": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + } + } + }, + "metadata": [ + { + "breadcrumb": [], + "metadata": { + "table-key-properties": ["id", "invoice"], + "forced-replication-method": "INCREMENTAL" + } + }, + { + "breadcrumb": ["properties", "id"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "invoice"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "subscription_item"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "metadata"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "description"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "object"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "discountable"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "quantity"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "amount"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "type"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "livemode"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "proration"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "period"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "subscription"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "plan"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "invoice_item"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "currency"], + "metadata": { + "inclusion": "available" + } + } + ], + "key_properties": ["id", "invoice"] + }, + { + "stream": "transfers", + "tap_stream_id": "transfers", + "schema": { + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "reversals": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": {} + } + }, + "id": { + "type": ["null", "string"] + }, + "statement_description": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "balance_transaction": { + "type": ["null", "string"] + }, + "reversed": { + "type": ["null", "boolean"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "amount_reversed": { + "type": ["null", "integer"] + }, + "source_type": { + "type": ["null", "string"] + }, + "source_transaction": { + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"], + "format": "date-time" + }, + "livemode": { + "type": ["null", "boolean"] + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "failure_balance_transaction": { + "type": ["null", "string"] + }, + "recipient": { + "type": ["null", "string"] + }, + "destination": { + "type": ["null", "string"] + }, + "automatic": { + "type": ["null", "boolean"] + }, + "object": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "transfer_group": { + "type": ["null", "string"] + }, + "arrival_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "description": { + "type": ["null", "string"] + }, + "updated": { + "type": ["null", "string"], + "format": "date-time" + } + }, + "type": ["null", "object"] + }, + "metadata": [ + { + "breadcrumb": [], + "metadata": { + "table-key-properties": ["id"], + "forced-replication-method": "INCREMENTAL", + "valid-replication-keys": ["created"] + } + }, + { + "breadcrumb": ["properties", "metadata"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "reversals"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "id"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "statement_description"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "amount"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "balance_transaction"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "reversed"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "created"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "amount_reversed"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "source_type"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "source_transaction"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "date"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "livemode"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "statement_descriptor"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "failure_balance_transaction"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "recipient"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "destination"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "automatic"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "object"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "currency"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "transfer_group"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "arrival_date"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "description"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "updated"], + "metadata": { + "inclusion": "automatic" + } + } + ], + "key_properties": ["id"] + }, + { + "stream": "coupons", + "tap_stream_id": "coupons", + "schema": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "times_redeemed": { + "type": ["null", "integer"] + }, + "percent_off_precise": { + "type": ["null", "number"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "object": { + "type": ["null", "string"] + }, + "redeem_by": { + "type": ["null", "string"], + "format": "date-time" + }, + "duration": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "valid": { + "type": ["null", "boolean"] + }, + "currency": { + "type": ["null", "string"] + }, + "duration_in_months": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "max_redemptions": { + "type": ["null", "integer"] + }, + "amount_off": { + "type": ["null", "integer"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "percent_off": { + "type": ["null", "number"] + }, + "updated": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "metadata": [ + { + "breadcrumb": [], + "metadata": { + "table-key-properties": ["id"], + "forced-replication-method": "INCREMENTAL", + "valid-replication-keys": ["created"] + } + }, + { + "breadcrumb": ["properties", "metadata"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "times_redeemed"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "percent_off_precise"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "livemode"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "object"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "redeem_by"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "duration"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "id"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "valid"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "currency"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "duration_in_months"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "name"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "max_redemptions"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "amount_off"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "created"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "percent_off"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "updated"], + "metadata": { + "inclusion": "automatic" + } + } + ], + "key_properties": ["id"] + }, + { + "stream": "subscriptions", + "tap_stream_id": "subscriptions", + "schema": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "canceled_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "livemode": { + "type": ["null", "boolean"] + }, + "start": { + "type": ["null", "string"], + "format": "date-time" + }, + "items": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "id": { + "type": ["null", "string"] + }, + "trial_start": { + "type": ["null", "string"], + "format": "date-time" + }, + "application_fee_percent": { + "type": ["null", "number"] + }, + "billing_cycle_anchor": { + "type": ["null", "string"], + "format": "date-time" + }, + "cancel_at_period_end": { + "type": ["null", "boolean"] + }, + "tax_percent": { + "type": ["null", "number"] + }, + "discount": { + "type": ["null", "object"], + "properties": { + "end": { + "type": ["null", "string"], + "format": "date-time" + }, + "coupon": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "valid": { + "type": ["null", "boolean"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "amount_off": { + "type": ["null", "integer"] + }, + "redeem_by": { + "type": ["null", "string"], + "format": "date-time" + }, + "duration_in_months": { + "type": ["null", "integer"] + }, + "percent_off_precise": { + "type": ["null", "number"] + }, + "max_redemptions": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "times_redeemed": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, + "duration": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "percent_off": { + "type": ["null", "integer"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "customer": { + "type": ["null", "string"] + }, + "start": { + "type": ["null", "string"], + "format": "date-time" + }, + "object": { + "type": ["null", "string"] + }, + "subscription": { + "type": ["null", "string"] + } + } + }, + "current_period_end": { + "type": ["null", "string"], + "format": "date-time" + }, + "plan": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "product": { + "type": ["null", "string"] + }, + "statement_description": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "tiers_mode": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "string"] + }, + "tiers": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer", "object"], + "properties": { + "flat_amount": { + "type": ["null", "integer"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "up_to": { + "type": ["null", "integer"] + } + } + } + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "nickname": { + "type": ["null", "string"] + }, + "transform_usage": { + "type": ["null", "string"] + }, + "interval_count": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "interval": { + "type": ["null", "string"] + }, + "aggregate_usage": { + "type": ["null", "string"] + }, + "trial_period_days": { + "type": ["null", "integer"] + }, + "billing_scheme": { + "type": ["null", "string"] + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "usage_type": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + } + } + }, + "billing": { + "type": ["null", "string"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "days_until_due": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "ended_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "customer": { + "type": ["null", "string"] + }, + "current_period_start": { + "type": ["null", "string"], + "format": "date-time" + }, + "trial_end": { + "type": ["null", "string"], + "format": "date-time" + }, + "object": { + "type": ["null", "string"] + }, + "updated": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "metadata": [ + { + "breadcrumb": [], + "metadata": { + "table-key-properties": ["id"], + "forced-replication-method": "INCREMENTAL", + "valid-replication-keys": ["created"] + } + }, + { + "breadcrumb": ["properties", "metadata"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "canceled_at"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "livemode"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "start"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "items"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "id"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "trial_start"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "application_fee_percent"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "billing_cycle_anchor"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "cancel_at_period_end"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "tax_percent"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "discount"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "current_period_end"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "plan"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "billing"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "quantity"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "days_until_due"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "status"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "created"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "ended_at"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "customer"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "current_period_start"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "trial_end"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "object"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "updated"], + "metadata": { + "inclusion": "automatic" + } + } + ], + "key_properties": ["id"] + }, + { + "stream": "subscription_items", + "tap_stream_id": "subscription_items", + "schema": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "canceled_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "current_period_end": { + "type": ["null", "string"], + "format": "date-time" + }, + "plan": { + "type": ["null", "object", "string"], + "properties": { + "nickname": { + "type": ["null", "string"] + }, + "tiers": { + "type": ["null", "array"], + "items": { + "type": ["null", "string", "object"], + "properties": { + "flat_amount": { + "type": ["null", "integer"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "up_to": { + "type": ["null", "integer"] + } + } + } + }, + "object": { + "type": ["null", "string"] + }, + "aggregate_usage": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "statement_description": { + "type": ["null", "string"] + }, + "product": { + "type": ["null", "string"] + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "interval_count": { + "type": ["null", "integer"] + }, + "transform_usage": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "interval": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "trial_period_days": { + "type": ["null", "integer"] + }, + "usage_type": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "tiers_mode": { + "type": ["null", "string"] + }, + "billing_scheme": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "currency": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "updated": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "subscription": { + "type": ["null", "string"] + }, + "trial_start": { + "type": ["null", "string"], + "format": "date-time" + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "cancel_at_period_end": { + "type": ["null", "boolean"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "tax_percent": { + "type": ["null", "number"] + }, + "current_period_start": { + "type": ["null", "string"], + "format": "date-time" + }, + "start": { + "type": ["null", "string"], + "format": "date-time" + }, + "discount": { + "type": ["null", "object"], + "properties": {} + }, + "application_fee_percent": { + "type": ["null", "number"] + }, + "id": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "customer": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "ended_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "trial_end": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "metadata": [ + { + "breadcrumb": [], + "metadata": { + "table-key-properties": ["id"], + "forced-replication-method": "INCREMENTAL", + "valid-replication-keys": ["created"] + } + }, + { + "breadcrumb": ["properties", "metadata"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "canceled_at"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "current_period_end"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "plan"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "subscription"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "trial_start"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "created"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "cancel_at_period_end"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "quantity"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "tax_percent"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "current_period_start"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "start"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "discount"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "application_fee_percent"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "id"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "status"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "customer"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "object"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "livemode"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "ended_at"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "trial_end"], + "metadata": { + "inclusion": "available" + } + } + ], + "key_properties": ["id"] + }, + { + "stream": "balance_transactions", + "tap_stream_id": "balance_transactions", + "schema": { + "properties": { + "fee": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "string"] + }, + "fee_details": { + "type": ["null", "array"], + "items": { + "properties": { + "application": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "available_on": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "net": { + "type": ["null", "integer"] + }, + "exchange_rate": { + "type": ["null", "number"] + }, + "type": { + "type": ["null", "string"] + }, + "sourced_transfers": { + "items": {}, + "type": ["null", "array"] + }, + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "amount": { + "type": ["null", "integer"] + }, + "updated": { + "type": ["null", "string"], + "format": "date-time" + } + }, + "type": ["null", "object"] + }, + "metadata": [ + { + "breadcrumb": [], + "metadata": { + "table-key-properties": ["id"], + "forced-replication-method": "INCREMENTAL", + "valid-replication-keys": ["created"] + } + }, + { + "breadcrumb": ["properties", "fee"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "currency"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "source"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "fee_details"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "available_on"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "status"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "description"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "net"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "exchange_rate"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "type"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "sourced_transfers"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "id"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "object"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "created"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "amount"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "updated"], + "metadata": { + "inclusion": "automatic" + } + } + ], + "key_properties": ["id"] + }, + { + "stream": "payouts", + "tap_stream_id": "payouts", + "schema": { + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "failure_code": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "statement_description": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "balance_transaction": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "amount_reversed": { + "type": ["null", "integer"] + }, + "source_type": { + "type": ["null", "string"] + }, + "bank_account": { + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "routing_number": { + "type": ["null", "string"] + }, + "account_holder_type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "bank_name": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "account_holder_name": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "date": { + "type": ["null", "string"], + "format": "date-time" + }, + "method": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "failure_message": { + "type": ["null", "string"] + }, + "failure_balance_transaction": { + "type": ["null", "string"] + }, + "recipient": { + "type": ["null", "string"] + }, + "destination": { + "type": ["null", "string"] + }, + "automatic": { + "type": ["null", "boolean"] + }, + "object": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "transfer_group": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "arrival_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "description": { + "type": ["null", "string"] + }, + "source_transaction": { + "type": ["null", "string"] + }, + "updated": { + "type": ["null", "string"], + "format": "date-time" + } + }, + "type": ["null", "object"] + }, + "metadata": [ + { + "breadcrumb": [], + "metadata": { + "table-key-properties": ["id"], + "forced-replication-method": "INCREMENTAL", + "valid-replication-keys": ["created"] + } + }, + { + "breadcrumb": ["properties", "metadata"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "failure_code"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "id"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "statement_description"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "amount"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "balance_transaction"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "created"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "amount_reversed"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "source_type"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "bank_account"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "date"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "method"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "livemode"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "statement_descriptor"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "failure_message"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "failure_balance_transaction"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "recipient"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "destination"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "automatic"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "object"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "status"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "currency"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "transfer_group"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "type"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "arrival_date"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "description"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "source_transaction"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "updated"], + "metadata": { + "inclusion": "automatic" + } + } + ], + "key_properties": ["id"] + }, + { + "stream": "payout_transactions", + "tap_stream_id": "payout_transactions", + "schema": { + "properties": { + "payout_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "metadata": [ + { + "breadcrumb": [], + "metadata": { + "table-key-properties": ["id"], + "forced-replication-method": "INCREMENTAL", + "valid-replication-keys": ["id"] + } + }, + { + "breadcrumb": ["properties", "payout_id"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "id"], + "metadata": { + "inclusion": "automatic" + } + } + ], + "key_properties": ["id"] + }, + { + "stream": "disputes", + "tap_stream_id": "disputes", + "schema": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["string"] + }, + "object": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "balance_transactions": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["string"] + } + } + } + }, + "charge": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "currency": { + "type": ["null", "string"] + }, + "evidence": { + "type": ["null", "string", "object"], + "properties": { + "refund_policy": { + "type": ["null", "string"] + }, + "shipping_address": { + "type": ["null", "string"] + }, + "duplicate_charge_explanation": { + "type": ["null", "string"] + }, + "shipping_tracking_number": { + "type": ["null", "string"] + }, + "customer_signature": { + "type": ["null", "string"] + }, + "uncategorized_text": { + "type": ["null", "string"] + }, + "cancellation_policy_disclosure": { + "type": ["null", "string"] + }, + "refund_policy_disclosure": { + "type": ["null", "string"] + }, + "receipt": { + "type": ["null", "string"] + }, + "customer_name": { + "type": ["null", "string"] + }, + "refund_refusal_explanation": { + "type": ["null", "string"] + }, + "cancellation_rebuttal": { + "type": ["null", "string"] + }, + "product_description": { + "type": ["null", "string"] + }, + "shipping_date": { + "type": ["null", "string"] + }, + "customer_email_address": { + "type": ["null", "string"] + }, + "duplicate_charge_id": { + "type": ["null", "string"] + }, + "shipping_documentation": { + "type": ["null", "string"] + }, + "access_activity_log": { + "type": ["null", "string"] + }, + "customer_purchase_ip": { + "type": ["null", "string"] + }, + "service_date": { + "type": ["null", "string"] + }, + "shipping_carrier": { + "type": ["null", "string"] + }, + "service_documentation": { + "type": ["null", "string"] + }, + "duplicate_charge_documentation": { + "type": ["null", "string"] + }, + "cancellation_policy": { + "type": ["null", "string"] + }, + "customer_communication": { + "type": ["null", "string"] + }, + "uncategorized_file": { + "type": ["null", "string"] + }, + "billing_address": { + "type": ["null", "string"] + } + } + }, + "evidence_details": { + "type": ["null", "object"], + "properties": { + "due_by": { + "type": ["null", "string"], + "format": "date-time" + }, + "has_evidence": { + "type": ["null", "boolean"] + }, + "past_due": { + "type": ["null", "boolean"] + }, + "submission_count": { + "type": ["null", "integer"] + } + } + }, + "is_charge_refundable": { + "type": ["null", "boolean"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "reason": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "updated": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "metadata": [ + { + "breadcrumb": [], + "metadata": { + "table-key-properties": ["id"], + "forced-replication-method": "INCREMENTAL", + "valid-replication-keys": ["created"] + } + }, + { + "breadcrumb": ["properties", "id"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "object"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "amount"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "balance_transactions"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "charge"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "created"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "currency"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "evidence"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "evidence_details"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "is_charge_refundable"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "livemode"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "metadata"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "reason"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "status"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "updated"], + "metadata": { + "inclusion": "automatic" + } + } + ], + "key_properties": ["id"] + }, + { + "stream": "products", + "tap_stream_id": "products", + "schema": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "attributes": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "caption": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "deactivate_on": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "description": { + "type": ["null", "string"] + }, + "images": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "livemode": { + "type": ["null", "boolean"] + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "name": { + "type": ["null", "string"] + }, + "package_dimensions": { + "type": ["null", "object"], + "properties": { + "width": { + "type": ["null", "number"] + }, + "length": { + "type": ["null", "number"] + }, + "weight": { + "type": ["null", "number"] + }, + "height": { + "type": ["null", "number"] + } + } + }, + "shippable": { + "type": ["null", "boolean"] + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "unit_label": { + "type": ["null", "string"] + }, + "updated": { + "type": ["null", "string"], + "format": "date-time" + }, + "url": { + "type": ["null", "string"] + } + } + }, + "metadata": [ + { + "breadcrumb": [], + "metadata": { + "table-key-properties": ["id"], + "forced-replication-method": "INCREMENTAL", + "valid-replication-keys": ["created"] + } + }, + { + "breadcrumb": ["properties", "id"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "object"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "active"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "attributes"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "caption"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "created"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "deactivate_on"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "description"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "images"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "livemode"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "metadata"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "name"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "package_dimensions"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "shippable"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "statement_descriptor"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "type"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "unit_label"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "updated"], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": ["properties", "url"], + "metadata": { + "inclusion": "available" + } + } + ], + "key_properties": ["id"] + } + ] +} diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/stripe_schema_message.json b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/stripe_schema_message.json new file mode 100644 index 0000000000000..2fc0cc3de9889 --- /dev/null +++ b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/stripe_schema_message.json @@ -0,0 +1,862 @@ +{ + "type": "SCHEMA", + "stream": "customers", + "schema": { + "properties": { + "metadata": { + "properties": {}, + "type": ["null", "object"] + }, + "shipping": { + "properties": { + "address": { + "properties": { + "line2": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "sources": { + "anyOf": [ + { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "type": { + "type": ["null", "string"] + }, + "address_zip": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "card": { + "type": ["null", "object"], + "properties": { + "fingerprint": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "three_d_secure": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "brand": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "address_country": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "last4": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "brand": { + "type": ["null", "string"] + }, + "address_line2": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "usage": { + "type": ["null", "string"] + }, + "address_line1": { + "type": ["null", "string"] + }, + "owner": { + "type": ["null", "object"], + "properties": { + "verified_address": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "object"], + "properties": { + "line2": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + } + } + }, + "verified_email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "verified_name": { + "type": ["null", "string"] + }, + "verified_phone": { + "type": ["null", "string"] + } + } + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "client_secret": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "address_city": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "receiver": { + "type": ["null", "object"], + "properties": { + "refund_attributes_method": { + "type": ["null", "string"] + }, + "amount_returned": { + "type": ["null", "integer"] + }, + "amount_received": { + "type": ["null", "integer"] + }, + "refund_attributes_status": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "amount_charged": { + "type": ["null", "integer"] + } + } + }, + "flow": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "ach_credit_transfer": { + "type": ["null", "object"], + "properties": { + "bank_name": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "routing_number": { + "type": ["null", "string"] + }, + "swift_code": { + "type": ["null", "string"] + }, + "refund_account_holder_type": { + "type": ["null", "string"] + }, + "refund_account_holder_name": { + "type": ["null", "string"] + }, + "refund_account_number": { + "type": ["null", "string"] + }, + "refund_routing_number": { + "type": ["null", "string"] + }, + "account_number": { + "type": ["null", "string"] + } + } + }, + "customer": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "address_state": { + "type": ["null", "string"] + }, + "alipay": { + "type": ["null", "object"], + "properties": {} + }, + "bancontact": { + "type": ["null", "object"], + "properties": {} + }, + "eps": { + "type": ["null", "object"], + "properties": {} + }, + "ideal": { + "type": ["null", "object"], + "properties": {} + }, + "multibanco": { + "type": ["null", "object"], + "properties": {} + }, + "redirect": { + "type": ["null", "object"], + "properties": { + "failure_reason": { + "type": ["null", "string"] + }, + "return_url": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + } + } + } + }, + { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "type": { + "type": ["null", "string"] + }, + "address_zip": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "card": { + "type": ["null", "object"], + "properties": { + "fingerprint": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "three_d_secure": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "brand": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "address_country": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "last4": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "brand": { + "type": ["null", "string"] + }, + "address_line2": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "usage": { + "type": ["null", "string"] + }, + "address_line1": { + "type": ["null", "string"] + }, + "owner": { + "type": ["null", "object"], + "properties": { + "verified_address": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "object"], + "properties": { + "line2": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + } + } + }, + "verified_email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "verified_name": { + "type": ["null", "string"] + }, + "verified_phone": { + "type": ["null", "string"] + } + } + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "client_secret": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "address_city": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "receiver": { + "type": ["null", "object"], + "properties": { + "refund_attributes_method": { + "type": ["null", "string"] + }, + "amount_returned": { + "type": ["null", "integer"] + }, + "amount_received": { + "type": ["null", "integer"] + }, + "refund_attributes_status": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "amount_charged": { + "type": ["null", "integer"] + } + } + }, + "flow": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "ach_credit_transfer": { + "type": ["null", "object"], + "properties": { + "bank_name": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "routing_number": { + "type": ["null", "string"] + }, + "swift_code": { + "type": ["null", "string"] + }, + "refund_account_holder_type": { + "type": ["null", "string"] + }, + "refund_account_holder_name": { + "type": ["null", "string"] + }, + "refund_account_number": { + "type": ["null", "string"] + }, + "refund_routing_number": { + "type": ["null", "string"] + }, + "account_number": { + "type": ["null", "string"] + } + } + }, + "customer": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "address_state": { + "type": ["null", "string"] + }, + "alipay": { + "type": ["null", "object"], + "properties": {} + }, + "bancontact": { + "type": ["null", "object"], + "properties": {} + }, + "eps": { + "type": ["null", "object"], + "properties": {} + }, + "ideal": { + "type": ["null", "object"], + "properties": {} + }, + "multibanco": { + "type": ["null", "object"], + "properties": {} + }, + "redirect": { + "type": ["null", "object"], + "properties": { + "failure_reason": { + "type": ["null", "string"] + }, + "return_url": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + } + } + } + ] + }, + "delinquent": { + "type": ["null", "boolean"] + }, + "description": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "default_source": { + "type": ["null", "string"] + }, + "cards": { + "items": { + "properties": { + "metadata": { + "properties": {}, + "type": ["null", "object"] + }, + "object": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "last4": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "brand": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "customer": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "address_line2": { + "type": ["null", "string"] + }, + "address_line1": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "address_zip": { + "type": ["null", "string"] + }, + "address_city": { + "type": ["null", "string"] + }, + "address_country": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address_state": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "email": { + "type": ["null", "string"] + }, + "default_card": { + "type": ["null", "string"] + }, + "subscriptions": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "discount": { + "properties": { + "end": { + "format": "date-time", + "type": ["null", "string"] + }, + "coupon": { + "properties": { + "metadata": { + "properties": {}, + "type": ["null", "object"] + }, + "valid": { + "type": ["null", "boolean"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "amount_off": { + "type": ["null", "integer"] + }, + "redeem_by": { + "format": "date-time", + "type": ["null", "string"] + }, + "duration_in_months": { + "type": ["null", "integer"] + }, + "percent_off_precise": { + "type": ["null", "number"] + }, + "max_redemptions": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "times_redeemed": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, + "duration": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "percent_off": { + "type": ["null", "integer"] + }, + "created": { + "format": "date-time", + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "customer": { + "type": ["null", "string"] + }, + "start": { + "format": "date-time", + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "subscription": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "account_balance": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "invoice_prefix": { + "type": ["null", "string"] + }, + "tax_info_verification": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "created": { + "format": "date-time", + "type": ["null", "string"] + }, + "tax_info": { + "type": ["null", "string"] + }, + "updated": { + "format": "date-time", + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "key_properties": ["id"] +} + diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/sync_output_subset.txt b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/sync_output_subset.txt new file mode 100644 index 0000000000000..fac32348c4450 --- /dev/null +++ b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/sync_output_subset.txt @@ -0,0 +1,4 @@ +{"type": "SCHEMA", "stream": "customers", "schema": {"properties": {"metadata": {"properties": {}, "type": ["null", "object"]}, "shipping": {"properties": {"address": {"properties": {"line2": {"type": ["null", "string"]}, "state": {"type": ["null", "string"]}, "city": {"type": ["null", "string"]}, "postal_code": {"type": ["null", "string"]}, "country": {"type": ["null", "string"]}, "line1": {"type": ["null", "string"]}}, "type": ["null", "object"]}, "name": {"type": ["null", "string"]}, "phone": {"type": ["null", "string"]}}, "type": ["null", "object"]}, "sources": {"anyOf": [{"type": ["null", "array"], "items": {"type": ["null", "object"], "properties": {"metadata": {"type": ["null", "object"], "properties": {}}, "type": {"type": ["null", "string"]}, "address_zip": {"type": ["null", "string"]}, "livemode": {"type": ["null", "boolean"]}, "card": {"type": ["null", "object"], "properties": {"fingerprint": {"type": ["null", "string"]}, "last4": {"type": ["null", "string"]}, "dynamic_last4": {"type": ["null", "string"]}, "address_line1_check": {"type": ["null", "string"]}, "exp_month": {"type": ["null", "integer"]}, "tokenization_method": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "exp_year": {"type": ["null", "integer"]}, "three_d_secure": {"type": ["null", "string"]}, "funding": {"type": ["null", "string"]}, "brand": {"type": ["null", "string"]}, "cvc_check": {"type": ["null", "string"]}, "country": {"type": ["null", "string"]}, "address_zip_check": {"type": ["null", "string"]}, "type": {"type": ["null", "string"]}}}, "statement_descriptor": {"type": ["null", "string"]}, "id": {"type": ["null", "string"]}, "address_country": {"type": ["null", "string"]}, "funding": {"type": ["null", "string"]}, "dynamic_last4": {"type": ["null", "string"]}, "exp_year": {"type": ["null", "integer"]}, "last4": {"type": ["null", "string"]}, "exp_month": {"type": ["null", "integer"]}, "brand": {"type": ["null", "string"]}, "address_line2": {"type": ["null", "string"]}, "country": {"type": ["null", "string"]}, "object": {"type": ["null", "string"]}, "amount": {"type": ["null", "integer"]}, "cvc_check": {"type": ["null", "string"]}, "usage": {"type": ["null", "string"]}, "address_line1": {"type": ["null", "string"]}, "owner": {"type": ["null", "object"], "properties": {"verified_address": {"type": ["null", "string"]}, "email": {"type": ["null", "string"]}, "address": {"type": ["null", "object"], "properties": {"line2": {"type": ["null", "string"]}, "state": {"type": ["null", "string"]}, "city": {"type": ["null", "string"]}, "postal_code": {"type": ["null", "string"]}, "country": {"type": ["null", "string"]}, "line1": {"type": ["null", "string"]}}}, "verified_email": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "phone": {"type": ["null", "string"]}, "verified_name": {"type": ["null", "string"]}, "verified_phone": {"type": ["null", "string"]}}}, "tokenization_method": {"type": ["null", "string"]}, "client_secret": {"type": ["null", "string"]}, "fingerprint": {"type": ["null", "string"]}, "address_city": {"type": ["null", "string"]}, "currency": {"type": ["null", "string"]}, "address_line1_check": {"type": ["null", "string"]}, "receiver": {"type": ["null", "object"], "properties": {"refund_attributes_method": {"type": ["null", "string"]}, "amount_returned": {"type": ["null", "integer"]}, "amount_received": {"type": ["null", "integer"]}, "refund_attributes_status": {"type": ["null", "string"]}, "address": {"type": ["null", "string"]}, "amount_charged": {"type": ["null", "integer"]}}}, "flow": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "ach_credit_transfer": {"type": ["null", "object"], "properties": {"bank_name": {"type": ["null", "string"]}, "fingerprint": {"type": ["null", "string"]}, "routing_number": {"type": ["null", "string"]}, "swift_code": {"type": ["null", "string"]}, "refund_account_holder_type": {"type": ["null", "string"]}, "refund_account_holder_name": {"type": ["null", "string"]}, "refund_account_number": {"type": ["null", "string"]}, "refund_routing_number": {"type": ["null", "string"]}, "account_number": {"type": ["null", "string"]}}}, "customer": {"type": ["null", "string"]}, "address_zip_check": {"type": ["null", "string"]}, "status": {"type": ["null", "string"]}, "created": {"type": ["null", "string"], "format": "date-time"}, "address_state": {"type": ["null", "string"]}, "alipay": {"type": ["null", "object"], "properties": {}}, "bancontact": {"type": ["null", "object"], "properties": {}}, "eps": {"type": ["null", "object"], "properties": {}}, "ideal": {"type": ["null", "object"], "properties": {}}, "multibanco": {"type": ["null", "object"], "properties": {}}, "redirect": {"type": ["null", "object"], "properties": {"failure_reason": {"type": ["null", "string"]}, "return_url": {"type": ["null", "string"]}, "status": {"type": ["null", "string"]}, "url": {"type": ["null", "string"]}}}}}}, {"type": ["null", "object"], "properties": {"metadata": {"type": ["null", "object"], "properties": {}}, "type": {"type": ["null", "string"]}, "address_zip": {"type": ["null", "string"]}, "livemode": {"type": ["null", "boolean"]}, "card": {"type": ["null", "object"], "properties": {"fingerprint": {"type": ["null", "string"]}, "last4": {"type": ["null", "string"]}, "dynamic_last4": {"type": ["null", "string"]}, "address_line1_check": {"type": ["null", "string"]}, "exp_month": {"type": ["null", "integer"]}, "tokenization_method": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "exp_year": {"type": ["null", "integer"]}, "three_d_secure": {"type": ["null", "string"]}, "funding": {"type": ["null", "string"]}, "brand": {"type": ["null", "string"]}, "cvc_check": {"type": ["null", "string"]}, "country": {"type": ["null", "string"]}, "address_zip_check": {"type": ["null", "string"]}, "type": {"type": ["null", "string"]}}}, "statement_descriptor": {"type": ["null", "string"]}, "id": {"type": ["null", "string"]}, "address_country": {"type": ["null", "string"]}, "funding": {"type": ["null", "string"]}, "dynamic_last4": {"type": ["null", "string"]}, "exp_year": {"type": ["null", "integer"]}, "last4": {"type": ["null", "string"]}, "exp_month": {"type": ["null", "integer"]}, "brand": {"type": ["null", "string"]}, "address_line2": {"type": ["null", "string"]}, "country": {"type": ["null", "string"]}, "object": {"type": ["null", "string"]}, "amount": {"type": ["null", "integer"]}, "cvc_check": {"type": ["null", "string"]}, "usage": {"type": ["null", "string"]}, "address_line1": {"type": ["null", "string"]}, "owner": {"type": ["null", "object"], "properties": {"verified_address": {"type": ["null", "string"]}, "email": {"type": ["null", "string"]}, "address": {"type": ["null", "object"], "properties": {"line2": {"type": ["null", "string"]}, "state": {"type": ["null", "string"]}, "city": {"type": ["null", "string"]}, "postal_code": {"type": ["null", "string"]}, "country": {"type": ["null", "string"]}, "line1": {"type": ["null", "string"]}}}, "verified_email": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "phone": {"type": ["null", "string"]}, "verified_name": {"type": ["null", "string"]}, "verified_phone": {"type": ["null", "string"]}}}, "tokenization_method": {"type": ["null", "string"]}, "client_secret": {"type": ["null", "string"]}, "fingerprint": {"type": ["null", "string"]}, "address_city": {"type": ["null", "string"]}, "currency": {"type": ["null", "string"]}, "address_line1_check": {"type": ["null", "string"]}, "receiver": {"type": ["null", "object"], "properties": {"refund_attributes_method": {"type": ["null", "string"]}, "amount_returned": {"type": ["null", "integer"]}, "amount_received": {"type": ["null", "integer"]}, "refund_attributes_status": {"type": ["null", "string"]}, "address": {"type": ["null", "string"]}, "amount_charged": {"type": ["null", "integer"]}}}, "flow": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "ach_credit_transfer": {"type": ["null", "object"], "properties": {"bank_name": {"type": ["null", "string"]}, "fingerprint": {"type": ["null", "string"]}, "routing_number": {"type": ["null", "string"]}, "swift_code": {"type": ["null", "string"]}, "refund_account_holder_type": {"type": ["null", "string"]}, "refund_account_holder_name": {"type": ["null", "string"]}, "refund_account_number": {"type": ["null", "string"]}, "refund_routing_number": {"type": ["null", "string"]}, "account_number": {"type": ["null", "string"]}}}, "customer": {"type": ["null", "string"]}, "address_zip_check": {"type": ["null", "string"]}, "status": {"type": ["null", "string"]}, "created": {"type": ["null", "string"], "format": "date-time"}, "address_state": {"type": ["null", "string"]}, "alipay": {"type": ["null", "object"], "properties": {}}, "bancontact": {"type": ["null", "object"], "properties": {}}, "eps": {"type": ["null", "object"], "properties": {}}, "ideal": {"type": ["null", "object"], "properties": {}}, "multibanco": {"type": ["null", "object"], "properties": {}}, "redirect": {"type": ["null", "object"], "properties": {"failure_reason": {"type": ["null", "string"]}, "return_url": {"type": ["null", "string"]}, "status": {"type": ["null", "string"]}, "url": {"type": ["null", "string"]}}}}}]}, "delinquent": {"type": ["null", "boolean"]}, "description": {"type": ["null", "string"]}, "livemode": {"type": ["null", "boolean"]}, "default_source": {"type": ["null", "string"]}, "cards": {"items": {"properties": {"metadata": {"properties": {}, "type": ["null", "object"]}, "object": {"type": ["null", "string"]}, "id": {"type": ["null", "string"]}, "exp_month": {"type": ["null", "integer"]}, "dynamic_last4": {"type": ["null", "string"]}, "exp_year": {"type": ["null", "integer"]}, "last4": {"type": ["null", "string"]}, "funding": {"type": ["null", "string"]}, "brand": {"type": ["null", "string"]}, "country": {"type": ["null", "string"]}, "customer": {"type": ["null", "string"]}, "cvc_check": {"type": ["null", "string"]}, "address_line2": {"type": ["null", "string"]}, "address_line1": {"type": ["null", "string"]}, "fingerprint": {"type": ["null", "string"]}, "address_zip": {"type": ["null", "string"]}, "address_city": {"type": ["null", "string"]}, "address_country": {"type": ["null", "string"]}, "address_line1_check": {"type": ["null", "string"]}, "tokenization_method": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "address_state": {"type": ["null", "string"]}, "address_zip_check": {"type": ["null", "string"]}, "type": {"type": ["null", "string"]}}, "type": ["null", "object"]}, "type": ["null", "array"]}, "email": {"type": ["null", "string"]}, "default_card": {"type": ["null", "string"]}, "subscriptions": {"items": {"type": ["null", "string"]}, "type": ["null", "array"]}, "discount": {"properties": {"end": {"format": "date-time", "type": ["null", "string"]}, "coupon": {"properties": {"metadata": {"properties": {}, "type": ["null", "object"]}, "valid": {"type": ["null", "boolean"]}, "livemode": {"type": ["null", "boolean"]}, "amount_off": {"type": ["null", "integer"]}, "redeem_by": {"format": "date-time", "type": ["null", "string"]}, "duration_in_months": {"type": ["null", "integer"]}, "percent_off_precise": {"type": ["null", "number"]}, "max_redemptions": {"type": ["null", "integer"]}, "currency": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "times_redeemed": {"type": ["null", "integer"]}, "id": {"type": ["null", "string"]}, "duration": {"type": ["null", "string"]}, "object": {"type": ["null", "string"]}, "percent_off": {"type": ["null", "integer"]}, "created": {"format": "date-time", "type": ["null", "string"]}}, "type": ["null", "object"]}, "customer": {"type": ["null", "string"]}, "start": {"format": "date-time", "type": ["null", "string"]}, "object": {"type": ["null", "string"]}, "subscription": {"type": ["null", "string"]}}, "type": ["null", "object"]}, "account_balance": {"type": ["null", "integer"]}, "currency": {"type": ["null", "string"]}, "id": {"type": ["null", "string"]}, "invoice_prefix": {"type": ["null", "string"]}, "tax_info_verification": {"type": ["null", "string"]}, "object": {"type": ["null", "string"]}, "created": {"format": "date-time", "type": ["null", "string"]}, "tax_info": {"type": ["null", "string"]}, "updated": {"format": "date-time", "type": ["null", "string"]}}, "type": ["null", "object"]}, "key_properties": ["id"]}{"type": "RECORD", "stream": "customers", "record": {"id": "cus_I1l0AbdUZF4Wrl", "object": "customer", "account_balance": 0, "created": "2020-09-15T16:58:53.000000Z", "currency": null, "default_source": null, "delinquent": false, "description": "Customer 4", "discount": null, "email": "customer4@test.com", "invoice_prefix": "AFC537CE", "livemode": false, "metadata": {}, "shipping": null, "sources": [], "subscriptions": null, "tax_info": null, "tax_info_verification": null, "updated": "2020-09-15T16:58:53.000000Z"}, "time_extracted": "2020-09-15T18:01:23.634272Z"} +{"type": "RECORD", "stream": "customers", "record": {"id": "cus_I1l0XHrjLYwLR2", "object": "customer", "account_balance": 0, "created": "2020-09-15T16:58:52.000000Z", "currency": null, "default_source": null, "delinquent": false, "description": "Customer 3", "discount": null, "email": "customer3@test.com", "invoice_prefix": "EC156D8F", "livemode": false, "metadata": {}, "shipping": null, "sources": [], "subscriptions": null, "tax_info": null, "tax_info_verification": null, "updated": "2020-09-15T16:58:52.000000Z"}, "time_extracted": "2020-09-15T18:01:23.634272Z"} +{"type": "RECORD", "stream": "customers", "record": {"id": "cus_I1l0INzfeSf2MM", "object": "customer", "account_balance": 0, "created": "2020-09-15T16:58:52.000000Z", "currency": null, "default_source": null, "delinquent": false, "description": "Customer 2", "discount": null, "email": "customer2@test.com", "invoice_prefix": "D4564D22", "livemode": false, "metadata": {}, "shipping": null, "sources": [], "subscriptions": null, "tax_info": null, "tax_info_verification": null, "updated": "2020-09-15T16:58:52.000000Z"}, "time_extracted": "2020-09-15T18:01:23.634272Z"} +{"type": "RECORD", "stream": "customers", "record": {"id": "cus_I1l0cRVFy4ZhwQ", "object": "customer", "account_balance": 0, "created": "2020-09-15T16:58:52.000000Z", "currency": null, "default_source": null, "delinquent": false, "description": "Customer 1", "discount": null, "email": "customer1@test.com", "invoice_prefix": "92A8C396", "livemode": false, "metadata": {}, "shipping": null, "sources": [], "subscriptions": null, "tax_info": null, "tax_info_verification": null, "updated": "2020-09-15T16:58:52.000000Z"}, "time_extracted": "2020-09-15T18:01:23.634272Z"} From f9268381e644b0567e7f355c2d86bd36d7ceb964 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 13 Oct 2020 08:51:04 -0700 Subject: [PATCH 09/31] fmt --- airbyte-integrations/singer/stripe_abprotocol/source/.gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/.gitignore b/airbyte-integrations/singer/stripe_abprotocol/source/.gitignore index f3fb4d3ca286b..dd04e98646d75 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/.gitignore +++ b/airbyte-integrations/singer/stripe_abprotocol/source/.gitignore @@ -1,2 +1 @@ config/* - From 4d5c0c9b372306350ecf2ca38502501f1ccf7ad5 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 13 Oct 2020 08:51:15 -0700 Subject: [PATCH 10/31] fix invalid credential test --- .../sources/SingerStripeSourceTest.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java index 037e8da645da0..ca8e523042f60 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java +++ b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java @@ -38,6 +38,10 @@ import io.airbyte.workers.process.AirbyteIntegrationLauncher; import io.airbyte.workers.process.DockerProcessBuilderFactory; import io.airbyte.workers.process.IntegrationLauncher; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; @@ -49,11 +53,9 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import org.apache.commons.lang3.RandomStringUtils; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; public class SingerStripeSourceTest { @@ -138,7 +140,7 @@ public void testInvalidCredentialsDiscover() throws IOException, InterruptedExce Process process = createDiscoveryProcess(INVALID_CONFIG); process.waitFor(); - assertEquals(2, process.exitValue()); + assertNotEquals(0, process.exitValue()); } @Test From 84c7db99b1541395e558a6e2b112cd4a6f3bfb01 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 13 Oct 2020 08:59:26 -0700 Subject: [PATCH 11/31] restructure catalog helpers --- .../singer/base-singer/base_singer/singer_helpers.py | 2 +- .../source_exchangeratesapi_singer.py | 8 ++------ .../source/source_stripe_singer/source_stripe_singer.py | 4 +++- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py index 64b3e52cb4627..6d2cfd6d7deee 100644 --- a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py +++ b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py @@ -32,7 +32,7 @@ def spec_from_file(spec_path) -> AirbyteSpec: return AirbyteSpec(spec_text) @staticmethod - def discover(logger, shell_command, singer_transform=(lambda catalog: catalog), airbyte_transform=(lambda catalog: catalog)) -> Catalogs: + def get_catalogs(logger, shell_command, singer_transform=(lambda catalog: catalog), airbyte_transform=(lambda catalog: catalog)) -> Catalogs: completed_process = subprocess.run(shell_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) diff --git a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py index e955bdfa36fee..d285caed44fe8 100644 --- a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py +++ b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py @@ -8,11 +8,6 @@ from base_singer import SingerHelper from base_singer import Catalogs - -def get_catalogs(logger) -> Catalogs: - return SingerHelper.discover(logger, "tap-exchangeratesapi | grep '\"type\": \"SCHEMA\"' | head -1 | jq -c '{\"streams\":[{\"stream\": .stream, \"schema\": .schema}]}'") - - class SourceExchangeRatesApiSinger(Source): def __init__(self): pass @@ -26,7 +21,8 @@ def check(self, logger, config_container) -> AirbyteCheckResponse: return AirbyteCheckResponse(code == 200, {}) def discover(self, logger, config_container) -> AirbyteCatalog: - return get_catalogs(logger).airbyte_catalog + catalogs = SingerHelper.get_catalogs(logger, "tap-exchangeratesapi | grep '\"type\": \"SCHEMA\"' | head -1 | jq -c '{\"streams\":[{\"stream\": .stream, \"schema\": .schema}]}'") + return catalogs.airbyte_catalog def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: if state: diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py index e43b8a1d41a36..69e5b8d61635e 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py +++ b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py @@ -22,7 +22,9 @@ def check(self, logger, config_container) -> AirbyteCheckResponse: return AirbyteCheckResponse(r.status_code == 200, {}) def discover(self, logger, config_container) -> AirbyteCatalog: - return SingerHelper.discover(f"tap-stripe --config {config_container.rendered_config_path} --discover") + catalogs = SingerHelper.get_catalogs(logger, f"tap-stripe --config {config_container.rendered_config_path} --discover") + print(catalogs) + return catalogs.airbyte_catalog def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: if state: From 09a442c8c550bf212d79014836af85121eeefcf3 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 13 Oct 2020 09:59:49 -0700 Subject: [PATCH 12/31] terrible hack to get stripe to not fail completely --- .../singer/base-singer/base_singer/singer_helpers.py | 5 +++++ .../source/source_stripe_singer/source_stripe_singer.py | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py index 6d2cfd6d7deee..160439f4270e9 100644 --- a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py +++ b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py @@ -45,6 +45,11 @@ def get_catalogs(logger, shell_command, singer_transform=(lambda catalog: catalo for stream in singer_catalog.get("streams"): name = stream.get("stream") schema = stream.get("schema").get("properties") + + # todo: figure out how to serialize an object with an items key in python_jsonschema_objects + if name == "subscriptions": + del schema["items"] + airbyte_streams += [AirbyteStream(name=name, schema=schema)] airbyte_catalog = airbyte_transform(AirbyteCatalog(streams=airbyte_streams)) diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py index 69e5b8d61635e..a9327808c724d 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py +++ b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py @@ -23,7 +23,6 @@ def check(self, logger, config_container) -> AirbyteCheckResponse: def discover(self, logger, config_container) -> AirbyteCatalog: catalogs = SingerHelper.get_catalogs(logger, f"tap-stripe --config {config_container.rendered_config_path} --discover") - print(catalogs) return catalogs.airbyte_catalog def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: From 4ad1db2249db5742ce6f7e9789bf023cde51d8bf Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 13 Oct 2020 10:03:03 -0700 Subject: [PATCH 13/31] fix discovery test now that a single message contains the catalog --- .../integration_tests/sources/SingerStripeSourceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java index ca8e523042f60..b3882310b9b44 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java +++ b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java @@ -152,7 +152,7 @@ public void testSuccessfulDiscover() throws IOException, InterruptedException, W final String catalog = IOs.readFile(jobRoot, catalogPath.toString()); - assertTrue(catalog.lines().count() > 10000); + assertTrue(catalog.length() > 20000); assertTrue(catalog.contains("customer")); assertTrue(catalog.contains("address_zip_check")); } From dda65c3ee59575dad31409bed5a7895ef8a2ac25 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 13 Oct 2020 10:08:51 -0700 Subject: [PATCH 14/31] all working tests except sync --- .../sources/SingerStripeSourceTest.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java index b3882310b9b44..4b100511efcd2 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java +++ b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java @@ -136,8 +136,16 @@ public void testGetSpec() throws WorkerException, IOException, InterruptedExcept } @Test - public void testInvalidCredentialsDiscover() throws IOException, InterruptedException, WorkerException { - Process process = createDiscoveryProcess(INVALID_CONFIG); + public void testSuccessfulCheck() throws IOException, InterruptedException, WorkerException { + Process process = createCheckProcess(CONFIG); + process.waitFor(); + + assertEquals(0, process.exitValue()); + } + + @Test + public void testInvalidCredentialsCheck() throws IOException, InterruptedException, WorkerException { + Process process = createCheckProcess(INVALID_CONFIG); process.waitFor(); assertNotEquals(0, process.exitValue()); @@ -225,6 +233,13 @@ private void writeInvalidConfigFile() throws IOException { Files.writeString(Path.of(jobRoot.toString(), INVALID_CONFIG), Jsons.serialize(fullConfig)); } + private Process createCheckProcess(String configFileName) throws IOException, WorkerException { + return launcher.check(jobRoot, configFileName) + .redirectOutput(catalogPath.toFile()) + .redirectError(ProcessBuilder.Redirect.INHERIT) + .start(); + } + private Process createDiscoveryProcess(String configFileName) throws IOException, WorkerException { return launcher.discover(jobRoot, configFileName) .redirectOutput(catalogPath.toFile()) From bbe34ad681e8e5dbafa5c1c95c04dc7e123653b8 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 13 Oct 2020 11:28:31 -0700 Subject: [PATCH 15/31] partially working mapping (in wrong location) --- .../source_stripe_singer.py | 51 ++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py index a9327808c724d..ddb2dbfa7c9d2 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py +++ b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py @@ -7,6 +7,15 @@ from typing import Generator from base_singer import SingerHelper import sys +import json + + +def is_field_metadata(metadata): + if len(metadata.get("breadcrumb")) != 2: + return False + else: + return metadata.get("breadcrumb")[0] != "property" + class SourceStripeSinger(Source): def __init__(self): @@ -26,7 +35,45 @@ def discover(self, logger, config_container) -> AirbyteCatalog: return catalogs.airbyte_catalog def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: + masked_airbyte_catalog = self.read_config(catalog_path) + discovered_singer_catalog = SingerHelper.get_catalogs(logger, f"tap-stripe --config {config_container.rendered_config_path} --discover").singer_catalog + + # todo: combine discovered singer catalog with the generated airbyte catalog + + combined_catalog_path = "/mount/rendered_catalog.json" + masked_singer_streams = [] + + stream_to_airbyte_schema = {} + for stream in masked_airbyte_catalog["streams"]: + stream_to_airbyte_schema[stream.get("name")] = stream + + for singer_stream in discovered_singer_catalog.get("streams"): + print("stream", singer_stream.get("stream")) + if singer_stream.get("stream") in stream_to_airbyte_schema: + new_metadatas = [] + metadatas = singer_stream.get("metadata") + for metadata in metadatas: + new_metadata = metadata + new_metadata["metadata"]["selected"] = True + if not is_field_metadata(new_metadata): + new_metadata["metadata"]["replication-method"] = "FULL_TABLE" + new_metadatas += [new_metadata] + singer_stream["metadata"] = new_metadatas + + masked_singer_streams += [singer_stream] + + combined_catalog = { "streams": masked_singer_streams} + with open(combined_catalog_path, 'w') as fh: + fh.write(json.dumps(combined_catalog)) + + # print("discovered_singer") + # print(json.dumps(discovered_singer_catalog)) + print("combined_catalog") + print(json.dumps(combined_catalog)) + + # todo: figure out how to make this easier to consume for new implementers + if state: - return SingerHelper.read(logger, f"tap-stripe --config {config_container.rendered_config_path} --state {state}") + return SingerHelper.read(logger, f"tap-stripe --config {config_container.rendered_config_path} --properties {combined_catalog_path} --state {state}") else: - return SingerHelper.read(logger, f"tap-stripe --config {config_container.rendered_config_path}") + return SingerHelper.read(logger, f"tap-stripe --config {config_container.rendered_config_path} --properties {combined_catalog_path}") From b4a6e927d287421453757847b48f9087cfb72acc Mon Sep 17 00:00:00 2001 From: jrhizor Date: Tue, 13 Oct 2020 15:55:38 -0700 Subject: [PATCH 16/31] working but untested reads --- .../source_stripe_singer/source_stripe_singer.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py index ddb2dbfa7c9d2..a1d2e1a5b1802 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py +++ b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py @@ -38,8 +38,6 @@ def read(self, logger, config_container, catalog_path, state=None) -> Generator[ masked_airbyte_catalog = self.read_config(catalog_path) discovered_singer_catalog = SingerHelper.get_catalogs(logger, f"tap-stripe --config {config_container.rendered_config_path} --discover").singer_catalog - # todo: combine discovered singer catalog with the generated airbyte catalog - combined_catalog_path = "/mount/rendered_catalog.json" masked_singer_streams = [] @@ -48,7 +46,6 @@ def read(self, logger, config_container, catalog_path, state=None) -> Generator[ stream_to_airbyte_schema[stream.get("name")] = stream for singer_stream in discovered_singer_catalog.get("streams"): - print("stream", singer_stream.get("stream")) if singer_stream.get("stream") in stream_to_airbyte_schema: new_metadatas = [] metadatas = singer_stream.get("metadata") @@ -56,24 +53,19 @@ def read(self, logger, config_container, catalog_path, state=None) -> Generator[ new_metadata = metadata new_metadata["metadata"]["selected"] = True if not is_field_metadata(new_metadata): - new_metadata["metadata"]["replication-method"] = "FULL_TABLE" + new_metadata["metadata"]["forced-replication-method"] = "FULL_TABLE" new_metadatas += [new_metadata] singer_stream["metadata"] = new_metadatas masked_singer_streams += [singer_stream] - combined_catalog = { "streams": masked_singer_streams} + combined_catalog = {"streams": masked_singer_streams} with open(combined_catalog_path, 'w') as fh: fh.write(json.dumps(combined_catalog)) - # print("discovered_singer") - # print(json.dumps(discovered_singer_catalog)) - print("combined_catalog") - print(json.dumps(combined_catalog)) - # todo: figure out how to make this easier to consume for new implementers if state: - return SingerHelper.read(logger, f"tap-stripe --config {config_container.rendered_config_path} --properties {combined_catalog_path} --state {state}") + return SingerHelper.read(logger, f"tap-stripe --config {config_container.rendered_config_path} --catalog {combined_catalog_path} --state {state}") else: - return SingerHelper.read(logger, f"tap-stripe --config {config_container.rendered_config_path} --properties {combined_catalog_path}") + return SingerHelper.read(logger, f"tap-stripe --config {config_container.rendered_config_path} --catalog {combined_catalog_path}") From 5fe3a1eced58a9de076e546fefc7c9e1e57b05be Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 14 Oct 2020 08:12:18 -0700 Subject: [PATCH 17/31] split logic out --- .../base-singer/base_singer/singer_helpers.py | 37 +++++++++++++++++++ .../source_stripe_singer.py | 37 +------------------ 2 files changed, 38 insertions(+), 36 deletions(-) diff --git a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py index 160439f4270e9..1c0a44d34050b 100644 --- a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py +++ b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py @@ -11,6 +11,7 @@ from datetime import datetime from dataclasses import dataclass + def to_json(string): try: return json.loads(string) @@ -18,6 +19,13 @@ def to_json(string): return False +def is_field_metadata(metadata): + if len(metadata.get("breadcrumb")) != 2: + return False + else: + return metadata.get("breadcrumb")[0] != "property" + + @dataclass class Catalogs: singer_catalog: object @@ -89,3 +97,32 @@ def read(logger, shell_command, is_message=(lambda x: True), transform=(lambda x if err_line: logger(err_line, "ERROR") + + @staticmethod + def combine_catalogs(masked_airbyte_catalog, discovered_singer_catalog) -> str: + combined_catalog_path = "/mount/rendered_catalog.json" + masked_singer_streams = [] + + stream_to_airbyte_schema = {} + for stream in masked_airbyte_catalog["streams"]: + stream_to_airbyte_schema[stream.get("name")] = stream + + for singer_stream in discovered_singer_catalog.get("streams"): + if singer_stream.get("stream") in stream_to_airbyte_schema: + new_metadatas = [] + metadatas = singer_stream.get("metadata") + for metadata in metadatas: + new_metadata = metadata + new_metadata["metadata"]["selected"] = True + if not is_field_metadata(new_metadata): + new_metadata["metadata"]["forced-replication-method"] = "FULL_TABLE" + new_metadatas += [new_metadata] + singer_stream["metadata"] = new_metadatas + + masked_singer_streams += [singer_stream] + + combined_catalog = {"streams": masked_singer_streams} + with open(combined_catalog_path, 'w') as fh: + fh.write(json.dumps(combined_catalog)) + + return combined_catalog_path diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py index a1d2e1a5b1802..a3d3d4e9ef922 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py +++ b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py @@ -6,15 +6,6 @@ import requests from typing import Generator from base_singer import SingerHelper -import sys -import json - - -def is_field_metadata(metadata): - if len(metadata.get("breadcrumb")) != 2: - return False - else: - return metadata.get("breadcrumb")[0] != "property" class SourceStripeSinger(Source): @@ -37,33 +28,7 @@ def discover(self, logger, config_container) -> AirbyteCatalog: def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: masked_airbyte_catalog = self.read_config(catalog_path) discovered_singer_catalog = SingerHelper.get_catalogs(logger, f"tap-stripe --config {config_container.rendered_config_path} --discover").singer_catalog - - combined_catalog_path = "/mount/rendered_catalog.json" - masked_singer_streams = [] - - stream_to_airbyte_schema = {} - for stream in masked_airbyte_catalog["streams"]: - stream_to_airbyte_schema[stream.get("name")] = stream - - for singer_stream in discovered_singer_catalog.get("streams"): - if singer_stream.get("stream") in stream_to_airbyte_schema: - new_metadatas = [] - metadatas = singer_stream.get("metadata") - for metadata in metadatas: - new_metadata = metadata - new_metadata["metadata"]["selected"] = True - if not is_field_metadata(new_metadata): - new_metadata["metadata"]["forced-replication-method"] = "FULL_TABLE" - new_metadatas += [new_metadata] - singer_stream["metadata"] = new_metadatas - - masked_singer_streams += [singer_stream] - - combined_catalog = {"streams": masked_singer_streams} - with open(combined_catalog_path, 'w') as fh: - fh.write(json.dumps(combined_catalog)) - - # todo: figure out how to make this easier to consume for new implementers + combined_catalog_path = SingerHelper.combine_catalogs(masked_airbyte_catalog, discovered_singer_catalog) if state: return SingerHelper.read(logger, f"tap-stripe --config {config_container.rendered_config_path} --catalog {combined_catalog_path} --state {state}") From da2d155e94bbb606190fbff9a0018e1cbe5f09ed Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 14 Oct 2020 08:23:17 -0700 Subject: [PATCH 18/31] fmt --- .../SingerExchangeRatesApiSourceTest.java | 39 +++++++++---------- .../singer/stripe_abprotocol/source/spec.json | 1 - .../sources/SingerStripeSourceTest.java | 7 ++-- .../test-integration/resources/catalog.json | 1 - .../src/test-integration/resources/spec.json | 1 - .../resources/stripe_schema_message.json | 1 - 6 files changed, 22 insertions(+), 28 deletions(-) diff --git a/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceTest.java b/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceTest.java index caeb159715e8f..cec4ae97b4822 100644 --- a/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceTest.java +++ b/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceTest.java @@ -24,14 +24,14 @@ package io.airbyte.integration_tests.sources; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.workers.WorkerException; import io.airbyte.workers.process.DockerProcessBuilderFactory; import io.airbyte.workers.process.ProcessBuilderFactory; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -40,9 +40,8 @@ import java.time.temporal.ChronoUnit; import java.util.Date; import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class SingerExchangeRatesApiSourceTest { @@ -133,24 +132,24 @@ public void testSync() throws IOException, InterruptedException, WorkerException private Process createSpecProcess() throws IOException, WorkerException { return pbf.create( - jobRoot, - IMAGE_NAME, - "spec") - .redirectOutput(ProcessBuilder.Redirect.INHERIT) - .redirectError(ProcessBuilder.Redirect.INHERIT) - .start(); + jobRoot, + IMAGE_NAME, + "spec") + .redirectOutput(ProcessBuilder.Redirect.INHERIT) + .redirectError(ProcessBuilder.Redirect.INHERIT) + .start(); } private Process createCheckProcess(String configFileName) throws IOException, WorkerException { return pbf.create( - jobRoot, - IMAGE_NAME, - "check", - "--config", - configFileName) - .redirectOutput(ProcessBuilder.Redirect.INHERIT) - .redirectError(ProcessBuilder.Redirect.INHERIT) - .start(); + jobRoot, + IMAGE_NAME, + "check", + "--config", + configFileName) + .redirectOutput(ProcessBuilder.Redirect.INHERIT) + .redirectError(ProcessBuilder.Redirect.INHERIT) + .start(); } private Process createDiscoveryProcess(String configFileName) throws IOException, WorkerException { diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/spec.json b/airbyte-integrations/singer/stripe_abprotocol/source/spec.json index 35d10e8f5128a..250f43dd40efa 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/spec.json +++ b/airbyte-integrations/singer/stripe_abprotocol/source/spec.json @@ -26,4 +26,3 @@ } } } - diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java index 4b100511efcd2..374c4be980851 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java +++ b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java @@ -235,9 +235,9 @@ private void writeInvalidConfigFile() throws IOException { private Process createCheckProcess(String configFileName) throws IOException, WorkerException { return launcher.check(jobRoot, configFileName) - .redirectOutput(catalogPath.toFile()) - .redirectError(ProcessBuilder.Redirect.INHERIT) - .start(); + .redirectOutput(catalogPath.toFile()) + .redirectError(ProcessBuilder.Redirect.INHERIT) + .start(); } private Process createDiscoveryProcess(String configFileName) throws IOException, WorkerException { @@ -255,4 +255,3 @@ private Process createSyncProcess(Path syncOutputPath) throws IOException, Worke } } - diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/catalog.json b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/catalog.json index 09de4fe599824..e70a967eb2f6c 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/catalog.json +++ b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/catalog.json @@ -1000,4 +1000,3 @@ } ] } - diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/spec.json b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/spec.json index 35d10e8f5128a..250f43dd40efa 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/spec.json +++ b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/spec.json @@ -26,4 +26,3 @@ } } } - diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/stripe_schema_message.json b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/stripe_schema_message.json index 2fc0cc3de9889..7fc75052df8d5 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/stripe_schema_message.json +++ b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/stripe_schema_message.json @@ -859,4 +859,3 @@ }, "key_properties": ["id"] } - From 8fc3b1af2b33c67eeb9f7eafb2b01e9edaf3ede5 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 14 Oct 2020 09:56:13 -0700 Subject: [PATCH 19/31] fix deadlock bug --- .../base-singer/base_singer/singer_helpers.py | 71 ++++++++++--------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py index 1c0a44d34050b..e81ff6756a1f0 100644 --- a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py +++ b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py @@ -1,5 +1,8 @@ import json +import os +import selectors import subprocess +import tempfile from airbyte_protocol import AirbyteSpec from airbyte_protocol import AirbyteCatalog from airbyte_protocol import AirbyteMessage @@ -66,41 +69,45 @@ def get_catalogs(logger, shell_command, singer_transform=(lambda catalog: catalo @staticmethod def read(logger, shell_command, is_message=(lambda x: True), transform=(lambda x: x)) -> Generator[AirbyteMessage, None, None]: - with subprocess.Popen(shell_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1, - universal_newlines=True) as p: - for line_tuple in zip(p.stdout, p.stderr): - out_line = line_tuple[0] - err_line = line_tuple[1] - - if out_line: - out_json = to_json(out_line) - if out_json is not None and is_message(out_json): - transformed_json = transform(out_json) - if transformed_json is not None: - if transformed_json.get('type') == "SCHEMA": - pass - elif transformed_json.get('type') == "STATE": - out_record = AirbyteStateMessage(data=transformed_json["value"]) - out_message = AirbyteMessage(type="STATE", state=out_record) - yield transform(out_message) - else: - # todo: check that messages match the discovered schema - stream_name = transformed_json["stream"] - out_record = AirbyteRecordMessage( - stream=stream_name, - data=transformed_json["record"], - emitted_at=int(datetime.now().timestamp()) * 1000) - out_message = AirbyteMessage(type="RECORD", record=out_record) - yield transform(out_message) - elif out_line: - logger(out_line, "INFO") - - if err_line: - logger(err_line, "ERROR") + with subprocess.Popen(shell_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) as p: + sel = selectors.DefaultSelector() + sel.register(p.stdout, selectors.EVENT_READ) + sel.register(p.stderr, selectors.EVENT_READ) + + ok = True + while ok: + for key, val1 in sel.select(): + line = key.fileobj.readline() + if not line: + ok = False + break + if key.fileobj is p.stdout: + out_json = to_json(line) + if out_json is not None and is_message(out_json): + transformed_json = transform(out_json) + if transformed_json is not None: + if transformed_json.get('type') == "SCHEMA": + pass + elif transformed_json.get('type') == "STATE": + out_record = AirbyteStateMessage(data=transformed_json["value"]) + out_message = AirbyteMessage(type="STATE", state=out_record) + yield transform(out_message) + else: + # todo: check that messages match the discovered schema + stream_name = transformed_json["stream"] + out_record = AirbyteRecordMessage( + stream=stream_name, + data=transformed_json["record"], + emitted_at=int(datetime.now().timestamp()) * 1000) + out_message = AirbyteMessage(type="RECORD", record=out_record) + yield transform(out_message) + else: + logger(line, "ERROR") + @staticmethod def combine_catalogs(masked_airbyte_catalog, discovered_singer_catalog) -> str: - combined_catalog_path = "/mount/rendered_catalog.json" + combined_catalog_path = os.path.join(tempfile.mkdtemp(), 'rendered_catalog.json') masked_singer_streams = [] stream_to_airbyte_schema = {} From 9a23ff30334ee8002c9d5a83fc7c22c9cee3b675 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 14 Oct 2020 09:56:45 -0700 Subject: [PATCH 20/31] convert catalog in test to airbyte catalog format --- .../test-integration/resources/catalog.json | 143 +----------------- 1 file changed, 2 insertions(+), 141 deletions(-) diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/catalog.json b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/catalog.json index e70a967eb2f6c..de802f60139cd 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/catalog.json +++ b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/catalog.json @@ -1,8 +1,7 @@ { "streams": [ { - "stream": "customers", - "tap_stream_id": "customers", + "name": "customers", "schema": { "type": ["null", "object"], "properties": { @@ -858,145 +857,7 @@ "format": "date-time" } } - }, - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": ["id"], - "forced-replication-method": "INCREMENTAL", - "selected": "true", - "valid-replication-keys": ["created"] - } - }, - { - "breadcrumb": ["properties", "metadata"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "shipping"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "sources"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "delinquent"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "description"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "livemode"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "default_source"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "cards"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "email"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "default_card"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "subscriptions"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "discount"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "account_balance"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "currency"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "invoice_prefix"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "tax_info_verification"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "object"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "created"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "tax_info"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "updated"], - "metadata": { - "inclusion": "automatic" - } - } - ], - "key_properties": ["id"] + } } ] } From f485b299d6ef8a1dc05697034aac65993afe0402 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 14 Oct 2020 10:04:26 -0700 Subject: [PATCH 21/31] fix stripe tests --- .../integration_tests/sources/SingerStripeSourceTest.java | 5 +++-- .../src/test-integration/resources/sync_output_subset.txt | 7 +++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java index 374c4be980851..e521c6e0e7a76 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java +++ b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java @@ -52,6 +52,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -173,7 +174,7 @@ public void testSync() throws IOException, InterruptedException, WorkerException // run syn process Path syncOutputPath = jobRoot.resolve("sync_output.txt"); Process process = createSyncProcess(syncOutputPath); - process.waitFor(); + process.waitFor(1, TimeUnit.MINUTES); assertEquals(0, process.exitValue()); @@ -192,7 +193,7 @@ private static JsonNode normalize(JsonNode node) { ObjectNode normalized = node.deepCopy(); if (normalized.get("type").textValue().equals("RECORD")) { - ObjectNode record = (ObjectNode) normalized.get("record"); + ObjectNode record = (ObjectNode) normalized.get("record").get("data"); record.put("id", "id"); record.put("created", "created"); record.put("invoice_prefix", "invoice_prefix"); diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/sync_output_subset.txt b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/sync_output_subset.txt index fac32348c4450..d6a61e1055c29 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/sync_output_subset.txt +++ b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/resources/sync_output_subset.txt @@ -1,4 +1,3 @@ -{"type": "SCHEMA", "stream": "customers", "schema": {"properties": {"metadata": {"properties": {}, "type": ["null", "object"]}, "shipping": {"properties": {"address": {"properties": {"line2": {"type": ["null", "string"]}, "state": {"type": ["null", "string"]}, "city": {"type": ["null", "string"]}, "postal_code": {"type": ["null", "string"]}, "country": {"type": ["null", "string"]}, "line1": {"type": ["null", "string"]}}, "type": ["null", "object"]}, "name": {"type": ["null", "string"]}, "phone": {"type": ["null", "string"]}}, "type": ["null", "object"]}, "sources": {"anyOf": [{"type": ["null", "array"], "items": {"type": ["null", "object"], "properties": {"metadata": {"type": ["null", "object"], "properties": {}}, "type": {"type": ["null", "string"]}, "address_zip": {"type": ["null", "string"]}, "livemode": {"type": ["null", "boolean"]}, "card": {"type": ["null", "object"], "properties": {"fingerprint": {"type": ["null", "string"]}, "last4": {"type": ["null", "string"]}, "dynamic_last4": {"type": ["null", "string"]}, "address_line1_check": {"type": ["null", "string"]}, "exp_month": {"type": ["null", "integer"]}, "tokenization_method": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "exp_year": {"type": ["null", "integer"]}, "three_d_secure": {"type": ["null", "string"]}, "funding": {"type": ["null", "string"]}, "brand": {"type": ["null", "string"]}, "cvc_check": {"type": ["null", "string"]}, "country": {"type": ["null", "string"]}, "address_zip_check": {"type": ["null", "string"]}, "type": {"type": ["null", "string"]}}}, "statement_descriptor": {"type": ["null", "string"]}, "id": {"type": ["null", "string"]}, "address_country": {"type": ["null", "string"]}, "funding": {"type": ["null", "string"]}, "dynamic_last4": {"type": ["null", "string"]}, "exp_year": {"type": ["null", "integer"]}, "last4": {"type": ["null", "string"]}, "exp_month": {"type": ["null", "integer"]}, "brand": {"type": ["null", "string"]}, "address_line2": {"type": ["null", "string"]}, "country": {"type": ["null", "string"]}, "object": {"type": ["null", "string"]}, "amount": {"type": ["null", "integer"]}, "cvc_check": {"type": ["null", "string"]}, "usage": {"type": ["null", "string"]}, "address_line1": {"type": ["null", "string"]}, "owner": {"type": ["null", "object"], "properties": {"verified_address": {"type": ["null", "string"]}, "email": {"type": ["null", "string"]}, "address": {"type": ["null", "object"], "properties": {"line2": {"type": ["null", "string"]}, "state": {"type": ["null", "string"]}, "city": {"type": ["null", "string"]}, "postal_code": {"type": ["null", "string"]}, "country": {"type": ["null", "string"]}, "line1": {"type": ["null", "string"]}}}, "verified_email": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "phone": {"type": ["null", "string"]}, "verified_name": {"type": ["null", "string"]}, "verified_phone": {"type": ["null", "string"]}}}, "tokenization_method": {"type": ["null", "string"]}, "client_secret": {"type": ["null", "string"]}, "fingerprint": {"type": ["null", "string"]}, "address_city": {"type": ["null", "string"]}, "currency": {"type": ["null", "string"]}, "address_line1_check": {"type": ["null", "string"]}, "receiver": {"type": ["null", "object"], "properties": {"refund_attributes_method": {"type": ["null", "string"]}, "amount_returned": {"type": ["null", "integer"]}, "amount_received": {"type": ["null", "integer"]}, "refund_attributes_status": {"type": ["null", "string"]}, "address": {"type": ["null", "string"]}, "amount_charged": {"type": ["null", "integer"]}}}, "flow": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "ach_credit_transfer": {"type": ["null", "object"], "properties": {"bank_name": {"type": ["null", "string"]}, "fingerprint": {"type": ["null", "string"]}, "routing_number": {"type": ["null", "string"]}, "swift_code": {"type": ["null", "string"]}, "refund_account_holder_type": {"type": ["null", "string"]}, "refund_account_holder_name": {"type": ["null", "string"]}, "refund_account_number": {"type": ["null", "string"]}, "refund_routing_number": {"type": ["null", "string"]}, "account_number": {"type": ["null", "string"]}}}, "customer": {"type": ["null", "string"]}, "address_zip_check": {"type": ["null", "string"]}, "status": {"type": ["null", "string"]}, "created": {"type": ["null", "string"], "format": "date-time"}, "address_state": {"type": ["null", "string"]}, "alipay": {"type": ["null", "object"], "properties": {}}, "bancontact": {"type": ["null", "object"], "properties": {}}, "eps": {"type": ["null", "object"], "properties": {}}, "ideal": {"type": ["null", "object"], "properties": {}}, "multibanco": {"type": ["null", "object"], "properties": {}}, "redirect": {"type": ["null", "object"], "properties": {"failure_reason": {"type": ["null", "string"]}, "return_url": {"type": ["null", "string"]}, "status": {"type": ["null", "string"]}, "url": {"type": ["null", "string"]}}}}}}, {"type": ["null", "object"], "properties": {"metadata": {"type": ["null", "object"], "properties": {}}, "type": {"type": ["null", "string"]}, "address_zip": {"type": ["null", "string"]}, "livemode": {"type": ["null", "boolean"]}, "card": {"type": ["null", "object"], "properties": {"fingerprint": {"type": ["null", "string"]}, "last4": {"type": ["null", "string"]}, "dynamic_last4": {"type": ["null", "string"]}, "address_line1_check": {"type": ["null", "string"]}, "exp_month": {"type": ["null", "integer"]}, "tokenization_method": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "exp_year": {"type": ["null", "integer"]}, "three_d_secure": {"type": ["null", "string"]}, "funding": {"type": ["null", "string"]}, "brand": {"type": ["null", "string"]}, "cvc_check": {"type": ["null", "string"]}, "country": {"type": ["null", "string"]}, "address_zip_check": {"type": ["null", "string"]}, "type": {"type": ["null", "string"]}}}, "statement_descriptor": {"type": ["null", "string"]}, "id": {"type": ["null", "string"]}, "address_country": {"type": ["null", "string"]}, "funding": {"type": ["null", "string"]}, "dynamic_last4": {"type": ["null", "string"]}, "exp_year": {"type": ["null", "integer"]}, "last4": {"type": ["null", "string"]}, "exp_month": {"type": ["null", "integer"]}, "brand": {"type": ["null", "string"]}, "address_line2": {"type": ["null", "string"]}, "country": {"type": ["null", "string"]}, "object": {"type": ["null", "string"]}, "amount": {"type": ["null", "integer"]}, "cvc_check": {"type": ["null", "string"]}, "usage": {"type": ["null", "string"]}, "address_line1": {"type": ["null", "string"]}, "owner": {"type": ["null", "object"], "properties": {"verified_address": {"type": ["null", "string"]}, "email": {"type": ["null", "string"]}, "address": {"type": ["null", "object"], "properties": {"line2": {"type": ["null", "string"]}, "state": {"type": ["null", "string"]}, "city": {"type": ["null", "string"]}, "postal_code": {"type": ["null", "string"]}, "country": {"type": ["null", "string"]}, "line1": {"type": ["null", "string"]}}}, "verified_email": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "phone": {"type": ["null", "string"]}, "verified_name": {"type": ["null", "string"]}, "verified_phone": {"type": ["null", "string"]}}}, "tokenization_method": {"type": ["null", "string"]}, "client_secret": {"type": ["null", "string"]}, "fingerprint": {"type": ["null", "string"]}, "address_city": {"type": ["null", "string"]}, "currency": {"type": ["null", "string"]}, "address_line1_check": {"type": ["null", "string"]}, "receiver": {"type": ["null", "object"], "properties": {"refund_attributes_method": {"type": ["null", "string"]}, "amount_returned": {"type": ["null", "integer"]}, "amount_received": {"type": ["null", "integer"]}, "refund_attributes_status": {"type": ["null", "string"]}, "address": {"type": ["null", "string"]}, "amount_charged": {"type": ["null", "integer"]}}}, "flow": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "ach_credit_transfer": {"type": ["null", "object"], "properties": {"bank_name": {"type": ["null", "string"]}, "fingerprint": {"type": ["null", "string"]}, "routing_number": {"type": ["null", "string"]}, "swift_code": {"type": ["null", "string"]}, "refund_account_holder_type": {"type": ["null", "string"]}, "refund_account_holder_name": {"type": ["null", "string"]}, "refund_account_number": {"type": ["null", "string"]}, "refund_routing_number": {"type": ["null", "string"]}, "account_number": {"type": ["null", "string"]}}}, "customer": {"type": ["null", "string"]}, "address_zip_check": {"type": ["null", "string"]}, "status": {"type": ["null", "string"]}, "created": {"type": ["null", "string"], "format": "date-time"}, "address_state": {"type": ["null", "string"]}, "alipay": {"type": ["null", "object"], "properties": {}}, "bancontact": {"type": ["null", "object"], "properties": {}}, "eps": {"type": ["null", "object"], "properties": {}}, "ideal": {"type": ["null", "object"], "properties": {}}, "multibanco": {"type": ["null", "object"], "properties": {}}, "redirect": {"type": ["null", "object"], "properties": {"failure_reason": {"type": ["null", "string"]}, "return_url": {"type": ["null", "string"]}, "status": {"type": ["null", "string"]}, "url": {"type": ["null", "string"]}}}}}]}, "delinquent": {"type": ["null", "boolean"]}, "description": {"type": ["null", "string"]}, "livemode": {"type": ["null", "boolean"]}, "default_source": {"type": ["null", "string"]}, "cards": {"items": {"properties": {"metadata": {"properties": {}, "type": ["null", "object"]}, "object": {"type": ["null", "string"]}, "id": {"type": ["null", "string"]}, "exp_month": {"type": ["null", "integer"]}, "dynamic_last4": {"type": ["null", "string"]}, "exp_year": {"type": ["null", "integer"]}, "last4": {"type": ["null", "string"]}, "funding": {"type": ["null", "string"]}, "brand": {"type": ["null", "string"]}, "country": {"type": ["null", "string"]}, "customer": {"type": ["null", "string"]}, "cvc_check": {"type": ["null", "string"]}, "address_line2": {"type": ["null", "string"]}, "address_line1": {"type": ["null", "string"]}, "fingerprint": {"type": ["null", "string"]}, "address_zip": {"type": ["null", "string"]}, "address_city": {"type": ["null", "string"]}, "address_country": {"type": ["null", "string"]}, "address_line1_check": {"type": ["null", "string"]}, "tokenization_method": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "address_state": {"type": ["null", "string"]}, "address_zip_check": {"type": ["null", "string"]}, "type": {"type": ["null", "string"]}}, "type": ["null", "object"]}, "type": ["null", "array"]}, "email": {"type": ["null", "string"]}, "default_card": {"type": ["null", "string"]}, "subscriptions": {"items": {"type": ["null", "string"]}, "type": ["null", "array"]}, "discount": {"properties": {"end": {"format": "date-time", "type": ["null", "string"]}, "coupon": {"properties": {"metadata": {"properties": {}, "type": ["null", "object"]}, "valid": {"type": ["null", "boolean"]}, "livemode": {"type": ["null", "boolean"]}, "amount_off": {"type": ["null", "integer"]}, "redeem_by": {"format": "date-time", "type": ["null", "string"]}, "duration_in_months": {"type": ["null", "integer"]}, "percent_off_precise": {"type": ["null", "number"]}, "max_redemptions": {"type": ["null", "integer"]}, "currency": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}, "times_redeemed": {"type": ["null", "integer"]}, "id": {"type": ["null", "string"]}, "duration": {"type": ["null", "string"]}, "object": {"type": ["null", "string"]}, "percent_off": {"type": ["null", "integer"]}, "created": {"format": "date-time", "type": ["null", "string"]}}, "type": ["null", "object"]}, "customer": {"type": ["null", "string"]}, "start": {"format": "date-time", "type": ["null", "string"]}, "object": {"type": ["null", "string"]}, "subscription": {"type": ["null", "string"]}}, "type": ["null", "object"]}, "account_balance": {"type": ["null", "integer"]}, "currency": {"type": ["null", "string"]}, "id": {"type": ["null", "string"]}, "invoice_prefix": {"type": ["null", "string"]}, "tax_info_verification": {"type": ["null", "string"]}, "object": {"type": ["null", "string"]}, "created": {"format": "date-time", "type": ["null", "string"]}, "tax_info": {"type": ["null", "string"]}, "updated": {"format": "date-time", "type": ["null", "string"]}}, "type": ["null", "object"]}, "key_properties": ["id"]}{"type": "RECORD", "stream": "customers", "record": {"id": "cus_I1l0AbdUZF4Wrl", "object": "customer", "account_balance": 0, "created": "2020-09-15T16:58:53.000000Z", "currency": null, "default_source": null, "delinquent": false, "description": "Customer 4", "discount": null, "email": "customer4@test.com", "invoice_prefix": "AFC537CE", "livemode": false, "metadata": {}, "shipping": null, "sources": [], "subscriptions": null, "tax_info": null, "tax_info_verification": null, "updated": "2020-09-15T16:58:53.000000Z"}, "time_extracted": "2020-09-15T18:01:23.634272Z"} -{"type": "RECORD", "stream": "customers", "record": {"id": "cus_I1l0XHrjLYwLR2", "object": "customer", "account_balance": 0, "created": "2020-09-15T16:58:52.000000Z", "currency": null, "default_source": null, "delinquent": false, "description": "Customer 3", "discount": null, "email": "customer3@test.com", "invoice_prefix": "EC156D8F", "livemode": false, "metadata": {}, "shipping": null, "sources": [], "subscriptions": null, "tax_info": null, "tax_info_verification": null, "updated": "2020-09-15T16:58:52.000000Z"}, "time_extracted": "2020-09-15T18:01:23.634272Z"} -{"type": "RECORD", "stream": "customers", "record": {"id": "cus_I1l0INzfeSf2MM", "object": "customer", "account_balance": 0, "created": "2020-09-15T16:58:52.000000Z", "currency": null, "default_source": null, "delinquent": false, "description": "Customer 2", "discount": null, "email": "customer2@test.com", "invoice_prefix": "D4564D22", "livemode": false, "metadata": {}, "shipping": null, "sources": [], "subscriptions": null, "tax_info": null, "tax_info_verification": null, "updated": "2020-09-15T16:58:52.000000Z"}, "time_extracted": "2020-09-15T18:01:23.634272Z"} -{"type": "RECORD", "stream": "customers", "record": {"id": "cus_I1l0cRVFy4ZhwQ", "object": "customer", "account_balance": 0, "created": "2020-09-15T16:58:52.000000Z", "currency": null, "default_source": null, "delinquent": false, "description": "Customer 1", "discount": null, "email": "customer1@test.com", "invoice_prefix": "92A8C396", "livemode": false, "metadata": {}, "shipping": null, "sources": [], "subscriptions": null, "tax_info": null, "tax_info_verification": null, "updated": "2020-09-15T16:58:52.000000Z"}, "time_extracted": "2020-09-15T18:01:23.634272Z"} +{"type":"RECORD","record":{"stream":"customers","data":{"id":"cus_I1l0XHrjLYwLR2","object":"customer","created":"2020-09-15T16:58:52.000000Z","delinquent":false,"description":"Customer 3","email":"customer3@test.com","invoice_prefix":"EC156D8F","livemode":false,"metadata":{},"updated":"2020-09-16T18:07:22.000000Z"},"emitted_at":1602694534000}} +{"type":"RECORD","record":{"stream":"customers","data":{"id":"cus_I1l0INzfeSf2MM","object":"customer","created":"2020-09-15T16:58:52.000000Z","delinquent":false,"description":"Customer 2","email":"customer2@test.com","invoice_prefix":"D4564D22","livemode":false,"metadata":{},"updated":"2020-09-16T18:07:23.000000Z"},"emitted_at":1602694534000}} +{"type":"RECORD","record":{"stream":"customers","data":{"id":"cus_I1l0cRVFy4ZhwQ","object":"customer","created":"2020-09-15T16:58:52.000000Z","delinquent":false,"description":"Customer 1","email":"customer1@test.com","invoice_prefix":"92A8C396","livemode":false,"metadata":{},"updated":"2020-09-16T18:07:24.000000Z"},"emitted_at":1602694534000}} From 8bf9d50074fd5cfa802a81045f07e86629c3834b Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 14 Oct 2020 10:06:17 -0700 Subject: [PATCH 22/31] fmt --- .../sources/SingerStripeSourceTest.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java index e521c6e0e7a76..b1e5cc699581c 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java +++ b/airbyte-integrations/singer/stripe_abprotocol/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerStripeSourceTest.java @@ -24,6 +24,10 @@ package io.airbyte.integration_tests.sources; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.stripe.exception.StripeException; @@ -38,10 +42,6 @@ import io.airbyte.workers.process.AirbyteIntegrationLauncher; import io.airbyte.workers.process.DockerProcessBuilderFactory; import io.airbyte.workers.process.IntegrationLauncher; -import org.apache.commons.lang3.RandomStringUtils; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; @@ -54,10 +54,9 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class SingerStripeSourceTest { From 9342e084fca55c71bb2e881c5033f8b74978d29d Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 14 Oct 2020 10:16:19 -0700 Subject: [PATCH 23/31] remoce unused log command remove config --- airbyte-integrations/base-python/base.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/airbyte-integrations/base-python/base.py b/airbyte-integrations/base-python/base.py index 5fbd56e7ab4de..036b201e24d5e 100644 --- a/airbyte-integrations/base-python/base.py +++ b/airbyte-integrations/base-python/base.py @@ -17,12 +17,6 @@ impl = getattr(module, impl_class) -def log(level, text): - log_message = AirbyteLogMessage(level=level, message=text) - message = AirbyteMessage(type="LOG", log=log_message) - print(message.serialize) - - class AirbyteEntrypoint(object): def __init__(self, source): self.source = source From 75ddaa488b58b1e26e433b362983612bff51f3a2 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 14 Oct 2020 10:24:31 -0700 Subject: [PATCH 24/31] output credentials --- tools/bin/ci_credentials.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 6c562fdc0d206..19f3d92cde1f1 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -8,3 +8,6 @@ echo "$BIGQUERY_INTEGRATION_TEST_CREDS" > airbyte-integrations/bigquery-destinat mkdir airbyte-integrations/singer/stripe/source/config echo "$STRIPE_INTEGRATION_TEST_CREDS" > airbyte-integrations/singer/stripe/source/config/config.json + +mkdir airbyte-integrations/singer/stripe_abprotocol/source/config +echo "$STRIPE_INTEGRATION_TEST_CREDS" > airbyte-integrations/singer/stripe_abprotocol/source/config/config.json From 4cfaf8e300d57f15e91947f3349041ffa37285bc Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 14 Oct 2020 10:33:54 -0700 Subject: [PATCH 25/31] clean up optional state command --- .../source_exchangeratesapi_singer.py | 7 +++---- .../source/source_stripe_singer/source_stripe_singer.py | 8 ++++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py index d285caed44fe8..ef8c78391c2e0 100644 --- a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py +++ b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py @@ -25,7 +25,6 @@ def discover(self, logger, config_container) -> AirbyteCatalog: return catalogs.airbyte_catalog def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: - if state: - return SingerHelper.read(logger, f"tap-exchangeratesapi --config {config_container.rendered_config_path} --state {state}") - else: - return SingerHelper.read(logger, f"tap-exchangeratesapi --config {config_container.rendered_config_path}") + config_option = f"--config {config_container.rendered_config_path}" + state_option = f"--state {state}" if state else "" + return SingerHelper.read(logger, f"tap-exchangeratesapi {config_option} {state_option}") diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py index a3d3d4e9ef922..586ff7d0897d8 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py +++ b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py @@ -30,7 +30,7 @@ def read(self, logger, config_container, catalog_path, state=None) -> Generator[ discovered_singer_catalog = SingerHelper.get_catalogs(logger, f"tap-stripe --config {config_container.rendered_config_path} --discover").singer_catalog combined_catalog_path = SingerHelper.combine_catalogs(masked_airbyte_catalog, discovered_singer_catalog) - if state: - return SingerHelper.read(logger, f"tap-stripe --config {config_container.rendered_config_path} --catalog {combined_catalog_path} --state {state}") - else: - return SingerHelper.read(logger, f"tap-stripe --config {config_container.rendered_config_path} --catalog {combined_catalog_path}") + config_option = f"--config {config_container.rendered_config_path}" + catalog_option = f"--catalog {combined_catalog_path}" + state_option = f"--state {state}" if state else "" + return SingerHelper.read(logger, f"tap-stripe {config_option} {catalog_option} {state_option}") From 4aa95e06d17dc95c4f94cd4df75ede9cdb5c4ee8 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 14 Oct 2020 10:41:31 -0700 Subject: [PATCH 26/31] use integration launcher --- .../SingerExchangeRatesApiSourceTest.java | 54 +++++++------------ 1 file changed, 19 insertions(+), 35 deletions(-) diff --git a/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceTest.java b/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceTest.java index cec4ae97b4822..d6733d6db0988 100644 --- a/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceTest.java +++ b/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceTest.java @@ -24,14 +24,15 @@ package io.airbyte.integration_tests.sources; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.workers.WorkerException; +import io.airbyte.workers.process.AirbyteIntegrationLauncher; import io.airbyte.workers.process.DockerProcessBuilderFactory; import io.airbyte.workers.process.ProcessBuilderFactory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -40,8 +41,9 @@ import java.time.temporal.ChronoUnit; import java.util.Date; import java.util.Optional; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class SingerExchangeRatesApiSourceTest { @@ -53,7 +55,7 @@ public class SingerExchangeRatesApiSourceTest { protected Path jobRoot; protected Path workspaceRoot; - protected ProcessBuilderFactory pbf; + AirbyteIntegrationLauncher launcher; protected Path catalogPath; @BeforeEach @@ -66,11 +68,13 @@ public void setUp() throws IOException { Files.createDirectories(jobRoot); - pbf = new DockerProcessBuilderFactory( + final ProcessBuilderFactory pbf = new DockerProcessBuilderFactory( workspaceRoot, workspaceRoot.toString(), "", "host"); + + launcher = new AirbyteIntegrationLauncher(IMAGE_NAME, pbf); } @Test @@ -87,7 +91,7 @@ public void testSpec() throws InterruptedException, IOException, WorkerException public void testCheck() throws IOException, WorkerException, InterruptedException { IOs.writeFile(jobRoot, CONFIG, "{}"); - Process process = createCheckProcess(CONFIG); + Process process = createCheckProcess(); process.waitFor(); assertEquals(0, process.exitValue()); @@ -97,7 +101,7 @@ public void testCheck() throws IOException, WorkerException, InterruptedExceptio public void testSuccessfulDiscover() throws IOException, InterruptedException, WorkerException { IOs.writeFile(jobRoot, CONFIG, "{}"); - Process process = createDiscoveryProcess(CONFIG); + Process process = createDiscoveryProcess(); process.waitFor(); assertEquals(0, process.exitValue()); @@ -131,48 +135,28 @@ public void testSync() throws IOException, InterruptedException, WorkerException } private Process createSpecProcess() throws IOException, WorkerException { - return pbf.create( - jobRoot, - IMAGE_NAME, - "spec") + return launcher.spec(jobRoot) .redirectOutput(ProcessBuilder.Redirect.INHERIT) .redirectError(ProcessBuilder.Redirect.INHERIT) .start(); } - private Process createCheckProcess(String configFileName) throws IOException, WorkerException { - return pbf.create( - jobRoot, - IMAGE_NAME, - "check", - "--config", - configFileName) + private Process createCheckProcess() throws IOException, WorkerException { + return launcher.check(jobRoot, CONFIG) .redirectOutput(ProcessBuilder.Redirect.INHERIT) .redirectError(ProcessBuilder.Redirect.INHERIT) .start(); } - private Process createDiscoveryProcess(String configFileName) throws IOException, WorkerException { - return pbf.create( - jobRoot, - IMAGE_NAME, - "discover", - "--config", - configFileName) + private Process createDiscoveryProcess() throws IOException, WorkerException { + return launcher.discover(jobRoot, CONFIG) .redirectOutput(catalogPath.toFile()) .redirectError(ProcessBuilder.Redirect.INHERIT) .start(); } private Process createSyncProcess(Path syncOutputPath) throws IOException, WorkerException { - return pbf.create( - jobRoot, - IMAGE_NAME, - "read", - "--config", - CONFIG, - "--catalog", - "catalog.json") + return launcher.read(jobRoot, CONFIG, "catalog.json") .redirectOutput(syncOutputPath.toFile()) .redirectError(ProcessBuilder.Redirect.INHERIT) .start(); From 07510139f78be20e818d8908dff3d717b9fae663 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 14 Oct 2020 10:49:18 -0700 Subject: [PATCH 27/31] DRY up read eval --- airbyte-integrations/base/base.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/base/base.sh b/airbyte-integrations/base/base.sh index 6588424b43818..d7438206edcbe 100755 --- a/airbyte-integrations/base/base.sh +++ b/airbyte-integrations/base/base.sh @@ -48,11 +48,9 @@ function main() { eval "$AIRBYTE_DISCOVER_CMD" --config "$CONFIG_FILE" ;; read) - if [[ -z "$STATE_FILE" ]]; then - eval "$AIRBYTE_READ_CMD" --config "$CONFIG_FILE" --catalog "$CATALOG_FILE" - else - eval "$AIRBYTE_READ_CMD" --config "$CONFIG_FILE" --catalog "$CATALOG_FILE" --state "$STATE_FILE" - fi + READ_STATEMENT="$AIRBYTE_READ_CMD --config $CONFIG_FILE --catalog $CATALOG_FILE" + if [[ -z "$STATE_FILE" ]]; then READ_STATEMENT="$READ_STATEMENT --state $STATE_FILE"; fi + eval "$READ_STATEMENT" ;; write) eval "$AIRBYTE_WRITE_CMD" --config "$CONFIG_FILE" --catalog "$CATALOG_FILE" From df15670774ed6dd55092f3e99fe11273b7cd7d8c Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 14 Oct 2020 11:08:12 -0700 Subject: [PATCH 28/31] logging fixes and remove break --- .../airbyte_protocol/__init__.py | 52 +++++++++++++------ airbyte-integrations/base-python/base.py | 13 ++--- .../base-singer/base_singer/singer_helpers.py | 9 ++-- .../source_exchangeratesapi_singer.py | 2 +- 4 files changed, 50 insertions(+), 26 deletions(-) diff --git a/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py b/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py index 47659f3517a90..816db9eb18944 100644 --- a/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py +++ b/airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py @@ -79,21 +79,43 @@ def __init__(self): pass -valid_log_types = ["FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE"] - - -def log_line(line, default_level): - split_line = line.split() - first_word = next(iter(split_line), None) - if first_word in valid_log_types: - log_level = first_word - rendered_line = " ".join(split_line[1:]) - else: - log_level = default_level - rendered_line = line - log_record = AirbyteLogMessage(level=log_level, message=rendered_line) - log_message = AirbyteMessage(type="LOG", log=log_record) - print(log_message.serialize()) +class AirbyteLogger: + def __init__(self): + self.valid_log_types = ["FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE"] + + def log_by_prefix(self, message, default_level): + split_line = message.split() + first_word = next(iter(split_line), None) + if first_word in self.valid_log_types: + log_level = first_word + rendered_message = " ".join(split_line[1:]) + else: + log_level = default_level + rendered_message = message + self.log(log_level, rendered_message) + + def log(self, level, message): + log_record = AirbyteLogMessage(level=level, message=message) + log_message = AirbyteMessage(type="LOG", log=log_record) + print(log_message.serialize()) + + def fatal(self, message): + self.log("FATAL", message) + + def error(self, message): + self.log("ERROR", message) + + def warn(self, message): + self.log("WARN", message) + + def info(self, message): + self.log("INFO", message) + + def debug(self, message): + self.log("DEBUG", message) + + def trace(self, message): + self.log("TRACE", message) @dataclass class ConfigContainer: diff --git a/airbyte-integrations/base-python/base.py b/airbyte-integrations/base-python/base.py index 036b201e24d5e..86e5f32a53ad6 100644 --- a/airbyte-integrations/base-python/base.py +++ b/airbyte-integrations/base-python/base.py @@ -6,9 +6,9 @@ from airbyte_protocol import ConfigContainer from airbyte_protocol import Source +from airbyte_protocol import AirbyteLogger from airbyte_protocol import AirbyteLogMessage from airbyte_protocol import AirbyteMessage -from airbyte_protocol import log_line impl_module = os.environ['AIRBYTE_IMPL_MODULE'] impl_class = os.environ['AIRBYTE_IMPL_PATH'] @@ -16,6 +16,7 @@ module = importlib.import_module(impl_module) impl = getattr(module, impl_class) +logger = AirbyteLogger() class AirbyteEntrypoint(object): def __init__(self, source): @@ -81,19 +82,19 @@ def start(self): rendered_config_path=rendered_config_path) if cmd == "check": - check_result = source.check(log_line, config_container) + check_result = source.check(logger, config_container) if check_result.successful: - log_line("Check succeeded", "INFO") + logger.info("Check succeeded") sys.exit(0) else: - log_line("Check failed", "ERROR") + logger.error("Check failed") sys.exit(1) elif cmd == "discover": - catalog = source.discover(log_line, config_container) + catalog = source.discover(logger, config_container) print(catalog.serialize()) sys.exit(0) elif cmd == "read": - generator = source.read(log_line, config_container, parsed_args.catalog, parsed_args.state) + generator = source.read(logger, config_container, parsed_args.catalog, parsed_args.state) for message in generator: print(message.serialize()) sys.exit(0) diff --git a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py index e81ff6756a1f0..38b969c144b77 100644 --- a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py +++ b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py @@ -48,7 +48,7 @@ def get_catalogs(logger, shell_command, singer_transform=(lambda catalog: catalo universal_newlines=True) for line in completed_process.stderr.splitlines(): - logger(line, "ERROR") + logger.log_by_prefix(line, "ERROR") airbyte_streams = [] singer_catalog = singer_transform(json.loads(completed_process.stdout)) @@ -80,8 +80,7 @@ def read(logger, shell_command, is_message=(lambda x: True), transform=(lambda x line = key.fileobj.readline() if not line: ok = False - break - if key.fileobj is p.stdout: + elif key.fileobj is p.stdout: out_json = to_json(line) if out_json is not None and is_message(out_json): transformed_json = transform(out_json) @@ -101,8 +100,10 @@ def read(logger, shell_command, is_message=(lambda x: True), transform=(lambda x emitted_at=int(datetime.now().timestamp()) * 1000) out_message = AirbyteMessage(type="RECORD", record=out_record) yield transform(out_message) + else: + logger.log_by_prefix(line, "INFO") else: - logger(line, "ERROR") + logger.log_by_prefix(line, "ERROR") @staticmethod diff --git a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py index ef8c78391c2e0..4b507b193a9c6 100644 --- a/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py +++ b/airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py @@ -17,7 +17,7 @@ def spec(self) -> AirbyteSpec: def check(self, logger, config_container) -> AirbyteCheckResponse: code = urllib.request.urlopen("https://api.exchangeratesapi.io/").getcode() - logger(f"Ping response code: {code}", "INFO") + logger.info(f"Ping response code: {code}") return AirbyteCheckResponse(code == 200, {}) def discover(self, logger, config_container) -> AirbyteCatalog: From d3803306bc8f2a12e30a0cb499b86385203e2351 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 14 Oct 2020 11:32:09 -0700 Subject: [PATCH 29/31] rename --- .../singer/base-singer/base_singer/singer_helpers.py | 2 +- .../source/source_stripe_singer/source_stripe_singer.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py index 38b969c144b77..71adc5dc9f101 100644 --- a/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py +++ b/airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py @@ -107,7 +107,7 @@ def read(logger, shell_command, is_message=(lambda x: True), transform=(lambda x @staticmethod - def combine_catalogs(masked_airbyte_catalog, discovered_singer_catalog) -> str: + def create_singer_catalog_with_selection(masked_airbyte_catalog, discovered_singer_catalog) -> str: combined_catalog_path = os.path.join(tempfile.mkdtemp(), 'rendered_catalog.json') masked_singer_streams = [] diff --git a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py index 586ff7d0897d8..e5d409aa33140 100644 --- a/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py +++ b/airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py @@ -28,9 +28,9 @@ def discover(self, logger, config_container) -> AirbyteCatalog: def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: masked_airbyte_catalog = self.read_config(catalog_path) discovered_singer_catalog = SingerHelper.get_catalogs(logger, f"tap-stripe --config {config_container.rendered_config_path} --discover").singer_catalog - combined_catalog_path = SingerHelper.combine_catalogs(masked_airbyte_catalog, discovered_singer_catalog) + selected_singer_catalog = SingerHelper.create_singer_catalog_with_selection(masked_airbyte_catalog, discovered_singer_catalog) config_option = f"--config {config_container.rendered_config_path}" - catalog_option = f"--catalog {combined_catalog_path}" + catalog_option = f"--catalog {selected_singer_catalog}" state_option = f"--state {state}" if state else "" return SingerHelper.read(logger, f"tap-stripe {config_option} {catalog_option} {state_option}") From bef954b1083929e9b3df8ea9179f70802561983c Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 14 Oct 2020 11:35:05 -0700 Subject: [PATCH 30/31] fmt --- .../sources/SingerExchangeRatesApiSourceTest.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceTest.java b/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceTest.java index d6733d6db0988..95b857b503fc7 100644 --- a/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceTest.java +++ b/airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceTest.java @@ -24,15 +24,15 @@ package io.airbyte.integration_tests.sources; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.workers.WorkerException; import io.airbyte.workers.process.AirbyteIntegrationLauncher; import io.airbyte.workers.process.DockerProcessBuilderFactory; import io.airbyte.workers.process.ProcessBuilderFactory; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -41,9 +41,8 @@ import java.time.temporal.ChronoUnit; import java.util.Date; import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class SingerExchangeRatesApiSourceTest { From 8f673eea1c3cc30f43f445fd61f03b36be2d5e8d Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 14 Oct 2020 11:49:52 -0700 Subject: [PATCH 31/31] fix inverted if statement --- airbyte-integrations/base/base.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/base/base.sh b/airbyte-integrations/base/base.sh index d7438206edcbe..b72ad35e18eb0 100755 --- a/airbyte-integrations/base/base.sh +++ b/airbyte-integrations/base/base.sh @@ -49,7 +49,7 @@ function main() { ;; read) READ_STATEMENT="$AIRBYTE_READ_CMD --config $CONFIG_FILE --catalog $CATALOG_FILE" - if [[ -z "$STATE_FILE" ]]; then READ_STATEMENT="$READ_STATEMENT --state $STATE_FILE"; fi + if [[ ! -z "$STATE_FILE" ]]; then READ_STATEMENT="$READ_STATEMENT --state $STATE_FILE"; fi eval "$READ_STATEMENT" ;; write)