-
Notifications
You must be signed in to change notification settings - Fork 4.2k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
working python source #533
Changes from 1 commit
5f8f787
5a1c16e
13eea10
92b3bd7
d732de6
f701fa4
7cfb17c
07e7e4d
d394766
cde03b9
f926838
4d5c0c9
84c7db9
09a442c
4ad1db2
dda65c3
bbe34ad
b4a6e92
5fe3a1e
da2d155
8fc3b1a
9a23ff3
f485b29
8bf9d50
76a97e5
9342e08
75ddaa4
4cfaf8e
4aa95e0
0751013
df15670
d380330
bef954b
8f673ee
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,8 +1,10 @@ | ||
from typing import Generator | ||
import yaml | ||
import json | ||
import pkgutil | ||
import warnings | ||
import python_jsonschema_objects as pjs | ||
from dataclasses import dataclass | ||
|
||
|
||
def _load_classes(yaml_path: str): | ||
|
@@ -36,33 +38,30 @@ def __init__(self, successful, field_to_error): | |
self.field_to_error = field_to_error | ||
|
||
|
||
class AirbyteConfig(object): | ||
def __init__(self, config_string): | ||
self.config_string = config_string | ||
|
||
|
||
class Integration(object): | ||
def __init__(self): | ||
pass | ||
|
||
def spec(self) -> AirbyteSpec: | ||
raise Exception("Not Implemented") | ||
|
||
# default version reads the config_path to a string | ||
# this will often be overwritten to add fields for easy consumption or to modify the string for delegating to singer | ||
def read_config(self, config_path) -> AirbyteConfig: | ||
def read_config(self, config_path): | ||
with open(config_path, 'r') as file: | ||
contents = file.read() | ||
return AirbyteConfig(contents) | ||
return json.loads(contents) | ||
|
||
def render_config(self, config_object, rendered_config_path): | ||
with open(rendered_config_path, 'w') as fh: | ||
fh.write(config_object.config_string) | ||
# can be overridden to change an input file config | ||
def transform_config(self, raw_config): | ||
return raw_config | ||
|
||
def check(self, logger, rendered_config_path) -> AirbyteCheckResponse: | ||
def write_config(self, config_object, path): | ||
jrhizor marked this conversation as resolved.
Show resolved
Hide resolved
|
||
with open(path, 'w') as fh: | ||
fh.write(json.dumps(config_object)) | ||
|
||
def check(self, logger, config_container) -> AirbyteCheckResponse: | ||
raise Exception("Not Implemented") | ||
|
||
def discover(self, logger, rendered_config_path) -> AirbyteCatalog: | ||
def discover(self, logger, config_container) -> AirbyteCatalog: | ||
raise Exception("Not Implemented") | ||
|
||
|
||
|
@@ -71,7 +70,7 @@ def __init__(self): | |
pass | ||
|
||
# Iterator<AirbyteMessage> | ||
def read(self, logger, rendered_config_path, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: | ||
def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This will come in a future PR? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No, this is expected to be implemented by specific sources. There is no default implementation. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ah, makes sense |
||
raise Exception("Not Implemented") | ||
|
||
|
||
|
@@ -95,3 +94,10 @@ def log_line(line, default_level): | |
log_record = AirbyteLogMessage(level=log_level, message=rendered_line) | ||
log_message = AirbyteMessage(type="LOG", log=log_record) | ||
print(log_message.serialize()) | ||
|
||
@dataclass | ||
class ConfigContainer: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Maybe add a comment to explain what these fields are. |
||
raw_config: object | ||
rendered_config: object | ||
raw_config_path: str | ||
rendered_config_path: str |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -5,6 +5,7 @@ | |
import os.path | ||
import importlib | ||
|
||
from airbyte_protocol import ConfigContainer | ||
from airbyte_protocol import Source | ||
from airbyte_protocol import AirbyteLogMessage | ||
from airbyte_protocol import AirbyteMessage | ||
|
@@ -76,24 +77,30 @@ def start(self): | |
sys.exit(0) | ||
|
||
rendered_config_path = os.path.join(temp_dir, 'config.json') | ||
config_object = source.read_config(parsed_args.config) | ||
source.render_config(config_object, rendered_config_path) | ||
raw_config = source.read_config(parsed_args.config) | ||
rendered_config = source.transform_config(raw_config) | ||
source.write_config(rendered_config, rendered_config_path) | ||
|
||
config_container = ConfigContainer( | ||
raw_config=raw_config, | ||
rendered_config=rendered_config, | ||
raw_config_path=parsed_args.config, | ||
rendered_config_path=rendered_config_path) | ||
|
||
# todo: output message for check | ||
if cmd == "check": | ||
check_result = source.check(log_line, rendered_config_path) | ||
check_result = source.check(log_line, config_container) | ||
if check_result.successful: | ||
log("INFO", "Check succeeded") | ||
sys.exit(0) | ||
else: | ||
log("ERROR", "Check failed") | ||
sys.exit(1) | ||
elif cmd == "discover": | ||
catalog = source.discover(log_line, rendered_config_path) | ||
catalog = source.discover(log_line, config_container) | ||
print(catalog.serialize()) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. is There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Since we want to dictate the precise format of what we're outputting, we don't want an It's debatable if we should use |
||
sys.exit(0) | ||
elif cmd == "read": | ||
generator = source.read(log_line, rendered_config_path, parsed_args.catalog, parsed_args.state) | ||
generator = source.read(log_line, config_container, parsed_args.catalog, parsed_args.state) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should we have have a similar "container" for catalog and state? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. will do this in a separate PR |
||
for message in generator: | ||
print(message.serialize()) | ||
sys.exit(0) | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
why are there two
airbyte_protocol
directories? It's not clear to me what the difference between the two is?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This seems like standard structure for python packages:
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
SGTM, thanks for bearing with my un-pythonic ways :P