Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Include tcp port in Clickhouse destination configuration for normalization #9340

Merged
merged 4 commits into from
Jan 6, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
- name: Clickhouse
destinationDefinitionId: ce0d828e-1dc4-496c-b122-2da42e637e48
dockerRepository: airbyte/destination-clickhouse
dockerImageTag: 0.1.1
dockerImageTag: 0.1.2
documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse
- name: DynamoDB
destinationDefinitionId: 8ccd8909-4e99-4141-b48d-4984b70b2d89
Expand Down
20 changes: 15 additions & 5 deletions airbyte-config/init/src/main/resources/seed/destination_specs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -646,7 +646,7 @@
supported_destination_sync_modes:
- "overwrite"
- "append"
- dockerImage: "airbyte/destination-clickhouse:0.1.1"
- dockerImage: "airbyte/destination-clickhouse:0.1.2"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse"
connectionSpecification:
Expand Down Expand Up @@ -675,28 +675,38 @@
examples:
- "8123"
order: 1
tcp-port:
title: "Native Port"
description: "Native port (not the JDBC) of the database."
type: "integer"
minimum: 0
maximum: 65536
default: 9000
examples:
- "9000"
order: 2
database:
title: "DB Name"
description: "Name of the database."
type: "string"
order: 2
order: 3
username:
title: "User"
description: "Username to use to access the database."
type: "string"
order: 3
order: 4
password:
title: "Password"
description: "Password associated with the username."
type: "string"
airbyte_secret: true
order: 4
order: 5
ssl:
title: "SSL Connection"
description: "Encrypt data using SSL."
type: "boolean"
default: false
order: 5
order: 6
tunnel_method:
type: "object"
title: "SSH Tunnel Method"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,8 @@ def transform_clickhouse(config: Dict[str, Any]):
"user": config["username"],
"password": config["password"],
}
if "tcp-port" in config:
dbt_config["port"] = config["tcp-port"]
return dbt_config

@staticmethod
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ private JsonNode getObjectNode(final FieldList fields, final JsonNode root) {

@Override
public Schema getBigQuerySchema(final JsonNode jsonSchema) {
final List<Field> fieldList = getSchemaFields(namingResolver, jsonSchema);
final List<Field> fieldList = getSchemaFields(namingResolver, jsonSchema);
if (fieldList.stream().noneMatch(f -> f.getName().equals(JavaBaseConstants.COLUMN_NAME_AB_ID))) {
fieldList.add(Field.of(JavaBaseConstants.COLUMN_NAME_AB_ID, StandardSQLTypeName.STRING));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse

COPY --from=build /airbyte /airbyte

LABEL io.airbyte.version=0.1.1
LABEL io.airbyte.version=0.1.2
LABEL io.airbyte.name=airbyte/destination-clickhouse
Original file line number Diff line number Diff line change
Expand Up @@ -27,31 +27,41 @@
"examples": ["8123"],
"order": 1
},
"tcp-port": {
"title": "Native Port",
"description": "Native port (not the JDBC) of the database.",
"type": "integer",
"minimum": 0,
"maximum": 65536,
"default": 9000,
"examples": ["9000"],
"order": 2
},
"database": {
"title": "DB Name",
"description": "Name of the database.",
"type": "string",
"order": 2
"order": 3
},
"username": {
"title": "User",
"description": "Username to use to access the database.",
"type": "string",
"order": 3
"order": 4
},
"password": {
"title": "Password",
"description": "Password associated with the username.",
"type": "string",
"airbyte_secret": true,
"order": 4
"order": 5
},
"ssl": {
"title": "SSL Connection",
"description": "Encrypt data using SSL.",
"type": "boolean",
"default": false,
"order": 5
"order": 6
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -644,4 +644,4 @@ def request_params(
return params

def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]:
yield from response.json().get(self.data_field, {}).get("shippingLabels", [])
yield from response.json().get(self.data_field, {}).get("shippingLabels", [])
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
#

from http import HTTPStatus
from unittest.mock import patch

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
**incremental_args,
additional_fields=config.get("additional_fields", []),
expand_changelog=config.get("expand_issue_changelog", False),
render_fields=render_fields
render_fields=render_fields,
)
issue_fields_stream = IssueFields(**args)
experimental_streams = []
Expand Down
Loading