Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🐛 Add method field on spec.json connectors (snowflake and postgres) #3960

Merged
merged 4 commits into from
Jun 8, 2021
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"destinationDefinitionId": "424892c4-daac-4491-b35d-c6688ba547ba",
"name": "Snowflake",
"dockerRepository": "airbyte/destination-snowflake",
"dockerImageTag": "0.3.7",
"dockerImageTag": "0.3.8",
"documentationUrl": "https://docs.airbyte.io/integrations/destinations/snowflake"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"sourceDefinitionId": "decd338e-5647-4c0b-adf4-da0e75f5a750",
"name": "Postgres",
"dockerRepository": "airbyte/source-postgres",
"dockerImageTag": "0.3.2",
"dockerImageTag": "0.3.3",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-postgres",
"icon": "postgresql.svg"
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
- destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba
name: Snowflake
dockerRepository: airbyte/destination-snowflake
dockerImageTag: 0.3.7
dockerImageTag: 0.3.8
documentationUrl: https://docs.airbyte.io/integrations/destinations/snowflake
- destinationDefinitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362
name: S3
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
- sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
name: Postgres
dockerRepository: airbyte/source-postgres
dockerImageTag: 0.3.2
dockerImageTag: 0.3.3
documentationUrl: https://hub.docker.com/r/airbyte/source-postgres
icon: postgresql.svg
- sourceDefinitionId: cd42861b-01fc-4658-a8ab-5d11d0510f01
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,25 +79,38 @@
"title": "Standard Inserts",
"additionalProperties": false,
"description": "Uses <pre>INSERT</pre> statements to send batches of records to Snowflake. Easiest (no setup) but not recommended for large production workloads due to slow speed.",
"required": [],
"properties": {}
"required": ["method"],
"properties": {
"method": {
"type": "string",
"enum": ["Standard"],
"default": "Standard"
}
}
},
{
"title": "AWS S3 Staging",
"additionalProperties": false,
"description": "Writes large batches of records to a file, uploads the file to S3, then uses <pre>COPY INTO table</pre> to upload the file. Recommended for large production workloads for better speed and scalability.",
"required": [
"method",
"s3_bucket_name",
"access_key_id",
"secret_access_key"
],
"properties": {
"method": {
"type": "string",
"enum": ["S3 Staging"],
"default": "S3 Staging",
"order": 0
},
"s3_bucket_name": {
"title": "S3 Bucket Name",
"type": "string",
"description": "The name of the staging S3 bucket. Airbyte will write files to this bucket and read them via <pre>COPY</pre> statements on Snowflake.",
"examples": ["airbyte.staging"],
"order": 0
"order": 1
},
"s3_bucket_region": {
"title": "S3 Bucket Region",
Expand All @@ -121,6 +134,7 @@
"ca-central-1",
"cn-north-1",
"cn-northwest-1",
"eu-central-1",
"eu-west-1",
"eu-west-2",
"eu-west-3",
Expand All @@ -129,43 +143,49 @@
"sa-east-1",
"me-south-1"
],
"order": 1
"order": 2
},
"access_key_id": {
"type": "string",
"description": "The Access Key Id granting allow one to access the above S3 staging bucket. Airbyte requires Read and Write permissions to the given bucket.",
"title": "S3 Key Id",
"airbyte_secret": true,
"order": 2
"order": 3
},
"secret_access_key": {
"type": "string",
"description": "The corresponding secret to the above access key id.",
"title": "S3 Access Key",
"airbyte_secret": true,
"order": 3
"order": 4
}
}
},
{
"title": "GCS Staging",
"additionalProperties": false,
"description": "Writes large batches of records to a file, uploads the file to GCS, then uses <pre>COPY INTO table</pre> to upload the file. Recommended for large production workloads for better speed and scalability.",
"required": ["project_id", "bucket_name", "credentials_json"],
"required": ["method","project_id", "bucket_name", "credentials_json"],
"properties": {
"method": {
"type": "string",
"enum": ["GCS Staging"],
"default": "GCS Staging",
"order": 0
},
"project_id": {
"title": "GCP Project ID",
"type": "string",
"description": "The name of the GCP project ID for your credentials.",
"examples": ["my-project"],
"order": 0
"order": 1
},
"bucket_name": {
"title": "GCS Bucket Name",
"type": "string",
"description": "The name of the staging GCS bucket. Airbyte will write files to this bucket and read them via <pre>COPY</pre> statements on Snowflake.",
"examples": ["airbyte-staging"],
"order": 1
"order": 2
},
"credentials_json": {
"title": "Google Application Credentials",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,22 +59,37 @@
"title": "Standard",
"additionalProperties": false,
"description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.",
"required": [],
"properties": {}
"required": ["method"],
"properties": {
"method": {
"type": "string",
"enum": ["Standard"],
"default": "Standard",
"order": 0
}
}
},
{
"title": "Logical Replication (CDC)",
"additionalProperties": false,
"description": "Logical replication uses the Postgres write-ahead log (WAL) to detect inserts, updates, and deletes. This needs to be configured on the source database itself. Only available on Postgres 10 and above. Read the <a href=\"https://docs.airbyte.io/integrations/sources/postgres\">Postgres Source</a> docs for more information.",
"required": ["replication_slot", "publication"],
"required": ["method", "replication_slot", "publication"],
"properties": {
"method": {
"type": "string",
"enum": ["CDC"],
"default": "CDC",
"order": 0
},
"replication_slot": {
"type": "string",
"description": "A pgoutput logical replication slot."
"description": "A pgoutput logical replication slot.",
"order": 1
},
"publication": {
"type": "string",
"description": "A Postgres publication used for consuming changes."
"description": "A Postgres publication used for consuming changes.",
"order": 2
}
}
}
Expand Down