From 8fc5dcbf260323ceab855ed6507a97f9d056be15 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Thu, 29 Sep 2022 09:39:18 -0400 Subject: [PATCH] Source Fauna: add seed file information (#17355) * Add fauna source * Update changelog to include the correct PR * Improve docs (#1) * Applied suggestions to improve docs (#2) * Applied suggestions to improve docs * Cleaned up the docs * Apply suggestions from code review Co-authored-by: Ewan Edwards <46354154+faunaee@users.noreply.github.com> * Update airbyte-integrations/connectors/source-fauna/source_fauna/spec.yaml Co-authored-by: Ewan Edwards <46354154+faunaee@users.noreply.github.com> Co-authored-by: Ewan Edwards <46354154+faunaee@users.noreply.github.com> * Flake Checker (#3) * Run ./gradlew :airbyte-integrations:connectors:source-fauna:flakeCheck * Fix all the warnings * Set additionalProperties to true to adhere to acceptance tests * Remove custom fields (#4) * Remove custom fields from source.py * Remove custom fields from spec.yaml * Collections that support incremental sync are found correctly * Run formatter * Index values and termins are verified * Stripped additional_columns from collection config and check() * We now search for an index at the start of each sync * Add default for missing data in collection * Add a log message about the index chosen to sync an incremental stream * Add an example for a configured incremental catalog * Check test now validates the simplified check function * Remove collection name from spec.yaml and CollectionConfig * Update test_util.py to ahere to the new config * Update the first discover test to validate that we can find indexes correctly * Remove other discover tests, as they no longer apply * Full refresh test now works with simplified expanded columns * Remove unused imports * Incremental test now adheres to the find_index_for_stream system * Database test passes, so now all unit tests pass again * Remove extra fields from required section * ttl is nullable * Data defaults to an empty object * Update tests to reflect ttl and data select changes * Fix expected records. All unit tests and acceptance tests pass * Cleanup docs for find_index_for_stream * Update setup guide to reflect multiple collections * Add docs to install the fauna shell * Update examples and README to conform to the removal of additional columns * add seed file * auto-bump connector version [ci skip] * add icon in seed Co-authored-by: Neil Macneale Co-authored-by: Ewan Edwards <46354154+faunaee@users.noreply.github.com> Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 8 ++ .../src/main/resources/seed/source_specs.yaml | 97 +++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index e0c4b2db746d..d5f8aca7fd26 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -299,6 +299,14 @@ documentationUrl: https://docs.airbyte.com/integrations/sources/faker sourceType: api releaseStage: alpha +- name: Fauna + sourceDefinitionId: 3825db3e-c94b-42ac-bd53-b5a9507ace2b + dockerRepository: airbyte/source-fauna + dockerImageTag: dev + documentationUrl: https://docs.airbyte.com/integrations/sources/fauna + icon: fauna.svg + sourceType: database + releaseStage: alpha - name: File sourceDefinitionId: 778daa7c-feaf-4db6-96f3-70fd645acc77 dockerRepository: airbyte/source-file diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 200b1961abf6..c60059825a85 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2995,6 +2995,103 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-fauna:dev" + spec: + documentationUrl: "https://github.com/fauna/airbyte/blob/source-fauna/docs/integrations/sources/fauna.md" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Fauna Spec" + type: "object" + required: + - "domain" + - "port" + - "scheme" + - "secret" + additionalProperties: true + properties: + domain: + order: 0 + type: "string" + title: "Domain" + description: "Domain of Fauna to query. Defaults db.fauna.com. See the\ + \ docs." + default: "db.fauna.com" + port: + order: 1 + type: "integer" + title: "Port" + description: "Endpoint port." + default: 443 + scheme: + order: 2 + type: "string" + title: "Scheme" + description: "URL scheme." + default: "https" + secret: + order: 3 + type: "string" + title: "Fauna Secret" + description: "Fauna secret, used when authenticating with the database." + airbyte_secret: true + collection: + order: 5 + type: "object" + title: "Collection" + description: "Settings for the Fauna Collection." + required: + - "page_size" + - "deletions" + properties: + page_size: + order: 4 + type: "integer" + title: "Page Size" + default: 64 + description: "The page size used when reading documents from the database.\ + \ The larger the page size, the faster the connector processes documents.\ + \ However, if a page is too large, the connector may fail.
\n\ + Choose your page size based on how large the documents are.
\n\ + See the docs." + deletions: + order: 5 + type: "object" + title: "Deletion Mode" + description: "This only applies to incremental syncs.
\n\ + Enabling deletion mode informs your destination of deleted documents.
\n\ + Disabled - Leave this feature disabled, and ignore deleted documents.
\n\ + Enabled - Enables this feature. When a document is deleted, the connector\ + \ exports a record with a \"deleted at\" column containing the time\ + \ that the document was deleted." + oneOf: + - title: "Disabled" + type: "object" + order: 0 + required: + - "deletion_mode" + properties: + deletion_mode: + type: "string" + const: "ignore" + - title: "Enabled" + type: "object" + order: 1 + required: + - "deletion_mode" + - "column" + properties: + deletion_mode: + type: "string" + const: "deleted_field" + column: + type: "string" + title: "Deleted At Column" + description: "Name of the \"deleted at\" column." + default: "deleted_at" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-file:0.2.23" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/file"