Add field selection info to the StandardSync persistence (#20212)

* database migration to add column for field selection info * add field selection info to standard sync persistence * fix around persistence of field selection info
airbytehq · Dec 8, 2022 · 559cb63 · 559cb63
1 parent 315c152
commit 559cb63
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 0 deletions.
diff --git a/airbyte-config/config-models/src/main/resources/types/StandardSync.yaml b/airbyte-config/config-models/src/main/resources/types/StandardSync.yaml
@@ -42,6 +42,11 @@ properties:
     type: string
   catalog:
     existingJavaType: io.airbyte.protocol.models.ConfiguredAirbyteCatalog
+  fieldSelectionData:
+    type: object
+    description: A map of StreamDescriptor to an indicator of whether field selection is enabled for that stream.
+    additionalProperties:
+      type: boolean
   status:
     type: string
     enum:

diff --git a/...te-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java b/...te-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java
@@ -20,6 +20,7 @@
 import io.airbyte.config.ActorDefinitionResourceRequirements;
 import io.airbyte.config.DestinationConnection;
 import io.airbyte.config.DestinationOAuthParameter;
+import io.airbyte.config.FieldSelectionData;
 import io.airbyte.config.Geography;
 import io.airbyte.config.JobSyncConfig.NamespaceDefinitionType;
 import io.airbyte.config.Notification;
@@ -65,6 +66,8 @@ public static StandardSync buildStandardSync(final Record record, final List<UUI
         .withName(record.get(CONNECTION.NAME))
         .withCatalog(
             Jsons.deserialize(record.get(CONNECTION.CATALOG).data(), ConfiguredAirbyteCatalog.class))
+        .withFieldSelectionData(record.get(CONNECTION.FIELD_SELECTION_DATA) == null ? null
+            : Jsons.deserialize(record.get(CONNECTION.FIELD_SELECTION_DATA).data(), FieldSelectionData.class))
         .withStatus(
             record.get(CONNECTION.STATUS) == null ? null
                 : Enums.toEnum(record.get(CONNECTION.STATUS, String.class), Status.class).orElseThrow())

diff --git a/...nfig-persistence/src/main/java/io/airbyte/config/persistence/StandardSyncPersistence.java b/...nfig-persistence/src/main/java/io/airbyte/config/persistence/StandardSyncPersistence.java
@@ -187,6 +187,7 @@ private void writeStandardSync(final StandardSync standardSync, final DSLContext
           .set(CONNECTION.DESTINATION_ID, standardSync.getDestinationId())
           .set(CONNECTION.NAME, standardSync.getName())
           .set(CONNECTION.CATALOG, JSONB.valueOf(Jsons.serialize(standardSync.getCatalog())))
+          .set(CONNECTION.FIELD_SELECTION_DATA, JSONB.valueOf(Jsons.serialize(standardSync.getFieldSelectionData())))
           .set(CONNECTION.STATUS, standardSync.getStatus() == null ? null
               : Enums.toEnum(standardSync.getStatus().value(),
                   io.airbyte.db.instance.configs.jooq.generated.enums.StatusType.class).orElseThrow())
@@ -233,6 +234,7 @@ private void writeStandardSync(final StandardSync standardSync, final DSLContext
           .set(CONNECTION.DESTINATION_ID, standardSync.getDestinationId())
           .set(CONNECTION.NAME, standardSync.getName())
           .set(CONNECTION.CATALOG, JSONB.valueOf(Jsons.serialize(standardSync.getCatalog())))
+          .set(CONNECTION.FIELD_SELECTION_DATA, JSONB.valueOf(Jsons.serialize(standardSync.getFieldSelectionData())))
           .set(CONNECTION.STATUS, standardSync.getStatus() == null ? null
               : Enums.toEnum(standardSync.getStatus().value(),
                   io.airbyte.db.instance.configs.jooq.generated.enums.StatusType.class).orElseThrow())

diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java
@@ -12,6 +12,7 @@
 import io.airbyte.config.ActorDefinitionResourceRequirements;
 import io.airbyte.config.DestinationConnection;
 import io.airbyte.config.DestinationOAuthParameter;
+import io.airbyte.config.FieldSelectionData;
 import io.airbyte.config.Geography;
 import io.airbyte.config.JobSyncConfig.NamespaceDefinitionType;
 import io.airbyte.config.Notification;
@@ -471,6 +472,7 @@ public static List<StandardSync> standardSyncs() {
         .withSourceId(SOURCE_ID_1)
         .withDestinationId(DESTINATION_ID_1)
         .withCatalog(getConfiguredCatalog())
+        .withFieldSelectionData(new FieldSelectionData().withAdditionalProperty("foo", true))
         .withName("standard-sync-1")
         .withManual(true)
         .withNamespaceDefinition(NamespaceDefinitionType.CUSTOMFORMAT)