[DB sources] : Reduce CDC state compression limit to 1MB (airbytehq#3…

…5511)
ollionorg · Feb 26, 2024 · 132412f · 132412f
1 parent 066c160
commit 132412f
Show file tree

Hide file tree

Showing 6 changed files with 9 additions and 8 deletions.
diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md
@@ -166,6 +166,7 @@ MavenLocal debugging steps:
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                        |
 |:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.21.4  | 2024-02-21 | [\#35511](https://github.com/airbytehq/airbyte/pull/35511) | Add Junit progress information to the test logs                                                                                                                                  |
 | 0.21.3  | 2024-02-20 | [\#35394](https://github.com/airbytehq/airbyte/pull/35394) | Add Junit progress information to the test logs                                                                                                                                  |
 | 0.21.2  | 2024-02-20 | [\#34978](https://github.com/airbytehq/airbyte/pull/34978) | Reduce log noise in NormalizationLogParser.                                                                                                                    |
 | 0.21.1  | 2024-02-20 | [\#35199](https://github.com/airbytehq/airbyte/pull/35199) | Add thread names to the logs.                                                                                                                                  |

diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties
@@ -1 +1 @@
-version=0.21.3
+version=0.21.4
diff --git a/...main/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteSchemaHistoryStorage.java b/...main/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteSchemaHistoryStorage.java
@@ -39,7 +39,7 @@
 public class AirbyteSchemaHistoryStorage {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(AirbyteSchemaHistoryStorage.class);
-  private static final long SIZE_LIMIT_TO_COMPRESS_MB = 3;
+  private static final long SIZE_LIMIT_TO_COMPRESS_MB = 1;
   public static final int ONE_MB = 1024 * 1024;
   private static final Charset UTF8 = StandardCharsets.UTF_8;
 

diff --git a/.../java/io/airbyte/cdk/integrations/debezium/internals/AirbyteSchemaHistoryStorageTest.java b/.../java/io/airbyte/cdk/integrations/debezium/internals/AirbyteSchemaHistoryStorageTest.java
@@ -19,8 +19,8 @@
 public class AirbyteSchemaHistoryStorageTest {
 
   @Test
-  public void testForContentBiggerThan3MBLimit() throws IOException {
-    final String contentReadDirectlyFromFile = MoreResources.readResource("dbhistory_greater_than_3_mb.dat");
+  public void testForContentBiggerThan1MBLimit() throws IOException {
+    final String contentReadDirectlyFromFile = MoreResources.readResource("dbhistory_greater_than_1_mb.dat");
 
     final AirbyteSchemaHistoryStorage schemaHistoryStorageFromUncompressedContent = AirbyteSchemaHistoryStorage.initializeDBHistory(
         new SchemaHistory<>(Optional.of(Jsons.jsonNode(contentReadDirectlyFromFile)),
@@ -46,14 +46,14 @@ public void testForContentBiggerThan3MBLimit() throws IOException {
   @Test
   public void sizeTest() throws IOException {
     assertEquals(5.881045341491699,
-        AirbyteSchemaHistoryStorage.calculateSizeOfStringInMB(MoreResources.readResource("dbhistory_greater_than_3_mb.dat")));
+        AirbyteSchemaHistoryStorage.calculateSizeOfStringInMB(MoreResources.readResource("dbhistory_greater_than_1_mb.dat")));
     assertEquals(0.0038671493530273438,
-        AirbyteSchemaHistoryStorage.calculateSizeOfStringInMB(MoreResources.readResource("dbhistory_less_than_3_mb.dat")));
+        AirbyteSchemaHistoryStorage.calculateSizeOfStringInMB(MoreResources.readResource("dbhistory_less_than_1_mb.dat")));
   }
 
   @Test
-  public void testForContentLessThan3MBLimit() throws IOException {
-    final String contentReadDirectlyFromFile = MoreResources.readResource("dbhistory_less_than_3_mb.dat");
+  public void testForContentLessThan1MBLimit() throws IOException {
+    final String contentReadDirectlyFromFile = MoreResources.readResource("dbhistory_less_than_1_mb.dat");
 
     final AirbyteSchemaHistoryStorage schemaHistoryStorageFromUncompressedContent = AirbyteSchemaHistoryStorage.initializeDBHistory(
         new SchemaHistory<>(Optional.of(Jsons.jsonNode(contentReadDirectlyFromFile)),

diff --git a/...resources/dbhistory_greater_than_3_mb.dat → ...resources/dbhistory_greater_than_1_mb.dat b/...resources/dbhistory_greater_than_3_mb.dat → ...resources/dbhistory_greater_than_1_mb.dat
diff --git a/...st/resources/dbhistory_less_than_3_mb.dat → ...st/resources/dbhistory_less_than_1_mb.dat b/...st/resources/dbhistory_less_than_3_mb.dat → ...st/resources/dbhistory_less_than_1_mb.dat