Remove logging of unhandled debezium events (#45368)

airbytehq · Sep 10, 2024 · 41a242b · 41a242b
1 parent e4c26e8
commit 41a242b
Show file tree

Hide file tree

Showing 6 changed files with 14 additions and 7 deletions.
diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md
@@ -173,7 +173,8 @@ corresponds to that version.
 ### Java CDK
 
 | Version    | Date       | Pull Request                                                 | Subject                                                                                                                                                        |
-| :--------- | :--------- | :----------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+|:-----------|:-----------| :----------------------------------------------------------- |:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.44.22    | 2024-09-10 | [\#45368](https://github.com/airbytehq/airbyte/pull/45368)   | Remove excessive debezium logging                                                                                                                              |
 | 0.44.21    | 2024-09-04 | [\#45143](https://github.com/airbytehq/airbyte/pull/45143)  | S3-destination: don't overwrite existing files, skip those file indexes instead                                                                                |
 | 0.44.20    | 2024-08-30 | [\#44933](https://github.com/airbytehq/airbyte/pull/44933)   | Avro/Parquet destinations: handle `{}` schemas inside objects/arrays                                                                                           |
 | 0.44.19    | 2024-08-20 | [\#44476](https://github.com/airbytehq/airbyte/pull/44476)   | Increase Jackson message length limit to 100mb                                                                                                                 |

diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties
@@ -1 +1 @@
-version=0.44.21
+version=0.44.22
diff --git a/.../src/main/kotlin/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordIterator.kt b/.../src/main/kotlin/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordIterator.kt
@@ -174,7 +174,6 @@ class DebeziumRecordIterator<T>(
 
             // #41647: discard event type with op code 'm'
             if (!isEventTypeHandled(changeEventWithMetadata)) {
-                LOGGER.info { "WAL event type not handled: $next" }
                 continue
             }
 
@@ -332,7 +331,7 @@ class DebeziumRecordIterator<T>(
     companion object {
         val pollLogMaxTimeInterval: Duration = Duration.ofSeconds(5)
         const val POLL_LOG_MAX_CALLS_INTERVAL = 1_000
-
+        private val unhandledFoundTypeList: MutableList<String> = mutableListOf()
         /**
          * We are not interested in message events. According to debezium
          * [documentation](https://debezium.io/documentation/reference/stable/connectors/postgresql.html#postgresql-create-events)
@@ -341,7 +340,13 @@ class DebeziumRecordIterator<T>(
          */
         fun isEventTypeHandled(event: ChangeEventWithMetadata): Boolean {
             event.eventValueAsJson?.get("op")?.asText()?.let {
-                return it in listOf("c", "u", "d", "t")
+                val handled = it in listOf("c", "u", "d", "t")
+                if (!handled && !unhandledFoundTypeList.contains(it)) {
+                    unhandledFoundTypeList.add(it)
+                    LOGGER.info { "WAL event type not handled: $it" }
+                    LOGGER.debug { "event: $event" }
+                }
+                return handled
             }
                 ?: return false
         }

diff --git a/airbyte-integrations/connectors/source-mssql/build.gradle b/airbyte-integrations/connectors/source-mssql/build.gradle
@@ -3,7 +3,7 @@ plugins {
 }
 
 airbyteJavaConnector {
-    cdkVersionRequired = '0.44.18'
+    cdkVersionRequired = '0.44.22'
     features = ['db-sources']
     useLocalCdk = false
 }

diff --git a/airbyte-integrations/connectors/source-mssql/metadata.yaml b/airbyte-integrations/connectors/source-mssql/metadata.yaml
@@ -9,7 +9,7 @@ data:
   connectorSubtype: database
   connectorType: source
   definitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1
-  dockerImageTag: 4.1.11
+  dockerImageTag: 4.1.12
   dockerRepository: airbyte/source-mssql
   documentationUrl: https://docs.airbyte.com/integrations/sources/mssql
   githubIssueLabel: source-mssql

diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md
@@ -422,6 +422,7 @@ WHERE actor_definition_id ='b5ea17b1-f170-46dc-bc31-cc744ca984c1' AND (configura
 
 | Version | Date       | Pull Request                                                                                                      | Subject                                                                                                                                         |
 |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------|
+| 4.1.12  | 2024-09-10 | [45368](https://github.com/airbytehq/airbyte/pull/45368)                                                          | Remove excessive debezium logging.                                                                                                              |
 | 4.1.11  | 2024-09-04 | [45142](https://github.com/airbytehq/airbyte/pull/45142)                                                          | Fix incorrect datetimeoffset format in cursor state.                                                                                            |
 | 4.1.10  | 2024-08-27 | [44759](https://github.com/airbytehq/airbyte/pull/44759)                                                          | Improve null safety in parsing debezium change events.                                                                                          |
 | 4.1.9   | 2024-08-27 | [44841](https://github.com/airbytehq/airbyte/pull/44841)                                                          | Adopt latest CDK.                                                                                                                               |