write logs to file in addition to stdout when running java connector …

…tests (#35236)
airbytehq · Feb 14, 2024 · ccafbe6 · ccafbe6
1 parent 308c4ed
commit ccafbe6
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 9 deletions.
diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md
@@ -166,6 +166,7 @@ MavenLocal debugging steps:
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                        |
 |:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.20.7  | 2024-02-13 | [\#35236](https://github.com/airbytehq/airbyte/pull/35236) | output logs to files in addition to stdout when running tests                                                                                                  |
 | 0.20.6  | 2024-02-12 | [\#35036](https://github.com/airbytehq/airbyte/pull/35036) | Add trace utility to emit analytics messages.                                                                                                                  |
 | 0.20.5  | 2024-02-13 | [\#34869](https://github.com/airbytehq/airbyte/pull/34869) | Don't emit final state in SourceStateIterator there is an underlying stream failure.                                                                           |
 | 0.20.4  | 2024-02-12 | [\#35042](https://github.com/airbytehq/airbyte/pull/35042) | Use delegate's isDestinationV2 invocation in SshWrappedDestination.                                                                                            |

diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties
@@ -1 +1 @@
-version=0.20.6
+version=0.20.7
diff --git a/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/log4j2-test.xml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/log4j2-test.xml
@@ -2,33 +2,64 @@
 <Configuration status="INFO">
     <Properties>
         <!-- Set the LOG_SCRUB_PATTERN env var to a regex pattern to scrub the log messages of secrets or any other unwanted content. -->
-        <Property name="default-pattern">%d{yyyy-MM-dd HH:mm:ss}{GMT+0} %highlight{%p} %C{1.}(%M):%L - %replace{%m}{$${env:LOG_SCRUB_PATTERN:-\*\*\*\*\*}}{*****}%n</Property>
+        <Property name="jvm-log-pattern">%d{yyyy-MM-dd'T'HH:mm:ss,SSS}{GMT+0}`%highlight{%p}`%C{1.}(%M):%L - %replace{%m}{$${env:LOG_SCRUB_PATTERN:-\*\*\*\*\*}}{*****}%n</Property>
         <!--Logs the timestamp and log_source/application name in the beginning of the line if it exists with a > separator, and then always the rest of the line.-->
-        <Property name="simple-pattern">%d{yyyy-MM-dd HH:mm:ss}{GMT+0}%replace{ %X{log_source}}{^ -}{} > %replace{%m}{$${env:LOG_SCRUB_PATTERN:-\*\*\*\*\*}}{*****}%n</Property>
+        <Property name="container-log-pattern">%d{yyyy-MM-dd'T'HH:mm:ss,SSS}{GMT+0}`%replace{ %X{log_source}}{^ -}{} > %replace{%m}{$${env:LOG_SCRUB_PATTERN:-\*\*\*\*\*}}{*****}%n</Property>
         <!-- Always log INFO by default. -->
         <Property name="log-level">${sys:LOG_LEVEL:-${env:LOG_LEVEL:-INFO}}</Property>
+        <Property name="logSubDir">${env:AIRBYTE_LOG_SUBDIR:-${date:yyyy-MM-dd'T'HH:mm:ss}}</Property>
+        <Property name="logDir">build/test-logs/${logSubDir}</Property>
     </Properties>
 
     <Appenders>
-        <Console name="Default" target="SYSTEM_OUT">
+        <Console name="JvmLogsStdOut" target="SYSTEM_OUT">
             <ContextMapFilter onMatch="DENY" onMismatch="ACCEPT">
                 <KeyValuePair key="simple" value="true"/>
             </ContextMapFilter>
-            <PatternLayout pattern="${default-pattern}"/>
+            <PatternLayout pattern="${jvm-log-pattern}"/>
         </Console>
 
-        <Console name="SimpleDefault" target="SYSTEM_OUT">
+        <Console name="ContainerLogsStdOut" target="SYSTEM_OUT">
             <ContextMapFilter onMatch="ACCEPT" onMismatch="DENY">
                 <KeyValuePair key="simple" value="true"/>
             </ContextMapFilter>
-            <PatternLayout pattern="${simple-pattern}"/>
+            <PatternLayout pattern="${container-log-pattern}"/>
         </Console>
+
+        <File name="JvmLogsFile" fileName="${logDir}/airbyte_jvm.log">
+            <ContextMapFilter onMatch="DENY" onMismatch="ACCEPT">
+                <KeyValuePair key="simple" value="true"/>
+            </ContextMapFilter>
+            <PatternLayout pattern="${jvm-log-pattern}"/>
+        </File>
+        <File name="ContainerLogFiles" fileName="${logDir}/airbyte_containers.log">
+            <ContextMapFilter onMatch="ACCEPT" onMismatch="DENY">
+                <KeyValuePair key="simple" value="true"/>
+            </ContextMapFilter>
+            <PatternLayout pattern="${container-log-pattern}"/>
+        </File>
+        <File name="UnifiedFile-JvmLogs" fileName="${logDir}/airbyte_unified.log">
+            <ContextMapFilter onMatch="DENY" onMismatch="ACCEPT">
+                <KeyValuePair key="simple" value="true"/>
+            </ContextMapFilter>
+            <PatternLayout pattern="${jvm-log-pattern}"/>
+        </File>
+        <File name="UnifiedFile-ContainerLogs" fileName="${logDir}/airbyte_unified.log">
+            <ContextMapFilter onMatch="ACCEPT" onMismatch="DENY">
+                <KeyValuePair key="simple" value="true"/>
+            </ContextMapFilter>
+            <PatternLayout pattern="${container-log-pattern}"/>
+        </File>
     </Appenders>
 
     <Loggers>
         <Root level="${log-level}">
-            <AppenderRef ref="Default"/>
-            <AppenderRef ref="SimpleDefault"/>
+            <AppenderRef ref="JvmLogsStdOut"/>
+            <AppenderRef ref="ContainerLogsStdOut"/>
+            <AppenderRef ref="JvmLogsFile"/>
+            <AppenderRef ref="ContainerLogFiles"/>
+            <AppenderRef ref="UnifiedFile-JvmLogs"/>
+            <AppenderRef ref="UnifiedFile-ContainerLogs"/>
         </Root>
 
         <Logger name="org.eclipse.jetty" level="INFO" />