forked from delta-io/delta
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Kernel][Metrics][PR#3] Metrics report JSON serializer and LoggingMet…
…ricsReporter for the default engine (delta-io#3904) <!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://github.com/delta-io/delta/blob/master/CONTRIBUTING.md 2. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP] Your PR title ...'. 3. Be sure to keep the PR description updated to reflect all changes. 4. Please write your PR title to summarize what this PR proposes. 5. If possible, provide a concise example to reproduce the issue for a faster review. 6. If applicable, include the corresponding issue number in the PR title and link it in the body. --> #### Which Delta project/connector is this regarding? <!-- Please add the component selected below to the beginning of the pull request title For example: [Spark] Title of my pull request --> - [ ] Spark - [ ] Standalone - [ ] Flink - [X] Kernel - [ ] Other (fill in here) ## Description This PR is based off of delta-io#3903 See the diff for just this PR [here](https://github.com/delta-io/delta/pull/3904/files/aec95cf3dc0086c37f4c45e2b3e192b7b881768c..678ac473f4de65a8f7fd770696aad2d31a15aef7) Adds a JSON serializer for metrics reports with serialization logic for SnapshotReport. Also adds a `LoggingMetricsReporter` to the default implementation which simply logs the JSON serialized reports using Log4J. ## How was this patch tested? Adds a test suite. ## Does this PR introduce _any_ user-facing changes? No.
- Loading branch information
1 parent
dde5289
commit e1bdc33
Showing
11 changed files
with
293 additions
and
31 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
54 changes: 54 additions & 0 deletions
54
...l/kernel-api/src/main/java/io/delta/kernel/internal/metrics/MetricsReportSerializers.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
/* | ||
* Copyright (2024) The Delta Lake Project Authors. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
package io.delta.kernel.internal.metrics; | ||
|
||
import com.fasterxml.jackson.core.JsonProcessingException; | ||
import com.fasterxml.jackson.databind.ObjectMapper; | ||
import com.fasterxml.jackson.databind.module.SimpleModule; | ||
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; | ||
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; | ||
import io.delta.kernel.metrics.MetricsReport; | ||
import io.delta.kernel.metrics.SnapshotReport; | ||
|
||
/** Defines JSON serializers for {@link MetricsReport} types */ | ||
public final class MetricsReportSerializers { | ||
|
||
///////////////// | ||
// Public APIs // | ||
///////////////// | ||
|
||
/** | ||
* Serializes a {@link SnapshotReport} to a JSON string | ||
* | ||
* @throws JsonProcessingException | ||
*/ | ||
public static String serializeSnapshotReport(SnapshotReport snapshotReport) | ||
throws JsonProcessingException { | ||
return OBJECT_MAPPER.writeValueAsString(snapshotReport); | ||
} | ||
|
||
///////////////////////////////// | ||
// Private fields and methods // | ||
//////////////////////////////// | ||
|
||
private static final ObjectMapper OBJECT_MAPPER = | ||
new ObjectMapper() | ||
.registerModule(new Jdk8Module()) // To support Optional | ||
.registerModule( // Serialize Exception using toString() | ||
new SimpleModule().addSerializer(Exception.class, new ToStringSerializer())); | ||
|
||
private MetricsReportSerializers() {} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
35 changes: 35 additions & 0 deletions
35
kernel/kernel-api/src/main/java/io/delta/kernel/metrics/DeltaOperationReport.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
/* | ||
* Copyright (2024) The Delta Lake Project Authors. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
package io.delta.kernel.metrics; | ||
|
||
import java.util.Optional; | ||
import java.util.UUID; | ||
|
||
/** Defines the common fields that are shared by reports for Delta operations */ | ||
public interface DeltaOperationReport extends MetricsReport { | ||
|
||
/** @return the path of the table */ | ||
String getTablePath(); | ||
|
||
/** @return a string representation of the operation this report is for */ | ||
String getOperationType(); | ||
|
||
/** @return a unique ID for this report */ | ||
UUID getReportUUID(); | ||
|
||
/** @return the exception thrown if this report is for a failed operation, otherwise empty */ | ||
Optional<Exception> getException(); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
97 changes: 97 additions & 0 deletions
97
...el-api/src/test/scala/io/delta/kernel/internal/metrics/MetricsReportSerializerSuite.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,97 @@ | ||
/* | ||
* Copyright (2024) The Delta Lake Project Authors. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
package io.delta.kernel.internal.metrics | ||
|
||
import java.util.Optional | ||
|
||
import io.delta.kernel.metrics.SnapshotReport | ||
import org.scalatest.funsuite.AnyFunSuite | ||
|
||
class MetricsReportSerializerSuite extends AnyFunSuite { | ||
|
||
private def optionToString[T](option: Optional[T]): String = { | ||
if (option.isPresent) { | ||
if (option.get().isInstanceOf[String]) { | ||
s""""${option.get()}"""" // For string objects wrap with quotes | ||
} else { | ||
option.get().toString | ||
} | ||
} else { | ||
"null" | ||
} | ||
} | ||
|
||
private def testSnapshotReport(snapshotReport: SnapshotReport): Unit = { | ||
val timestampToVersionResolutionDuration = optionToString( | ||
snapshotReport.getSnapshotMetrics().getTimestampToVersionResolutionDurationNs()) | ||
val loadProtocolAndMetadataDuration = | ||
snapshotReport.getSnapshotMetrics().getLoadInitialDeltaActionsDurationNs() | ||
val exception: Optional[String] = snapshotReport.getException().map(_.toString) | ||
val expectedJson = | ||
s""" | ||
|{"tablePath":"${snapshotReport.getTablePath()}", | ||
|"operationType":"Snapshot", | ||
|"reportUUID":"${snapshotReport.getReportUUID()}", | ||
|"exception":${optionToString(exception)}, | ||
|"version":${optionToString(snapshotReport.getVersion())}, | ||
|"providedTimestamp":${optionToString(snapshotReport.getProvidedTimestamp())}, | ||
|"snapshotMetrics":{ | ||
|"timestampToVersionResolutionDurationNs":${timestampToVersionResolutionDuration}, | ||
|"loadInitialDeltaActionsDurationNs":${loadProtocolAndMetadataDuration} | ||
|} | ||
|} | ||
|""".stripMargin.replaceAll("\n", "") | ||
assert(expectedJson == MetricsReportSerializers.serializeSnapshotReport(snapshotReport)) | ||
} | ||
|
||
test("SnapshotReport serializer") { | ||
val snapshotContext1 = SnapshotQueryContext.forTimestampSnapshot("/table/path", 0) | ||
snapshotContext1.getSnapshotMetrics.timestampToVersionResolutionTimer.record(10) | ||
snapshotContext1.getSnapshotMetrics.loadInitialDeltaActionsTimer.record(1000) | ||
snapshotContext1.setVersion(1) | ||
val exception = new RuntimeException("something something failed") | ||
|
||
val snapshotReport1 = SnapshotReportImpl.forError( | ||
snapshotContext1, | ||
exception | ||
) | ||
|
||
// Manually check expected JSON | ||
val expectedJson = | ||
s""" | ||
|{"tablePath":"/table/path", | ||
|"operationType":"Snapshot", | ||
|"reportUUID":"${snapshotReport1.getReportUUID()}", | ||
|"exception":"$exception", | ||
|"version":1, | ||
|"providedTimestamp":0, | ||
|"snapshotMetrics":{ | ||
|"timestampToVersionResolutionDurationNs":10, | ||
|"loadInitialDeltaActionsDurationNs":1000 | ||
|} | ||
|} | ||
|""".stripMargin.replaceAll("\n", "") | ||
assert(expectedJson == MetricsReportSerializers.serializeSnapshotReport(snapshotReport1)) | ||
|
||
// Check with test function | ||
testSnapshotReport(snapshotReport1) | ||
|
||
// Empty options for all possible fields (version, providedTimestamp and exception) | ||
val snapshotContext2 = SnapshotQueryContext.forLatestSnapshot("/table/path") | ||
val snapshotReport2 = SnapshotReportImpl.forSuccess(snapshotContext2) | ||
testSnapshotReport(snapshotReport2) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
51 changes: 51 additions & 0 deletions
51
...kernel-defaults/src/main/java/io/delta/kernel/defaults/engine/LoggingMetricsReporter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
/* | ||
* Copyright (2024) The Delta Lake Project Authors. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
package io.delta.kernel.defaults.engine; | ||
|
||
import com.fasterxml.jackson.core.JsonProcessingException; | ||
import io.delta.kernel.engine.MetricsReporter; | ||
import io.delta.kernel.internal.metrics.MetricsReportSerializers; | ||
import io.delta.kernel.metrics.MetricsReport; | ||
import io.delta.kernel.metrics.SnapshotReport; | ||
import org.slf4j.Logger; | ||
import org.slf4j.LoggerFactory; | ||
|
||
/** | ||
* An implementation of {@link MetricsReporter} that logs the reports (as JSON) to Log4J at the info | ||
* level. | ||
*/ | ||
public class LoggingMetricsReporter implements MetricsReporter { | ||
|
||
private static final Logger logger = LoggerFactory.getLogger(LoggingMetricsReporter.class); | ||
|
||
@Override | ||
public void report(MetricsReport report) { | ||
try { | ||
if (report instanceof SnapshotReport) { | ||
logger.info( | ||
"SnapshotReport = {}", | ||
MetricsReportSerializers.serializeSnapshotReport((SnapshotReport) report)); | ||
} else { | ||
logger.info( | ||
"{} = [{} does not support serializing this type of MetricReport]", | ||
report.getClass(), | ||
this.getClass()); | ||
} | ||
} catch (JsonProcessingException e) { | ||
logger.info("Encountered exception while serializing report {}: {}", report, e); | ||
} | ||
} | ||
} |
Oops, something went wrong.