From 4d07a92bfd44089d1f355b0765bfa53709119654 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 9 Oct 2022 00:05:43 +0800 Subject: [PATCH 1/2] fix --- .../spark/sql/avro/SparkAvroKeyOutputFormat.java | 5 +++-- .../java/org/apache/spark/SparkThrowable.java | 2 +- .../SupportsAtomicPartitionManagement.java | 1 + .../connector/util/V2ExpressionSQLBuilder.java | 4 ++-- .../apache/spark/sql/util/NumericHistogram.java | 15 ++++++++------- .../service/cli/operation/LogDivertAppender.java | 3 ++- .../service/cli/operation/OperationManager.java | 4 ++-- 7 files changed, 19 insertions(+), 15 deletions(-) diff --git a/connector/avro/src/main/java/org/apache/spark/sql/avro/SparkAvroKeyOutputFormat.java b/connector/avro/src/main/java/org/apache/spark/sql/avro/SparkAvroKeyOutputFormat.java index df5d6d73f2f14..b2a57060fc2d9 100644 --- a/connector/avro/src/main/java/org/apache/spark/sql/avro/SparkAvroKeyOutputFormat.java +++ b/connector/avro/src/main/java/org/apache/spark/sql/avro/SparkAvroKeyOutputFormat.java @@ -25,6 +25,7 @@ import org.apache.avro.file.CodecFactory; import org.apache.avro.file.DataFileWriter; import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.generic.GenericRecord; import org.apache.avro.mapred.AvroKey; import org.apache.avro.mapreduce.AvroKeyOutputFormat; @@ -53,7 +54,7 @@ protected RecordWriter, NullWritable> create( CodecFactory compressionCodec, OutputStream outputStream, int syncInterval) throws IOException { - return new SparkAvroKeyRecordWriter( + return new SparkAvroKeyRecordWriter<>( writerSchema, dataModel, compressionCodec, outputStream, syncInterval, metadata); } } @@ -72,7 +73,7 @@ class SparkAvroKeyRecordWriter extends RecordWriter, NullWritable> OutputStream outputStream, int syncInterval, Map metadata) throws IOException { - this.mAvroFileWriter = new DataFileWriter(dataModel.createDatumWriter(writerSchema)); + this.mAvroFileWriter = new DataFileWriter<>(new GenericDatumWriter<>(writerSchema, dataModel)); for (Map.Entry entry : metadata.entrySet()) { this.mAvroFileWriter.setMeta(entry.getKey(), entry.getValue()); } diff --git a/core/src/main/java/org/apache/spark/SparkThrowable.java b/core/src/main/java/org/apache/spark/SparkThrowable.java index 7fb693d9c5569..e1235b2982ba0 100644 --- a/core/src/main/java/org/apache/spark/SparkThrowable.java +++ b/core/src/main/java/org/apache/spark/SparkThrowable.java @@ -51,7 +51,7 @@ default boolean isInternalError() { } default Map getMessageParameters() { - return new HashMap(); + return new HashMap<>(); } default QueryContext[] getQueryContext() { return new QueryContext[0]; } diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsAtomicPartitionManagement.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsAtomicPartitionManagement.java index e2c693f2d0a92..0c5e288d5bdd9 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsAtomicPartitionManagement.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsAtomicPartitionManagement.java @@ -46,6 +46,7 @@ @Experimental public interface SupportsAtomicPartitionManagement extends SupportsPartitionManagement { + @SuppressWarnings("unchecked") @Override default void createPartition( InternalRow ident, diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/util/V2ExpressionSQLBuilder.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/util/V2ExpressionSQLBuilder.java index b32958d13daf1..fe16174586bad 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/util/V2ExpressionSQLBuilder.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/util/V2ExpressionSQLBuilder.java @@ -47,7 +47,7 @@ public class V2ExpressionSQLBuilder { public String build(Expression expr) { if (expr instanceof Literal) { - return visitLiteral((Literal) expr); + return visitLiteral((Literal) expr); } else if (expr instanceof NamedReference) { return visitNamedReference((NamedReference) expr); } else if (expr instanceof Cast) { @@ -213,7 +213,7 @@ public String build(Expression expr) { } } - protected String visitLiteral(Literal literal) { + protected String visitLiteral(Literal literal) { return literal.toString(); } diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/util/NumericHistogram.java b/sql/catalyst/src/main/java/org/apache/spark/sql/util/NumericHistogram.java index 444263f31113e..283258ecb0a55 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/util/NumericHistogram.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/util/NumericHistogram.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Random; @@ -53,20 +54,20 @@ public class NumericHistogram { * * @since 3.3.0 */ - public static class Coord implements Comparable { + public static class Coord implements Comparable { public double x; public double y; @Override - public int compareTo(Object other) { - return Double.compare(x, ((Coord) other).x); + public int compareTo(Coord other) { + return Double.compare(x, other.x); } } // Class variables private int nbins; private int nusedbins; - private ArrayList bins; + private List bins; private Random prng; /** @@ -146,7 +147,7 @@ public void addBin(double x, double y, int b) { */ public void allocate(int num_bins) { nbins = num_bins; - bins = new ArrayList(); + bins = new ArrayList<>(); nusedbins = 0; } @@ -163,7 +164,7 @@ public void merge(NumericHistogram other) { // by deserializing the ArrayList of (x,y) pairs into an array of Coord objects nbins = other.nbins; nusedbins = other.nusedbins; - bins = new ArrayList(nusedbins); + bins = new ArrayList<>(nusedbins); for (int i = 0; i < other.nusedbins; i += 1) { Coord bin = new Coord(); bin.x = other.getBin(i).x; @@ -174,7 +175,7 @@ public void merge(NumericHistogram other) { // The aggregation buffer already contains a partial histogram. Therefore, we need // to merge histograms using Algorithm #2 from the Ben-Haim and Tom-Tov paper. - ArrayList tmp_bins = new ArrayList(nusedbins + other.nusedbins); + List tmp_bins = new ArrayList<>(nusedbins + other.nusedbins); // Copy all the histogram bins from us and 'other' into an overstuffed histogram for (int i = 0; i < nusedbins; i++) { Coord bin = new Coord(); diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/LogDivertAppender.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/LogDivertAppender.java index 79426e0e3de18..8ee606be314c2 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/LogDivertAppender.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/LogDivertAppender.java @@ -17,6 +17,7 @@ package org.apache.hive.service.cli.operation; import java.io.CharArrayWriter; +import java.io.Serializable; import java.util.Map; import java.util.regex.Pattern; @@ -265,7 +266,7 @@ private static StringLayout initLayout(OperationLog.LoggingLevel loggingMode) { Map appenders = root.getAppenders(); for (Appender ap : appenders.values()) { if (ap.getClass().equals(ConsoleAppender.class)) { - Layout l = ap.getLayout(); + Layout l = ap.getLayout(); if (l instanceof StringLayout) { layout = (StringLayout) l; break; diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/OperationManager.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/OperationManager.java index a261a54581828..6ee48186e7ea8 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/OperationManager.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/operation/OperationManager.java @@ -39,7 +39,7 @@ import org.apache.hive.service.cli.session.HiveSession; import org.apache.hive.service.rpc.thrift.TRowSet; import org.apache.hive.service.rpc.thrift.TTableSchema; -import org.apache.logging.log4j.core.appender.AbstractWriterAppender; +import org.apache.logging.log4j.core.Appender; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -82,7 +82,7 @@ public synchronized void stop() { private void initOperationLogCapture(String loggingMode) { // Register another Appender (with the same layout) that talks to us. - AbstractWriterAppender ap = new LogDivertAppender(this, OperationLog.getLoggingLevel(loggingMode)); + Appender ap = new LogDivertAppender(this, OperationLog.getLoggingLevel(loggingMode)); ((org.apache.logging.log4j.core.Logger)org.apache.logging.log4j.LogManager.getRootLogger()).addAppender(ap); ap.start(); } From 32c8bbfa288da0306083163bee612acd60277959 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 11 Oct 2022 14:16:58 +0800 Subject: [PATCH 2/2] fix test --- .../src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala index cf4a490b90273..9ef21c2954a0d 100644 --- a/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala +++ b/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala @@ -1075,7 +1075,7 @@ abstract class AvroSuite .save(s"$tempDir/${UUID.randomUUID()}") }.getMessage assert(message.contains("Caused by: java.lang.NullPointerException: ")) - assert(message.contains("null in string in field Name")) + assert(message.contains("null value for (non-nullable) string at test_schema.Name")) } }