Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bump: bump avro to 1.13.1 (was 1.8.2) #3039

Merged
merged 7 commits into from
Nov 23, 2023
Merged

bump: bump avro to 1.13.1 (was 1.8.2) #3039

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
eca7c17
bump: bump avro to 1.13.1
sebastian-alfers Nov 22, 2023
2e02b6f
override to fixed version
sebastian-alfers Nov 22, 2023
7d369b4
commenet to explain the override
sebastian-alfers Nov 22, 2023
54c85e2
sbt fmt, remove deprecated api
sebastian-alfers Nov 22, 2023
d81f6c8
fix more deprecations
sebastian-alfers Nov 22, 2023
4d705b9
more depreacation...
sebastian-alfers Nov 22, 2023
684e989
more deprecation
sebastian-alfers Nov 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion avroparquet-tests/src/test/java/docs/javadsl/AvroParquetSinkTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import org.apache.parquet.avro.AvroReadSupport;
import org.apache.parquet.hadoop.ParquetFileWriter;
import org.apache.parquet.hadoop.ParquetReader;
import org.apache.parquet.hadoop.util.HadoopOutputFile;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
Expand Down Expand Up @@ -75,7 +76,7 @@ public void createNewParquetFile()
Configuration conf = new Configuration();
conf.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, true);
ParquetWriter<GenericRecord> writer =
AvroParquetWriter.<GenericRecord>builder(new Path(file))
AvroParquetWriter.<GenericRecord>builder(HadoopOutputFile.fromPath(new Path(file), conf))
.withConf(conf)
.withWriteMode(ParquetFileWriter.Mode.OVERWRITE)
.withSchema(schema)
Expand Down
3 changes: 2 additions & 1 deletion avroparquet-tests/src/test/java/docs/javadsl/Examples.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.avro.Schema;
import akka.stream.javadsl.Source;
import org.apache.parquet.avro.AvroParquetReader;
import org.apache.parquet.hadoop.util.HadoopOutputFile;
// #init-reader

public class Examples {
Expand Down Expand Up @@ -53,7 +54,7 @@ public Examples() throws IOException {

// #init-flow
ParquetWriter<GenericRecord> writer =
AvroParquetWriter.<GenericRecord>builder(new Path("./test.parquet"))
AvroParquetWriter.<GenericRecord>builder(HadoopOutputFile.fromPath(new Path("./test.parquet"), conf))
.withConf(conf)
.withSchema(schema)
.build();
Expand Down
22 changes: 16 additions & 6 deletions avroparquet-tests/src/test/scala/docs/scaladsl/AbstractAvroParquet.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
package docs.scaladsl

import java.io.File

import akka.testkit.TestKit
import com.sksamuel.avro4s.RecordFormat
import org.apache.avro.Schema
Expand All @@ -14,7 +13,7 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.parquet.avro.{AvroParquetReader, AvroParquetWriter, AvroReadSupport}
import org.apache.parquet.hadoop.{ParquetReader, ParquetWriter}
import org.apache.parquet.hadoop.util.HadoopInputFile
import org.apache.parquet.hadoop.util.{HadoopInputFile, HadoopOutputFile}
import org.scalacheck.Gen
import org.scalatest.{BeforeAndAfterAll, Suite}

Expand Down Expand Up @@ -48,7 +47,11 @@ trait AbstractAvroParquet extends BeforeAndAfterAll {
conf.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, true)

def parquetWriter[T <: GenericRecord](file: String, conf: Configuration, schema: Schema): ParquetWriter[T] =
AvroParquetWriter.builder[T](new Path(file)).withConf(conf).withSchema(schema).build()
AvroParquetWriter
.builder[T](HadoopOutputFile.fromPath(new Path(file), conf))
.withConf(conf)
.withSchema(schema)
.build()

def parquetReader[T <: GenericRecord](file: String, conf: Configuration): ParquetReader[T] =
AvroParquetReader.builder[T](HadoopInputFile.fromPath(new Path(file), conf)).withConf(conf).build()
Expand Down Expand Up @@ -90,8 +93,11 @@ trait AbstractAvroParquet extends BeforeAndAfterAll {
val file: String = "./sample/path/test.parquet"
val conf: Configuration = new Configuration()
conf.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, true)
val writer: ParquetWriter[Record] =
AvroParquetWriter.builder[Record](new Path(file)).withConf(conf).withSchema(schema).build()
val writer: ParquetWriter[Record] = AvroParquetWriter
.builder[Record](HadoopOutputFile.fromPath(new Path(file), conf))
.withConf(conf)
.withSchema(schema)
.build()
// #prepare-sink
if (writer != null) { // forces val usage
}
Expand All @@ -107,7 +113,11 @@ trait AbstractAvroParquet extends BeforeAndAfterAll {

val file: String = "./sample/path/test.parquet"
val writer: ParquetWriter[GenericRecord] =
AvroParquetWriter.builder[GenericRecord](new Path(file)).withConf(conf).withSchema(schema).build()
AvroParquetWriter
.builder[GenericRecord](HadoopOutputFile.fromPath(new Path(file), conf))
.withConf(conf)
.withSchema(schema)
.build()
// #init-writer
// #init-reader
val reader: ParquetReader[GenericRecord] =
Expand Down
6 changes: 4 additions & 2 deletions project/Dependencies.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,9 @@ object Dependencies {

val AvroParquet = Seq(
libraryDependencies ++= Seq(
"org.apache.parquet" % "parquet-avro" % "1.10.1" //Apache2
"org.apache.parquet" % "parquet-avro" % "1.13.1", //Apache2
// override the version brought in by parquet-avro to fix CVE-2023-39410
"org.apache.avro" % "avro" % "1.11.3" //Apache2
)
)

Expand Down Expand Up @@ -230,7 +232,7 @@ object Dependencies {
libraryDependencies ++= Seq(
// https://github.com/googleapis/java-bigquerystorage/tree/master/proto-google-cloud-bigquerystorage-v1
"com.google.api.grpc" % "proto-google-cloud-bigquerystorage-v1" % "1.22.0" % "protobuf-src", // ApacheV2
"org.apache.avro" % "avro" % "1.9.2" % "provided",
"org.apache.avro" % "avro" % "1.11.3" % "provided",
"org.apache.arrow" % "arrow-vector" % "4.0.0" % "provided",
"io.grpc" % "grpc-auth" % akka.grpc.gen.BuildInfo.grpcVersion, // ApacheV2
"com.typesafe.akka" %% "akka-http-spray-json" % AkkaHttpVersion,
Expand Down
Loading