Bump Beam SDK to 2.60.0 (#2658)

Downgrade to 2.52.0
mozilla · Oct 22, 2024 · 50925a1 · 50925a1
1 parent e90cebc
commit 50925a1
Show file tree

Hide file tree

Showing 7 changed files with 16 additions and 9 deletions.
diff --git a/ingestion-beam/pom.xml b/ingestion-beam/pom.xml
@@ -59,6 +59,10 @@
             <groupId>org.apache.beam</groupId>
             <artifactId>beam-sdks-java-extensions-json-jackson</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.beam</groupId>
+            <artifactId>beam-sdks-java-extensions-avro</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.beam</groupId>
             <artifactId>beam-sdks-java-io-google-cloud-platform</artifactId>

diff --git a/ingestion-beam/src/main/java/com/mozilla/telemetry/avro/BinaryRecordFormatter.java b/ingestion-beam/src/main/java/com/mozilla/telemetry/avro/BinaryRecordFormatter.java
@@ -10,7 +10,7 @@
 import org.apache.avro.io.DatumReader;
 import org.apache.avro.io.Decoder;
 import org.apache.avro.io.DecoderFactory;
-import org.apache.beam.sdk.io.AvroIO.RecordFormatter;
+import org.apache.beam.sdk.extensions.avro.io.AvroIO.RecordFormatter;
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessage;
 
 /**

diff --git a/ingestion-beam/src/main/java/com/mozilla/telemetry/avro/PubsubMessageRecordFormatter.java b/ingestion-beam/src/main/java/com/mozilla/telemetry/avro/PubsubMessageRecordFormatter.java
@@ -10,7 +10,7 @@
 import org.apache.avro.generic.GenericRecord;
 import org.apache.avro.io.DatumReader;
 import org.apache.avro.io.Decoder;
-import org.apache.beam.sdk.io.AvroIO.RecordFormatter;
+import org.apache.beam.sdk.extensions.avro.io.AvroIO.RecordFormatter;
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessage;
 
 /**

diff --git a/ingestion-beam/src/main/java/com/mozilla/telemetry/io/Write.java b/ingestion-beam/src/main/java/com/mozilla/telemetry/io/Write.java
@@ -37,7 +37,7 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.ListCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.io.AvroIO;
+import org.apache.beam.sdk.extensions.avro.io.AvroIO;
 import org.apache.beam.sdk.io.Compression;
 import org.apache.beam.sdk.io.FileIO;
 import org.apache.beam.sdk.io.TextIO;

diff --git a/...stion-beam/src/test/java/com/mozilla/telemetry/avro/PubsubMessageRecordFormatterTest.java b/...stion-beam/src/test/java/com/mozilla/telemetry/avro/PubsubMessageRecordFormatterTest.java
@@ -1,6 +1,7 @@
 package com.mozilla.telemetry.avro;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 
 import com.fasterxml.jackson.databind.node.NullNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -96,7 +97,7 @@ public void testFormatWithNestedObjectStruct() {
     GenericRecord shape = (GenericRecord) record.get("shape");
     GenericRecord quad = (GenericRecord) shape.get("quadrilateral");
     assertEquals(true, quad.get("rhombus"));
-    assertEquals(null, shape.get("triangle"));
+    assertFalse(shape.hasField("triangle"));
   }
 
   @Test
@@ -338,6 +339,6 @@ public void testFormatCorrectsFieldNames() {
     assertEquals(true, record.get("test_dot"));
     assertEquals(true, record.get("_test_prefix_hyphen"));
     assertEquals(true, record.get("_0_test_prefix_number"));
-    assertEquals(null, record.get("$test_bad_symbol"));
+    assertFalse(record.hasField("$test_bad_symbol"));
   }
 }
diff --git a/...beam/src/test/java/com/mozilla/telemetry/integration/BeamDependenciesIntegrationTest.java b/...beam/src/test/java/com/mozilla/telemetry/integration/BeamDependenciesIntegrationTest.java
@@ -20,9 +20,11 @@ public class BeamDependenciesIntegrationTest {
   public void checkVersions() throws Exception {
     final String beamVersion = System.getProperty("beam.version");
     final Pom beamCore = getPom("org.apache.beam", "beam-sdks-java-core", beamVersion);
+    final Pom avroExtension = getPom("org.apache.beam", "beam-sdks-java-extensions-avro",
+        beamVersion);
 
     final Map<String, Optional<String>> expectedVersions = ImmutableMap.of(//
-        "avro.version", beamCore.getVersion("org.apache.avro", "avro"), //
+        "avro.version", avroExtension.getVersion("org.apache.avro", "avro"), //
         "jackson.version", beamCore.getVersion("com.fasterxml.jackson.core", "jackson-core"));
 
     final Map<String, Optional<String>> actualVersions = ImmutableMap.of(//

diff --git a/pom.xml b/pom.xml
@@ -55,12 +55,12 @@
         <argLine>-Xmx1024m -Djdk.net.URLClassPath.disableClassPathURLCheck=true</argLine>
 
         <auto-value.version>1.10.4</auto-value.version>
-        <beam.version>2.51.0</beam.version>
+        <beam.version>2.60.0</beam.version>
 
         <!-- Keep these dependency versions in sync with what is pulled in by beam;
              check https://mvnrepository.com/artifact/org.apache.beam -->
-        <jackson.version>2.14.1</jackson.version>
-        <avro.version>1.8.2</avro.version>
+        <jackson.version>2.15.4</jackson.version>
+        <avro.version>1.11.3</avro.version>
 
         <!-- The version pulled in by avro does not support Apple Silicon as of Jan 2022 -->
         <snappy.version>1.1.10.5</snappy.version>