[AMORO-2410] Upgrade iceberg to 1.4.3 (#2515)

* Upgrade iceberg to 1.4.3

* fix spark version missing

* fix UT

* fix UT

* add default properties(iceberg 1.4.x) for trino unit test

* fix trino spotless

* exclude rat check in ams-dashboard/dist

---------

Co-authored-by: ZhouJinsong <zhoujinsong0505@163.com>

.rat-excludes

@@ -17,4 +17,5 @@ docs/**
 **/node_modules/**
 **/pnpm-lock.yaml
 **/src/main/resources/static/**
+**/ams-dashboard/dist/**
 **/components.d.ts

docs/admin-guides/deployment.md

@@ -51,12 +51,12 @@ amoro-mixed-flink-runtime-1.15-x.y.z.jar # Flink 1.15 runtime package
 original-amoro-mixed-flink-runtime-1.15-x.y.z.jar
 maven-archiver/
 
-cd ${base_dir}/mixed/spark/v3.1/spark-runtime/target
+cd ${base_dir}/mixed/spark/v3.2/spark-runtime/target
 ls
-amoro-mixed-spark-3.1-runtime-x.y.z.jar # Spark v3.1 runtime package)
-amoro-mixed-spark-3.1-runtime-x.y.z-tests.jar
-amoro-mixed-spark-3.1-runtime-x.y.z-sources.jar
-original-amoro-mixed-spark-3.1-runtime-x.y.z.jar
+amoro-mixed-spark-3.2-runtime-x.y.z.jar # Spark v3.2 runtime package)
+amoro-mixed-spark-3.2-runtime-x.y.z-tests.jar
+amoro-mixed-spark-3.2-runtime-x.y.z-sources.jar
+original-amoro-mixed-spark-3.2-runtime-x.y.z.jar
 ```
 
 If you need to compile the Trino module at the same time, you need to install jdk17 locally and configure `toolchains.xml` in the user's `${user.home}/.m2/` directory,

mixed/flink/flink-common/src/main/java/com/netease/arctic/flink/table/ArcticTableLoader.java

@@ -103,6 +103,11 @@ public void open() {
     arcticCatalog = catalogBuilder.build();
   }
 
+  @Override
+  public boolean isOpen() {
+    return arcticCatalog != null;
+  }
+
   public ArcticTable loadArcticTable() {
     return ((ArcticTable)
         new FlinkTablePropertiesInvocationHandler(

mixed/spark/common/pom.xml

@@ -33,7 +33,7 @@
     <url>https://amoro.netease.com</url>
 
     <properties>
-        <spark.version>3.1.2</spark.version>
+        <spark.version>3.2.2</spark.version>
         <scala.version>2.12.15</scala.version>
     </properties>
     <dependencies>
@@ -144,7 +144,7 @@
 
         <dependency>
             <groupId>org.apache.iceberg</groupId>
-            <artifactId>iceberg-spark-3.1_2.12</artifactId>
+            <artifactId>iceberg-spark-3.2_2.12</artifactId>
             <version>${iceberg.version}</version>
             <scope>provided</scope>
             <exclusions>

mixed/spark/common/src/test/java/com/netease/arctic/spark/test/utils/Asserts.java

@@ -69,7 +69,10 @@ public static void assertPartition(PartitionSpec expectSpec, PartitionSpec actua
     Streams.zip(expectSpec.fields().stream(), actualSpec.fields().stream(), Pair::of)
         .forEach(
             x -> {
-              Assertions.assertEquals(x.getLeft().transform(), x.getRight().transform());
+              Assertions.assertTrue(
+                  x.getLeft()
+                      .transform()
+                      .canTransform(actualSchema.findType(x.getRight().sourceId())));
               Assertions.assertEquals(
                   expectSchema.findField(x.getLeft().sourceId()).name(),
                   actualSchema.findField(x.getRight().sourceId()).name());

mixed/spark/pom.xml

@@ -33,8 +33,6 @@
 
     <modules>
         <module>common</module>
-        <module>v3.1/spark</module>
-        <module>v3.1/spark-runtime</module>
         <module>v3.2/spark</module>
         <module>v3.2/spark-runtime</module>
         <module>v3.3/spark</module>