Support Spark4 Column Node API

src/main/scala-spark-3.0/org/apache/spark/extension/package.scala

@@ -0,0 +1 @@
+src/main/scala-spark-3.5/org/apache/spark/extension/package.scala

src/main/scala-spark-3.1/org/apache/spark/extension/package.scala

@@ -0,0 +1 @@
+src/main/scala-spark-3.5/org/apache/spark/extension/package.scala

src/main/scala-spark-3.2/org/apache/spark/extension/package.scala

@@ -0,0 +1 @@
+src/main/scala-spark-3.5/org/apache/spark/extension/package.scala

src/main/scala-spark-3.3/org/apache/spark/extension/package.scala

@@ -0,0 +1 @@
+src/main/scala-spark-3.5/org/apache/spark/extension/package.scala

src/main/scala-spark-3.4/org/apache/spark/extension/package.scala

@@ -0,0 +1 @@
+src/main/scala-spark-3.5/org/apache/spark/extension/package.scala

src/main/scala-spark-3.5/org/apache/spark/extension/package.scala

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2024 G-Research
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import org.apache.spark.sql.Column
+import org.apache.spark.sql.catalyst.expressions.Expression
+
+package object extension {
+  implicit class ExpressionExtension(expr: Expression) {
+    def toColumn: Column = new Column(expr)
+  }
+}

src/main/scala-spark-4.0/org/apache/spark/extension/package.scala

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2024 G-Research
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import org.apache.spark.sql.Column
+import org.apache.spark.sql.catalyst.expressions.Expression
+
+package object extension {
+  implicit class ExpressionExtension(expr: Expression) {
+    def toColumn: Column = Column(expr)
+  }
+}

src/main/scala/uk/co/gresearch/spark/diff/comparator/EquivDiffComparator.scala

@@ -16,6 +16,7 @@
 
 package uk.co.gresearch.spark.diff.comparator
 
+import org.apache.spark.extension.ExpressionExtension
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.encoders.encoderFor
 import org.apache.spark.sql.catalyst.expressions.codegen.Block.BlockHelper
@@ -31,8 +32,7 @@ trait EquivDiffComparator[T] extends DiffComparator {
 
 private trait ExpressionEquivDiffComparator[T] extends EquivDiffComparator[T] {
   def equiv(left: Expression, right: Expression): EquivExpression[T]
-  def equiv(left: Column, right: Column): Column =
-    new Column(equiv(left.expr, right.expr).asInstanceOf[Expression])
+  def equiv(left: Column, right: Column): Column = equiv(left.expr, right.expr).toColumn
 }
 
 trait TypedEquivDiffComparator[T] extends EquivDiffComparator[T] with TypedDiffComparator

src/main/scala/uk/co/gresearch/spark/package.scala

@@ -16,6 +16,7 @@
 
 package uk.co.gresearch
 
+import org.apache.spark.extension.ExpressionExtension
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.expressions.NamedExpression
@@ -271,7 +272,7 @@ package object spark extends Logging with SparkVersion with BuildVersion {
    *   result tick value column
    */
   def timestampToDotNetTicks(timestampColumn: Column): Column =
-    unixEpochTenthMicrosToDotNetTicks(new Column(UnixMicros.unixMicros(timestampColumn.expr)) * 10)
+    unixEpochTenthMicrosToDotNetTicks(UnixMicros.unixMicros(timestampColumn.expr).toColumn * 10)
 
   /**
    * Convert a Spark timestamp to a .Net `DateTime.Ticks` timestamp. The input column must be of TimestampType.