From 753019983c1e12926ad227924ef2cbb07edc5e33 Mon Sep 17 00:00:00 2001 From: Dylan Do Amaral <37652599+dylandoamaral@users.noreply.github.com> Date: Tue, 6 Dec 2022 17:54:10 +0100 Subject: [PATCH] fix: ci bug (#272) --- .../experimental/MapWithEffectSpec.scala | 7 ++ .../experimental/MapWithEffectSpec.scala | 7 ++ .../experimental/MapWithEffectSpec.scala | 7 +- .../experimental/MapWithEffectSpec.scala | 7 ++ .../scala-3/zio/spark/sql/ToSchemaSpec.scala | 109 ------------------ .../scala/zio/spark/ZioSparkTestSpec.scala | 9 +- .../zio/spark/rdd/PairRDDFunctionsSpec.scala | 3 +- 7 files changed, 30 insertions(+), 119 deletions(-) create mode 100644 zio-spark-core/src/test/scala-2.11/zio/spark/experimental/MapWithEffectSpec.scala create mode 100644 zio-spark-core/src/test/scala-2.12/zio/spark/experimental/MapWithEffectSpec.scala create mode 100644 zio-spark-core/src/test/scala-3/zio/spark/experimental/MapWithEffectSpec.scala delete mode 100644 zio-spark-core/src/test/scala-3/zio/spark/sql/ToSchemaSpec.scala diff --git a/zio-spark-core/src/test/scala-2.11/zio/spark/experimental/MapWithEffectSpec.scala b/zio-spark-core/src/test/scala-2.11/zio/spark/experimental/MapWithEffectSpec.scala new file mode 100644 index 00000000..3180e625 --- /dev/null +++ b/zio-spark-core/src/test/scala-2.11/zio/spark/experimental/MapWithEffectSpec.scala @@ -0,0 +1,7 @@ +package zio.spark.experimental + +import zio.test._ + +object MapWithEffectSpec { + def spec = suite("smoke")() +} diff --git a/zio-spark-core/src/test/scala-2.12/zio/spark/experimental/MapWithEffectSpec.scala b/zio-spark-core/src/test/scala-2.12/zio/spark/experimental/MapWithEffectSpec.scala new file mode 100644 index 00000000..3180e625 --- /dev/null +++ b/zio-spark-core/src/test/scala-2.12/zio/spark/experimental/MapWithEffectSpec.scala @@ -0,0 +1,7 @@ +package zio.spark.experimental + +import zio.test._ + +object MapWithEffectSpec { + def spec = suite("smoke")() +} diff --git a/zio-spark-core/src/test/scala-2.13/zio/spark/experimental/MapWithEffectSpec.scala b/zio-spark-core/src/test/scala-2.13/zio/spark/experimental/MapWithEffectSpec.scala index 4af4eca8..2455943f 100644 --- a/zio-spark-core/src/test/scala-2.13/zio/spark/experimental/MapWithEffectSpec.scala +++ b/zio-spark-core/src/test/scala-2.13/zio/spark/experimental/MapWithEffectSpec.scala @@ -3,15 +3,14 @@ package zio.spark.experimental import scala3encoders.given // scalafix:ok import zio.{Task, ZIO} -import zio.spark.ZioSparkTestSpec.session import zio.spark.experimental.MapWithEffect.RDDOps import zio.spark.rdd.RDD import zio.spark.sql._ import zio.spark.sql.implicits._ import zio.test._ -object MapWithEffectSpec extends ZIOSpecDefault { - override def spec: Spec[TestEnvironment, Any] = +object MapWithEffectSpec { + def spec: Spec[SparkSession, Throwable] = suite("smoke")( test("basic smoke test") { val getRddInt: SIO[RDD[Int]] = Seq(1, 2, 3).toRDD @@ -36,5 +35,5 @@ object MapWithEffectSpec extends ZIOSpecDefault { assertTrue(size == 10000, i == 1, count.toDouble < (0.95d * size)) } } - ).provideLayerShared(session) + ) } diff --git a/zio-spark-core/src/test/scala-3/zio/spark/experimental/MapWithEffectSpec.scala b/zio-spark-core/src/test/scala-3/zio/spark/experimental/MapWithEffectSpec.scala new file mode 100644 index 00000000..3180e625 --- /dev/null +++ b/zio-spark-core/src/test/scala-3/zio/spark/experimental/MapWithEffectSpec.scala @@ -0,0 +1,7 @@ +package zio.spark.experimental + +import zio.test._ + +object MapWithEffectSpec { + def spec = suite("smoke")() +} diff --git a/zio-spark-core/src/test/scala-3/zio/spark/sql/ToSchemaSpec.scala b/zio-spark-core/src/test/scala-3/zio/spark/sql/ToSchemaSpec.scala deleted file mode 100644 index 70d45b77..00000000 --- a/zio-spark-core/src/test/scala-3/zio/spark/sql/ToSchemaSpec.scala +++ /dev/null @@ -1,109 +0,0 @@ -package zio.spark.sql - -import org.apache.spark.sql.types._ - -import zio.Scope -import zio.spark.sql.ToSchema._ -import zio.test._ -import zio.test.Assertion._ -import zio.test.TestAspect._ - -object ToSchemaSpec extends ZIOSpecDefault { - override def spec: Spec[TestEnvironment with Scope, Any] = - suite("Convert case class into schema")( - test("Case class with one string") { - final case class Test(foo: String) - - val schema = StructType(Seq(StructField("foo", StringType, nullable = false))) - assert(schema.treeString)(equalTo(summon[ToStructSchema[Test]].toSchema.treeString)) - }, - test("Case class with all basic types") { - final case class Test(foo: String, bar: Boolean) - - val schema = - StructType( - Seq( - StructField("foo", StringType, nullable = false), - StructField("bar", BooleanType, nullable = false) - ) - ) - assert(summon[ToStructSchema[Test]].toSchema.treeString)(equalTo(schema.treeString)) - }, - test("Case class with one string and one optional boolean") { - final case class Test( - boolean: Boolean, - string: String, - short: Short, - integer: Int, - long: Long, - bigDecimal: BigDecimal, - float: Float, - double: Double, - byte: Byte, - binary: Array[Byte], - timestamp: java.sql.Timestamp, - date: java.sql.Date - ) - - val schema = - StructType( - Seq( - StructField("boolean", BooleanType, nullable = false), - StructField("string", StringType, nullable = false), - StructField("short", ShortType, nullable = false), - StructField("integer", IntegerType, nullable = false), - StructField("long", LongType, nullable = false), - StructField("bigDecimal", DecimalType(10, 0), nullable = false), - StructField("float", FloatType, nullable = false), - StructField("double", DoubleType, nullable = false), - StructField("byte", ByteType, nullable = false), - StructField("binary", BinaryType, nullable = false), - StructField("timestamp", TimestampType, nullable = false), - StructField("date", DateType, nullable = false) - ) - ) - assert(summon[ToStructSchema[Test]].toSchema.treeString)(equalTo(schema.treeString)) - }, - test("Case class with a map") { - final case class Test(foo: Map[String, Boolean]) - - val schema = - StructType( - Seq( - StructField("foo", DataTypes.createMapType(StringType, BooleanType), nullable = false) - ) - ) - assert(summon[ToStructSchema[Test]].toSchema.treeString)(equalTo(schema.treeString)) - }, - test("Case class with a nested case class") { - final case class Bar(baz: String) - final case class Test(foo: Int, bar: Bar) - - val nestedSchema = StructType(Seq(StructField("baz", StringType, nullable = false))) - val schema = - StructType( - Seq( - StructField("foo", IntegerType, nullable = false), - StructField("bar", nestedSchema, nullable = false) - ) - ) - - assert(summon[ToStructSchema[Test]].toSchema.treeString)(equalTo(schema.treeString)) - }, -// test("Case class with a nested nullable case class") { -// final case class Bar(baz: String) -// final case class Test(foo: Int, bar: Option[Bar]) -// -// val nestedSchema = StructType(Seq(StructField("baz", StringType, nullable = false))) -// val schema = -// StructType( -// Seq( -// StructField("foo", IntegerType, nullable = false), -// StructField("bar", nestedSchema, nullable = true) -// ) -// ) -// -// assert(summon[ToStructSchema[Test]].toSchema.treeString)(equalTo(schema.treeString)) -// } - ) -} diff --git a/zio-spark-core/src/test/scala/zio/spark/ZioSparkTestSpec.scala b/zio-spark-core/src/test/scala/zio/spark/ZioSparkTestSpec.scala index 6dfadf0c..1b5e0e68 100644 --- a/zio-spark-core/src/test/scala/zio/spark/ZioSparkTestSpec.scala +++ b/zio-spark-core/src/test/scala/zio/spark/ZioSparkTestSpec.scala @@ -3,7 +3,7 @@ package zio.spark import org.apache.log4j.{Level, Logger} import zio._ -import zio.spark.experimental.{CancellableEffectSpec, PipelineSpec} +import zio.spark.experimental.{CancellableEffectSpec, MapWithEffectSpec, PipelineSpec} import zio.spark.parameter.localAllNodes import zio.spark.rdd.{PairRDDFunctionsSpec, RDDSpec} import zio.spark.sql.{ @@ -31,7 +31,7 @@ object ZioSparkTestSpec extends ZIOSpecDefault { type SparkTestEnvironment = TestEnvironment with SparkSession type SparkTestSpec = Spec[SparkTestEnvironment, Any] - def spec: Spec[TestEnvironment, Any] = { + def spec: Spec[TestEnvironment with Scope, Any] = { val specs: Seq[Spec[SparkTestEnvironment, Any]] = Seq( DatasetSpec.datasetActionsSpec, @@ -52,9 +52,10 @@ object ZioSparkTestSpec extends ZIOSpecDefault { RelationalGroupedDatasetSpec.relationalGroupedDatasetAggregationSpec, CancellableEffectSpec.spec, StreamingSpec.streamingSpec, - DataStreamWriterSpec.dataStreamReaderConfigurationsSpec + DataStreamWriterSpec.dataStreamReaderConfigurationsSpec, + MapWithEffectSpec.spec ) - suite("Spark tests")(specs: _*).provideCustomLayerShared(session) + suite("Spark tests")(specs: _*).provideSomeLayerShared[TestEnvironment](session) } } diff --git a/zio-spark-core/src/test/scala/zio/spark/rdd/PairRDDFunctionsSpec.scala b/zio-spark-core/src/test/scala/zio/spark/rdd/PairRDDFunctionsSpec.scala index 44144202..930a2317 100644 --- a/zio-spark-core/src/test/scala/zio/spark/rdd/PairRDDFunctionsSpec.scala +++ b/zio-spark-core/src/test/scala/zio/spark/rdd/PairRDDFunctionsSpec.scala @@ -7,7 +7,6 @@ import zio.spark.sql.{Dataset, SparkSession} import zio.spark.sql.implicits._ import zio.test._ import zio.test.Assertion._ -import zio.test.TestAspect._ object PairRDDFunctionsSpec { def spec: Spec[SparkSession, Any] = @@ -20,5 +19,5 @@ object PairRDDFunctionsSpec { job.map(assert(_)(equalTo(67L))) } - ) @@ scala212(ignore) + ) }