Dynamic and SemiDynamic schemas (#205)

* Dynamic and SemiDynamic schemas * DeriveGen * JsonCodec * Fix semiDynamicEncoder * Fixes * Json codec works * BigInt, BigDecimal support for protobuf * Fix protobuf semi dynamic record test * Various thrift codec fixes * SemiDynamic support for thrift * Fix some warnings * Fix more warnings * Fix compilation on 2.12 * scalafix * format * Reproducer for schema ast materialization problem * Format * Fix * Fix * Finish todo items * Remove unused import Co-authored-by: Dan Harris <1327726+thinkharderdev@users.noreply.github.com> Co-authored-by: Dan Harris <dan@thinkharder.dev>
zio · Mar 23, 2022 · 41caaa5 · 41caaa5
1 parent dde5660
commit 41caaa5
Show file tree

Hide file tree

Showing 22 changed files with 924 additions and 171 deletions.
diff --git a/tests/shared/src/test/scala/zio/schema/DefaultValueSpec.scala b/tests/shared/src/test/scala/zio/schema/DefaultValueSpec.scala
@@ -1,7 +1,6 @@
 package zio.schema
 
 import java.time.format.DateTimeFormatter
-import java.time.temporal.ChronoUnit
 
 import zio.Chunk
 import zio.schema.CaseSet.caseOf
@@ -102,7 +101,7 @@ object DefaultValueSpec extends DefaultRunnableSpec {
         assert(Primitive(StandardType.ZoneOffsetType).defaultValue)(isRight(equalTo(java.time.ZoneOffset.UTC)))
       },
       test("Duration default value") {
-        assert(Primitive(StandardType.Duration(ChronoUnit.SECONDS)).defaultValue)(
+        assert(Primitive(StandardType.DurationType).defaultValue)(
           isRight(equalTo(java.time.Duration.ZERO))
         )
       },

diff --git a/tests/shared/src/test/scala/zio/schema/DiffSpec.scala b/tests/shared/src/test/scala/zio/schema/DiffSpec.scala
@@ -157,6 +157,32 @@ object DiffSpec extends DefaultRunnableSpec with DefaultJavaTimeSchemas {
         testM("sealed trait")(diffLaw[Pet]),
         testM("high arity")(diffLaw[Arities]) @@ TestAspect.ignore,
         testM("recursive")(diffLaw[Recursive])
+      ),
+      suite("semiDynamic")(
+        testM("identity") {
+          val schema            = Schema[Person]
+          val semiDynamicSchema = Schema.semiDynamic[Person]()
+          val gen               = DeriveGen.gen[Person]
+          check(gen) { value =>
+            assertTrue(semiDynamicSchema.diff(value -> schema, value -> schema).isIdentical)
+          }
+        },
+        testM("diffLaw") {
+          val schema            = Schema[Person]
+          val semiDynamicSchema = Schema.semiDynamic[Person]()
+          val gen               = DeriveGen.gen[Person]
+          check(gen <*> gen) {
+            case (l, r) =>
+              val diff = semiDynamicSchema.diff(l -> schema, r -> schema)
+              if (diff.isComparable) {
+                val patched = diff.patch(l -> schema)
+                if (patched.isLeft) println(diff)
+                assert(patched)(isRight(equalTo(r -> schema)))
+              } else {
+                assertTrue(true)
+              }
+          }
+        }
       )
     ),
     suite("not comparable")(

diff --git a/tests/shared/src/test/scala/zio/schema/DynamicValueGen.scala b/tests/shared/src/test/scala/zio/schema/DynamicValueGen.scala
@@ -24,7 +24,7 @@ object DynamicValueGen {
       case typ: StandardType.BigDecimalType.type => gen(typ, Gen.anyDouble.map(d => java.math.BigDecimal.valueOf(d)))
       case typ: StandardType.BigIntegerType.type => gen(typ, Gen.anyLong.map(n => java.math.BigInteger.valueOf(n)))
       case typ: StandardType.DayOfWeekType.type  => gen(typ, JavaTimeGen.anyDayOfWeek)
-      case typ: StandardType.Duration            => gen(typ, JavaTimeGen.anyDuration)
+      case typ: StandardType.DurationType.type   => gen(typ, JavaTimeGen.anyDuration)
       case typ: StandardType.InstantType         => gen(typ, JavaTimeGen.anyInstant)
       case typ: StandardType.LocalDateType       => gen(typ, JavaTimeGen.anyLocalDate)
       case typ: StandardType.LocalDateTimeType   => gen(typ, JavaTimeGen.anyLocalDateTime)
@@ -83,6 +83,8 @@ object DynamicValueGen {
       case Schema.Fail(message, _)              => Gen.const(DynamicValue.Error(message))
       case l @ Schema.Lazy(_)                   => anyDynamicValueOfSchema(l.schema)
       case Schema.Meta(meta, _)                 => anyDynamicValueOfSchema(meta.toSchema)
+      case Schema.Dynamic(_)                    => SchemaGen.anySchema.flatMap(anyDynamicValueOfSchema(_))
+      case Schema.SemiDynamic(_, _)             => ??? // cannot generate dynamic value that corresponds to A
     }
   //scalafmt: { maxColumn = 120 }
 

diff --git a/tests/shared/src/test/scala/zio/schema/DynamicValueSpec.scala b/tests/shared/src/test/scala/zio/schema/DynamicValueSpec.scala
@@ -69,6 +69,20 @@ object DynamicValueSpec extends DefaultRunnableSpec {
           case (schema, a) =>
             assert(schema.fromDynamic(schema.toDynamic(a)))(isRight(equalTo(a)))
         }
+      },
+      testM("round-trip semiDynamic") {
+        val gen = for {
+          schemaAndGen       <- SchemaGen.anyGenericRecordAndGen
+          (schema, valueGen) = schemaAndGen
+          value              <- valueGen
+        } yield schema -> value
+        check(gen) {
+          case (schema, value) =>
+            val semiDynamicSchema = Schema.semiDynamic(defaultValue = Right(value -> schema))
+            assert(semiDynamicSchema.fromDynamic(semiDynamicSchema.toDynamic(value -> schema)))(
+              isRight(equalTo(value -> schema))
+            )
+        }
       }
     )
 

diff --git a/tests/shared/src/test/scala/zio/schema/OrderingSpec.scala b/tests/shared/src/test/scala/zio/schema/OrderingSpec.scala
@@ -31,6 +31,29 @@ object OrderingSpec extends DefaultRunnableSpec {
             case (schema, x, _, z) => assert(schema.ordering.compare(x, z))(isLessThan(0))
           }
         }
+      ),
+      suite("semiDynamic")(
+        testM("reflexivity") {
+          check(anySchemaAndValue) {
+            case (schema, a) =>
+              val semiDynamicSchema = Schema.semiDynamic(defaultValue = Right(a -> schema))
+              assert(semiDynamicSchema.ordering.compare(a -> schema, a -> schema))(equalTo(0))
+          }
+        },
+        testM("antisymmetry") {
+          check(genAnyOrderedPair) {
+            case (schema, x, y) =>
+              val semiDynamicSchema = Schema.semiDynamic(defaultValue = Right(x -> schema))
+              assert(semiDynamicSchema.ordering.compare(y -> schema, x -> schema))(isGreaterThan(0))
+          }
+        },
+        testM("transitivity") {
+          check(genAnyOrderedTriplet) {
+            case (schema, x, _, z) =>
+              val semiDynamicSchema = Schema.semiDynamic(defaultValue = Right(x -> schema))
+              assert(semiDynamicSchema.ordering.compare(x -> schema, z -> schema))(isLessThan(0))
+          }
+        }
       )
     )
 

diff --git a/tests/shared/src/test/scala/zio/schema/SchemaGen.scala b/tests/shared/src/test/scala/zio/schema/SchemaGen.scala
@@ -1,7 +1,6 @@
 package zio.schema
 
 import java.time.format.DateTimeFormatter
-import java.time.temporal.ChronoUnit
 
 import scala.collection.immutable.ListMap
 
@@ -592,6 +591,10 @@ object SchemaGen {
       value  <- Json.gen
     } yield (schema, value)
 
+  lazy val anyDynamic: Gen[Any, Schema[DynamicValue]] = Gen.const(Schema.dynamicValue)
+
+  def anySemiDynamic[A]: Gen[Any, Schema[(A, Schema[A])]] = Gen.const(Schema.semiDynamic[A]())
+
   case class SchemaTest[A](name: String, schema: StandardType[A], gen: Gen[Sized with Random, A])
 
   def schemasAndGens: List[SchemaTest[_]] = List(
@@ -616,7 +619,7 @@ object SchemaGen {
       Gen.anyLong.map(n => java.math.BigInteger.valueOf(n))
     ),
     SchemaTest("DayOfWeek", StandardType.DayOfWeekType, JavaTimeGen.anyDayOfWeek),
-    SchemaTest("Duration", StandardType.Duration(ChronoUnit.SECONDS), JavaTimeGen.anyDuration),
+    SchemaTest("Duration", StandardType.DurationType, JavaTimeGen.anyDuration),
     SchemaTest("Instant", StandardType.InstantType(DateTimeFormatter.ISO_DATE_TIME), JavaTimeGen.anyInstant),
     SchemaTest("LocalDate", StandardType.LocalDateType(DateTimeFormatter.ISO_DATE), JavaTimeGen.anyLocalDate),
     SchemaTest(

diff --git a/tests/shared/src/test/scala/zio/schema/StandardTypeGen.scala b/tests/shared/src/test/scala/zio/schema/StandardTypeGen.scala
@@ -2,7 +2,6 @@ package zio.schema
 
 import java.math.{ BigDecimal => JBigDecimal, BigInteger => JBigInt }
 import java.time.format.DateTimeFormatter
-import java.time.temporal.ChronoUnit
 
 import zio.random.Random
 import zio.test.{ Gen, Sized }
@@ -24,9 +23,9 @@ object StandardTypeGen {
       (StandardType.CharType),
       (StandardType.UUIDType),
       (StandardType.DayOfWeekType),
-      (StandardType.Duration(ChronoUnit.SECONDS)),
-      (StandardType.InstantType(DateTimeFormatter.ISO_DATE_TIME)),
-      (StandardType.LocalDateType(DateTimeFormatter.ISO_DATE)),
+      (StandardType.DurationType),
+      (StandardType.InstantType(DateTimeFormatter.ISO_INSTANT)),
+      (StandardType.LocalDateType(DateTimeFormatter.ISO_LOCAL_DATE)),
       (StandardType.LocalDateTimeType(DateTimeFormatter.ISO_LOCAL_DATE_TIME)),
       (StandardType.LocalTimeType(DateTimeFormatter.ISO_LOCAL_TIME)),
       (StandardType.MonthType),
@@ -68,7 +67,7 @@ object StandardTypeGen {
       case typ: StandardType.BigDecimalType.type => typ -> javaBigDecimal
       case typ: StandardType.BigIntegerType.type => typ -> javaBigInt
       case typ: StandardType.DayOfWeekType.type  => typ -> JavaTimeGen.anyDayOfWeek
-      case typ: StandardType.Duration            => typ -> JavaTimeGen.anyDuration
+      case typ: StandardType.DurationType.type   => typ -> JavaTimeGen.anyDuration
       case typ: StandardType.InstantType         => typ -> JavaTimeGen.anyInstant
       case typ: StandardType.LocalDateType       => typ -> JavaTimeGen.anyLocalDate
       case typ: StandardType.LocalDateTimeType   => typ -> JavaTimeGen.anyLocalDateTime

diff --git a/tests/shared/src/test/scala/zio/schema/types.scala b/tests/shared/src/test/scala/zio/schema/types.scala
@@ -1,21 +1,6 @@
 package zio.schema
 
-import java.time.temporal.ChronoUnit
-import java.time.{
-  DayOfWeek,
-  Instant,
-  LocalDate,
-  Month,
-  MonthDay,
-  OffsetDateTime,
-  OffsetTime,
-  Period,
-  Year,
-  YearMonth,
-  ZoneId,
-  ZoneOffset,
-  ZonedDateTime
-}
+import java.time._
 import java.util.UUID
 
 import zio.Chunk
@@ -28,7 +13,7 @@ object types {
   sealed trait Arities
 
   object Arities extends DefaultJavaTimeSchemas {
-    implicit val durationSchema: Schema[java.time.Duration] = Schema.primitive(StandardType.Duration(ChronoUnit.SECONDS))
+    implicit val durationSchema: Schema[java.time.Duration] = Schema.primitive(StandardType.DurationType)
 
     case object Arity0         extends Arities
     case class Arity1(f1: Int) extends Arities

diff --git a/zio-schema-json/shared/src/main/scala/zio/schema/codec/JsonCodec.scala b/zio-schema-json/shared/src/main/scala/zio/schema/codec/JsonCodec.scala
@@ -7,7 +7,7 @@ import scala.collection.immutable.ListMap
 
 import zio.json.JsonCodec._
 import zio.json.JsonDecoder.{ JsonError, UnsafeJson }
-import zio.json.internal.{ Lexer, RetractReader, StringMatrix, Write }
+import zio.json.internal.{ Lexer, RecordingReader, RetractReader, StringMatrix, Write }
 import zio.json.{ JsonCodec => ZJsonCodec, JsonDecoder, JsonEncoder, JsonFieldDecoder, JsonFieldEncoder }
 import zio.schema.Schema.EitherSchema
 import zio.schema.ast.SchemaAst
@@ -22,7 +22,7 @@ object JsonCodec extends Codec {
       (opt: Option[Chunk[A]]) =>
         ZIO
           .effect(opt.map(values => values.flatMap(Encoder.encode(schema, _))).getOrElse(Chunk.empty))
-          .catchAll(_ => ZIO.succeed(Chunk.empty))
+          .orDie
     )
 
   override def decoder[A](schema: Schema[A]): ZTransducer[Any, String, Byte, A] =
@@ -67,7 +67,7 @@ object JsonCodec extends Codec {
         case StandardType.BigDecimalType        => ZJsonCodec.bigDecimal
         case StandardType.UUIDType              => ZJsonCodec.uuid
         case StandardType.DayOfWeekType         => ZJsonCodec.dayOfWeek // ZJsonCodec[java.time.DayOfWeek]
-        case StandardType.Duration(_)           => ZJsonCodec.duration //ZJsonCodec[java.time.Duration]
+        case StandardType.DurationType          => ZJsonCodec.duration //ZJsonCodec[java.time.Duration]
         case StandardType.InstantType(_)        => ZJsonCodec.instant //ZJsonCodec[java.time.Instant]
         case StandardType.LocalDateType(_)      => ZJsonCodec.localDate //ZJsonCodec[java.time.LocalDate]
         case StandardType.LocalDateTimeType(_)  => ZJsonCodec.localDateTime //ZJsonCodec[java.time.LocalDateTime]
@@ -197,13 +197,44 @@ object JsonCodec extends Codec {
       case Schema.Enum22(c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c20, c21, c22, _) =>
         enumEncoder(c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c20, c21, c22)
       case Schema.EnumN(cs, _) => enumEncoder(cs.toSeq: _*)
+      case Schema.Dynamic(_) =>
+        dynamicEncoder
+      case Schema.SemiDynamic(_, _) =>
+        semiDynamicEncoder
     }
     //scalafmt: { maxColumn = 120, optIn.configStyleArguments = true }
 
     private val astEncoder: JsonEncoder[Schema[_]] =
       (schema: Schema[_], indent: Option[Int], out: Write) =>
         schemaEncoder(Schema[SchemaAst]).unsafeEncode(SchemaAst.fromSchema(schema), indent, out)
 
+    private def dynamicEncoder[A]: JsonEncoder[A] =
+      schemaEncoder(DynamicValueSchema()).asInstanceOf[JsonEncoder[A]]
+
+    private def semiDynamicEncoder[A]: JsonEncoder[(A, Schema[A])] =
+      (value: (A, Schema[A]), indent: Option[Int], out: Write) => {
+        val schema = value._2
+
+        // open
+        out.write('{')
+        val indent_ = bump(indent)
+        pad(indent_, out)
+        // schema
+        string.unsafeEncode(JsonFieldEncoder.string.unsafeEncodeField("schema"), indent_, out)
+        if (indent.isEmpty) out.write(':')
+        else out.write(" : ")
+        astEncoder.unsafeEncode(schema, indent_, out)
+        out.write(',')
+        // value
+        string.unsafeEncode(JsonFieldEncoder.string.unsafeEncodeField("value"), indent_, out)
+        if (indent.isEmpty) out.write(':')
+        else out.write(" : ")
+        schemaEncoder(schema).unsafeEncode(value._1, indent_, out)
+        // close
+        pad(indent, out)
+        out.write('}')
+      }
+
     private def transformEncoder[A, B](schema: Schema[A], g: B => Either[String, A]): JsonEncoder[B] = {
       (b: B, indent: Option[Int], out: Write) =>
         g(b) match {
@@ -357,13 +388,62 @@ object JsonCodec extends Codec {
         enumDecoder(c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c20, c21)
       case Schema.Enum22(c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c20, c21, c22, _) =>
         enumDecoder(c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c20, c21, c22)
-      case Schema.EnumN(cs, _) => enumDecoder(cs.toSeq: _*)
+      case Schema.EnumN(cs, _)      => enumDecoder(cs.toSeq: _*)
+      case Schema.Dynamic(_)        => dynamicDecoder
+      case Schema.SemiDynamic(_, _) => semiDynamicDecoder
     }
     //scalafmt: { maxColumn = 120, optIn.configStyleArguments = true }
 
     private def astDecoder: JsonDecoder[Schema[_]] =
       schemaDecoder(Schema[SchemaAst]).map(ast => ast.toSchema)
 
+    private def dynamicDecoder[A]: JsonDecoder[A] =
+      schemaDecoder(DynamicValueSchema()).asInstanceOf[JsonDecoder[A]]
+
+    private def semiDynamicDecoder[A]: JsonDecoder[(A, Schema[A])] =
+      (trace: List[JsonError], in: RetractReader) => {
+        var schema: Schema[A]                = null
+        var value: Any                       = null
+        var reader: RetractReader            = in
+        var recordingReader: RecordingReader = null
+        val matrix                           = new StringMatrix(Array("schema", "value"))
+
+        Lexer.char(trace, reader, '{')
+        if (Lexer.firstField(trace, reader)) {
+          do {
+            var trace_ = trace
+            val field  = Lexer.field(trace, reader, matrix)
+            if (field == 0) {
+              trace_ = JsonError.ObjectAccess("schema") :: trace
+              schema = astDecoder.unsafeDecode(trace_, reader).asInstanceOf[Schema[A]]
+            } else if (field == 1) {
+              if (schema != null) {
+                trace_ = JsonError.ObjectAccess("value") :: trace
+                value = schemaDecoder(schema).unsafeDecode(trace_, reader)
+              } else {
+                recordingReader = RecordingReader(in)
+                reader = recordingReader
+                Lexer.skipValue(trace_, reader)
+              }
+            } else Lexer.skipValue(trace_, reader)
+          } while (Lexer.nextField(trace, reader))
+        }
+
+        if (value == null) {
+          if (recordingReader == null) {
+            throw UnsafeJson(JsonError.Message("missing 'value' field") :: trace)
+          } else if (schema == null) {
+            throw UnsafeJson(JsonError.Message("missing 'schema' field") :: trace)
+          } else {
+            recordingReader.rewind()
+            val trace_ = JsonError.ObjectAccess("value") :: trace
+            value = schemaDecoder(schema).unsafeDecode(trace_, reader)
+          }
+        }
+
+        (value.asInstanceOf[A], schema)
+      }
+
     private def enumDecoder[Z](cases: Schema.Case[_, Z]*): JsonDecoder[Z] = {
       (trace: List[JsonError], in: RetractReader) =>
         {