From 9733f834f11b9bfc4f8c37d1d7aacf6d6c7e018b Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Tue, 4 Jul 2023 11:58:30 +0330 Subject: [PATCH 01/63] refactor docs. --- docs/codecs.md | 32 +++ docs/combining-different-encoders.md | 42 ++++ docs/getting-started.md | 146 ++++++++++++ docs/motivation.md | 97 ++++++++ docs/our-first-schema.md | 90 +------ docs/protobuf-example.md | 31 +++ docs/sidebars.js | 11 +- docs/transforming-schemas.md | 33 +++ docs/understanding-zio-schema.md | 335 --------------------------- docs/use-cases.md | 24 +- 10 files changed, 408 insertions(+), 433 deletions(-) create mode 100644 docs/codecs.md create mode 100644 docs/combining-different-encoders.md create mode 100644 docs/getting-started.md create mode 100644 docs/motivation.md create mode 100644 docs/protobuf-example.md create mode 100644 docs/transforming-schemas.md delete mode 100644 docs/understanding-zio-schema.md diff --git a/docs/codecs.md b/docs/codecs.md new file mode 100644 index 000000000..f6e5d25a6 --- /dev/null +++ b/docs/codecs.md @@ -0,0 +1,32 @@ +--- +id: codecs +title: "Codecs" +--- + +Once we have our schema, we can combine it with a codec. A codec is a combination of a schema and a serializer. Unlike codecs in other libraries, a codec in ZIO Schema has no type parameter: + +```scala +trait Codec { + def encoder[A](schema: Schema[A]): ZTransducer[Any, Nothing, A, Byte] + def decoder[A](schema: Schema[A]): ZTransducer[Any, String, Byte, A] + + def encode[A](schema: Schema[A]): A => Chunk[Byte] + def decode[A](schema: Schema[A]): Chunk[Byte] => Either[String, A] +} +``` + +It basically says: +- `encoder[A]`: Given a `Schema[A]` it is capable of generating an `Encoder[A]` ( `A => Chunk[Byte]`) for any Schema. +- `decoder[A]`: Given a `Schema[A]` it is capable of generating a `Decoder[A]` ( `Chunk[Byte] => Either[String, A]`) for any Schema. + +Example of possible codecs are: + +- CSV Codec +- JSON Codec (already available) +- Apache Avro Codec (in progress) +- Apache Thrift Codec (in progress) +- XML Codec +- YAML Codec +- Protobuf Codec (already available) +- QueryString Codec +- etc. diff --git a/docs/combining-different-encoders.md b/docs/combining-different-encoders.md new file mode 100644 index 000000000..980905848 --- /dev/null +++ b/docs/combining-different-encoders.md @@ -0,0 +1,42 @@ +--- +id: combining-different-encoders +title: "Combining Different Encoders" +--- + +Let's take a look at a round-trip converting an object to JSON and back, then converting it to a protobuf and back. This is a simple example, but it shows how to combine different encoders to achieve a round-trip. + +```scala +object CombiningExample extends zio.App { + import zio.schema.codec.JsonCodec + import zio.schema.codec.ProtobufCodec + import ManualConstruction._ + import zio.stream.ZStream + + override def run(args: List[String]): UIO[ExitCode] = for { + _ <- ZIO.unit + _ <- ZIO.debug("combining roundtrip") + person = Person("Michelle", 32) + + personToJson = JsonCodec.encoder[Person](schemaPerson) + jsonToPerson = JsonCodec.decoder[Person](schemaPerson) + + personToProto = ProtobufCodec.encoder[Person](schemaPerson) + protoToPerson = ProtobufCodec.decoder[Person](schemaPerson) + + newPerson <- ZStream(person) + .tap(v => ZIO.debug("input object is: " + v)) + .transduce(personToJson) + .transduce(jsonToPerson) + .tap(v => ZIO.debug("object after json roundtrip: " + v)) + .transduce(personToProto) + .transduce(protoToPerson) + .tap(v => ZIO.debug("person after protobuf roundtrip: " + v)) + .runHead + .some + .catchAll(error => ZIO.debug(error)) + _ <- ZIO.debug("is old person the new person? " + (person == newPerson).toString) + _ <- ZIO.debug("old person: " + person) + _ <- ZIO.debug("new person: " + newPerson) + } yield ExitCode.success +} +``` diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 000000000..98503ef63 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,146 @@ +--- +id: getting-started +title: "Getting Started" +--- + +To get started, first we need to understand that a ZIO Schema is basically built-up from these three +sealed traits: `Record[R]`, `Enum[A]` and `Sequence[Col, Elem]`, along with the case class `Primitive[A]`. Every other type is just a specialisation of one of these (or not relevant to get you started). + +We will take a look at them now. + +## Basic Building Blocks + +### Schema + +The core data type of ZIO Schema is a `Schema[A]` which is **invariant in `A`** by necessity, because a Schema allows us to derive operations that produce an `A` but also operations that consume an `A` and that imposes limitations on the types of **transformation operators** and **composition operators** that we can provide based on a `Schema`. + +It looks kind of like this (simplified): + +```scala +sealed trait Schema[A] { self => + def zip[B](that: Schema[B]): Schema[(A, B)] + + def transform[B](f: A => B, g: B => A): Schema[B] +} +``` + +### Records + +Our data structures usually are composed of a lot of types. For example, we might have a `User` +type that has a `name` field, an `age` field, an `address` field, and a `friends` field. + +```scala +case class User(name: String, age: Int, address: Address, friends: List[User]) +``` + +This is called a **product type** in functional programming. The equivalent of a product type in ZIO Schema is called a record. + +In ZIO Schema such a record would be represented using the `Record[R]` typeclass: + +```scala +object Schema { + sealed trait Record[R] extends Schema[R] { + def fields: Chunk[Field[_]] + def construct(value: R): Chunk[Any] + def defaultValue: Either[String, R] + } +} + +``` + +### Enumerations + +Other times, you might have a type that represents a list of different types. For example, we might have a type, like this: + +```scala +sealed trait PaymentMethod + +object PaymentMethod { + final case class CreditCard(number: String, expirationMonth: Int, expirationYear: Int) extends PaymentMethod + final case class WireTransfer(accountNumber: String, bankCode: String) extends PaymentMethod +} +``` + +In functional programming, this kind of type is called a **sum type**: +- In Scala 2, this is called a **sealed trait**. +- In Scala3, this is called an **enum**. + +In ZIO Schema we call these types `enumeration` types, and they are represented using the `Enum[A]` type class. + +```scala +object Schema extends SchemaEquality { + sealed trait Enum[A] extends Schema[A] { + def annotations: Chunk[Any] + def structure: ListMap[String, Schema[_]] + } +} +``` + +### Sequence + +Often you have a type that is a collection of elements. For example, you might have a `List[User]`. This is called a `Sequence` and is represented using the `Sequence[Col, Elem]` type class: + +```scala +object Schema extends SchemaEquality { + + final case class Sequence[Col, Elem]( + elementSchema: Schema[Elem], + fromChunk: Chunk[Elem] => Col, + toChunk: Col => Chunk[Elem] + ) extends Schema[Col] { + self => + override type Accessors[Lens[_, _], Prism[_, _], Traversal[_, _]] = Traversal[Col, Elem] + override def makeAccessors(b: AccessorBuilder): b.Traversal[Col, Elem] = b.makeTraversal(self, schemaA) + override def toString: String = s"Sequence($elementSchema)" + } +} +``` + +### Optionals + +A special variant of a collection type is the `Optional[A]` type: + +```scala +object Schema extends SchemaEquality { + + final case class Optional[A](codec: Schema[A]) extends Schema[Option[A]] { + self => + + private[schema] val someCodec: Schema[Some[A]] = codec.transform(a => Some(a), _.get) + + override type Accessors[Lens[_, _], Prism[_, _], Traversal[_, _]] = + (Prism[Option[A], Some[A]], Prism[Option[A], None.type]) + + val toEnum: Enum2[Some[A], None.type, Option[A]] = Enum2( + Case[Some[A], Option[A]]("Some", someCodec, _.asInstanceOf[Some[A]], Chunk.empty), + Case[None.type, Option[A]]("None", singleton(None), _.asInstanceOf[None.type], Chunk.empty), + Chunk.empty + ) + + override def makeAccessors(b: AccessorBuilder): (b.Prism[Option[A], Some[A]], b.Prism[Option[A], None.type]) = + b.makePrism(toEnum, toEnum.case1) -> b.makePrism(toEnum, toEnum.case2) + } + +} +``` + +### Primitives + +Last but not least, we have primitive values. + +```scala +object Schema extends SchemaEquality { + final case class Primitive[A](standardType: StandardType[A]) extends Schema[A] { + type Accessors[Lens[_, _], Prism[_, _], Traversal[_, _]] = Unit + + override def makeAccessors(b: AccessorBuilder): Unit = () + } +} +``` + +Primitive values are represented using the `Primitive[A]` type class and represent the elements, +that we cannot further define through other means. If we visualize our data structure as a tree, primitives are the leaves. + +ZIO Schema provides a number of built-in primitive types, that we can use to represent our data. +These can be found in the `StandardType` companion-object. + diff --git a/docs/motivation.md b/docs/motivation.md new file mode 100644 index 000000000..77470f3da --- /dev/null +++ b/docs/motivation.md @@ -0,0 +1,97 @@ +--- +id: motivation +title: "Understanding The Motivation Behind ZIO Schema" +--- + +ZIO Schema is a library used in many ZIO projects such as _ZIO Flow_, _ZIO Redis_, _ZIO Web_, _ZIO SQL_ and _ZIO DynamoDB_. It is all about reification of our types. Reification means transforming something abstract (e.g. side effects, accessing fields, structure) into something "real" (values). + +## Reification: Functional Effects + +In functional effects, we reify by turning side-effects into values. For example, we might have a simple statement like; + +```scala +println("Hello") +println("World") +``` + +In ZIO we reify this statement to a value like + +```scala +val effect1 = Task(println("Hello")) +val effect2 = Task(println("World")) +``` + +And then we are able to do awesome things like: + +```scala +(Task(println("Hello")) zipPar Task(println("World"))).retryN(100) +``` + +## Reification: Optics + +In Scala, we have product types like this case class of a `Person`: + +```scala +final case class Person(name: String, age: Int) +``` + +This case class has two fields: + +- A field `name` of type `String` +- A field `age` of type `Int` + +The Scala language provides special support to access the fields inside case classes using the dot syntax: + +```scala +val person = Person("Michelle", 32) +val name = person.name +val age = person.age +``` + +However, this is a "special language feature", it's not "real" like the side effects we've seen in the previous example (`println(..) vs. Task(println(...))`). + +Because these basic operations are not "real," we are unable to create an operator that we can use, for example, we cannot combine two fields that are inside a nested structure. + +The solution to this kind of problem is called an "Optic". Optics provide a way to access the fields of a case class and nested structures. There are three main types of optics: +- `Lens`: A lens is a way to access a field of a case class. +- `Prism`: A prism is a way to access a field of a nested structure or a collection. +- `Traversal`: A traversal is a way to access all fields of a case class, nested structures or collections. + +Optics allow us to take things which are not a first-class **concept**, and turn that into a first-class **value**, namely the concept of +- drilling down into a field inside a case class or +- drilling down into a nested structure. + +Once we have a value, we can compose these things together to solve hard problems in functional programming, e.g. +- handling nested case class copies, +- iterations down deep inside on elements of a nested structure or collections + +For more information on optics, refer to the [ZIO Optics](https://zio.dev/zio-optics/) documentation. + + +## Reification: Schema + +So far we've looked at how to +- reify side-effects into values (ZIO) +- how to reify accessing and modifying fields inside case classes or arbitrary structures by turning these operations into values as well (Optics) + +**ZIO Schema** is now about how to **describe entire data structures using values**. + +The "built-in" way in scala on how to describe data structures are `case classes` and `classes`. + +For example, assume we have the `Person` data type, like this: + +```scala +final case class Person(name: String, age: Int) +``` + +It has the following information: + +- Name of the structure: `Person` +- Fields: `name` and `age` +- Type of the fields: `String` and `Int` +- Type of the structure: `Person` + +ZIO Schema tries to reify the concept of structure for datatypes by turning the above information into values. + +Not only for case classes, but also for other types like collections, tuples, enumerations etc. + diff --git a/docs/our-first-schema.md b/docs/our-first-schema.md index b8d20903f..6a2b7eb53 100644 --- a/docs/our-first-schema.md +++ b/docs/our-first-schema.md @@ -3,12 +3,11 @@ id: our-first-schema title: "Our First Schema" --- -ZIO Schema provides macros to help you create `Schema`s out of your data types. But before using the macros, -we should take a look at how to do this the manual way. +ZIO Schema provides macros to help you create `Schema`s out of our data types. But before using the macros, we should take a look at how to do this the manual way. -### The Domain +## The Domain -Like in [Overview](index.md), we define our example domain like this: +As described in the [Overview](index.md) section, we define the example domain as follows: ```scala object Domain { @@ -28,11 +27,9 @@ object Domain { ### Manual construction of a Schema -This part is similar to other libraries that you might know, e.g. for JSON processing. -Basically, you create a `Schema` for every data type in your domain: +This part is similar to other libraries that we might know, e.g. for JSON processing. Basically, we create a `Schema` for every data type in our domain: ```scala - object ManualConstruction { import zio.schema.Schema._ import Domain._ @@ -85,7 +82,6 @@ object ManualConstruction { extractField2 = c => c.paymentMethod ) } - ``` ### Macro derivation @@ -104,11 +100,9 @@ object MacroConstruction { } ``` -## Applying it to our domain - -### Json example +## Applying it to Our Domain -Lets put this all together in a small sample: +Let's put this all together in a small sample: ```scala object JsonSample extends zio.App { @@ -129,77 +123,7 @@ object JsonSample extends zio.App { ``` When we run this, we get our expected result printed out: + ```json {"name":"Michelle","age":32} ``` - -### Protobuf example - -```scala -object ProtobufExample extends zio.App { - import zio.schema.codec.ProtobufCodec - import ManualConstruction._ - import zio.stream.ZStream - - override def run(args: List[String]): UIO[ExitCode] = for { - _ <- ZIO.unit - _ <- ZIO.debug("protobuf roundtrip") - person = Person("Michelle", 32) - - personToProto = ProtobufCodec.encoder[Person](schemaPerson) - protoToPerson = ProtobufCodec.decoder[Person](schemaPerson) - - newPerson <- ZStream(person) - .transduce(personToProto) - .transduce(protoToPerson) - .runHead - .some - .catchAll(error => ZIO.debug(error)) - _ <- ZIO.debug("is old person the new person? " + (person == newPerson).toString) - _ <- ZIO.debug("old person: " + person) - _ <- ZIO.debug("new person: " + newPerson) - } yield ExitCode.success -} -``` - - -### Combining different encoders - -Let's take a look at a roundtrip converting an object to JSON and back, then converting it to a protobuf and back. -This is a simple example, but it shows how to combine different encoders to achieve a roundtrip. - -```scala -object CombiningExample extends zio.App { - import zio.schema.codec.JsonCodec - import zio.schema.codec.ProtobufCodec - import ManualConstruction._ - import zio.stream.ZStream - - override def run(args: List[String]): UIO[ExitCode] = for { - _ <- ZIO.unit - _ <- ZIO.debug("combining roundtrip") - person = Person("Michelle", 32) - - personToJson = JsonCodec.encoder[Person](schemaPerson) - jsonToPerson = JsonCodec.decoder[Person](schemaPerson) - - personToProto = ProtobufCodec.encoder[Person](schemaPerson) - protoToPerson = ProtobufCodec.decoder[Person](schemaPerson) - - newPerson <- ZStream(person) - .tap(v => ZIO.debug("input object is: " + v)) - .transduce(personToJson) - .transduce(jsonToPerson) - .tap(v => ZIO.debug("object after json roundtrip: " + v)) - .transduce(personToProto) - .transduce(protoToPerson) - .tap(v => ZIO.debug("person after protobuf roundtrip: " + v)) - .runHead - .some - .catchAll(error => ZIO.debug(error)) - _ <- ZIO.debug("is old person the new person? " + (person == newPerson).toString) - _ <- ZIO.debug("old person: " + person) - _ <- ZIO.debug("new person: " + newPerson) - } yield ExitCode.success -} -``` diff --git a/docs/protobuf-example.md b/docs/protobuf-example.md new file mode 100644 index 000000000..5e7591aa5 --- /dev/null +++ b/docs/protobuf-example.md @@ -0,0 +1,31 @@ +--- +id: protobuf-example +title: "Protobuf Example" +--- + +```scala +object ProtobufExample extends zio.App { + import zio.schema.codec.ProtobufCodec + import ManualConstruction._ + import zio.stream.ZStream + + override def run(args: List[String]): UIO[ExitCode] = for { + _ <- ZIO.unit + _ <- ZIO.debug("protobuf roundtrip") + person = Person("Michelle", 32) + + personToProto = ProtobufCodec.encoder[Person](schemaPerson) + protoToPerson = ProtobufCodec.decoder[Person](schemaPerson) + + newPerson <- ZStream(person) + .transduce(personToProto) + .transduce(protoToPerson) + .runHead + .some + .catchAll(error => ZIO.debug(error)) + _ <- ZIO.debug("is old person the new person? " + (person == newPerson).toString) + _ <- ZIO.debug("old person: " + person) + _ <- ZIO.debug("new person: " + newPerson) + } yield ExitCode.success +} +``` diff --git a/docs/sidebars.js b/docs/sidebars.js index 102859d51..3d3940693 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -6,9 +6,14 @@ const sidebars = { collapsed: false, link: { type: "doc", id: "index" }, items: [ - 'use-cases', - 'our-first-schema', - 'understanding-zio-schema', + "use-cases", + "our-first-schema", + "motivation", + "getting-started", + "transforming-schemas", + "codecs", + "protobuf-example", + "combining-different-encoders" ] } ] diff --git a/docs/transforming-schemas.md b/docs/transforming-schemas.md new file mode 100644 index 000000000..40d1263e8 --- /dev/null +++ b/docs/transforming-schemas.md @@ -0,0 +1,33 @@ +--- +id: transforming-schemas +title: "Transforming Schemas" +--- + +Once we have a `Schema`, we can transform it into another `Schema` by applying a `Transformer`. In normal Scala code this would be the equivalent of `map`. + +In ZIO Schema this is modelled by the `Transform` type class: + +```scala + final case class Transform[A, B](codec: Schema[A], f: A => Either[String, B], g: B => Either[String, A]) + extends Schema[B] { + override type Accessors[Lens[_, _], Prism[_, _], Traversal[_, _]] = codec.Accessors[Lens, Prism, Traversal] + + override def makeAccessors(b: AccessorBuilder): codec.Accessors[b.Lens, b.Prism, b.Traversal] = + codec.makeAccessors(b) + + override def serializable: Schema[Schema[_]] = Meta(SchemaAst.fromSchema(codec)) + override def toString: String = s"Transform($codec)" + } +``` + +In the previous example, we can transform the `User` Schema into a `UserRecord` Schema, which is a record, by using the `transform` method, which has to be an "isomorphism" (= providing methods to transform A to B _and_ B to A): + +```scala +/** + * Transforms this `Schema[A]` into a `Schema[B]`, by supplying two functions that can transform + * between `A` and `B`, without possibility of failure. + */ +def transform[B](f: A => B, g: B => A): Schema[B] = + Schema.Transform[A, B](self, a => Right(f(a)), b => Right(g(b))) +``` + diff --git a/docs/understanding-zio-schema.md b/docs/understanding-zio-schema.md deleted file mode 100644 index 0dd6e7999..000000000 --- a/docs/understanding-zio-schema.md +++ /dev/null @@ -1,335 +0,0 @@ ---- -id: understanding-zio-schema -title: "Understanding ZIO Schema" ---- - -ZIO Schema is a library used in many ZIO projects such as _ZIO Flow_, _ZIO Redis_, _ZIO Web_, _ZIO SQL_ and _ZIO DynamoDB_. -ZIO is all about reification of your types. Reification means transforming something abstract (e.g. side effects, accessing fields, structure) into something "real" (values). - -## Reification: Functional Effects - -In functional effects, we reify by turning side-effects into values. - -E.g. you might have a simple statement like -```scala -println("Hello") -println("World") -``` -and in ZIO we reify this statement to a value like - -```scala -val effect1 = Task(println("Hello")) -val effect2 = Task(println("World")) -``` - -and then are able to do awesome things like: -```scala -(Task(println("Hello")) zipPar Task(println("World"))).retryN(100) -``` - -## Reification: Optics - -In scala we have product types like this case class of a Person: -```scala -final case class Person(name: String, age: Int) -``` -This case class has two fields: -- a field "name" of type `String` -- a field "age" of type `Int` - -The Scala language provides special support to access the fields inside case classes using the dot syntax: - -```scala -val person = Person("Michelle", 32) -val name = person.name -val age = person.age -``` - -However, this is a "special language feature", it's not "real" like the side effects we've seen in the previous example ( `println(..) vs. Task`println(...)))` ). - -Because these basic operations are not "real", we're unable to create an operator that we can use to -e.g. combine two fields that are inside a nested structure. - -The solution to this kind of problem is called an "Optic". Optics provide a way to access the fields of a case class and nested structures. -There are three main types of optics: -- `Lens`: A lens is a way to access a field of a case class. -- `Prism`: A prism is a way to access a field of a nested structure or a collection. -- `Traversal`: A traversal is a way to access all fields of a case class, nested structures or collections. - -Optics allow us to take things which are not a first-class **concept**, and turn that into a first-class **value**, -namely the concept of -- drilling down into a field inside a case class or -- drilling down into a nested structure. - -Once we have a value, we can compose these things together to solve hard problems in functional programming, e.g. -- handling nested case class copies, -- iterations down deep inside on elements of a nested structure or collections - -For more information on optics, refer to the [ZIO Optics](https://zio.github.io/zio-optics/docs/overview/overview_index) documentation. - - -## Reification: Schema - -So far we've looked at how to -- reify side-effects into values (ZIO) -- how to reify accessing + modifying fields inside case classes or arbitrary structures by turning these operations into values as well (Optics) - -ZIO Schema is now about how to describe entire data structures using values. - -The "built-in" way in scala on how to describe data structures are `case classes` and `classes`. - -E.g. the following data type: -```scala -final case class Person(name: String, age: Int) -``` - -Has the following information: -- name of the structure: "Person" -- fields: "name" and "age" -- type of the fields: String and Int -- type of the structure: Person - -ZIO Schema tries to reify the concept of structure for datatypes by turning the above information into values. - -Not only for case classes, but also for other types like collections, tuples, enumerations etc. - -## Getting started - -To get started, first you need to understand that a ZIO Schema is basically built-up from these three -sealed traits: -- `Record[R]` -- `Enum[A]` -- `Sequence[Col, Elem]` - and the case class `Primitive[A]`. Every other type is just a specialisation of one of these (or not relevant to get you started). - -We will take a look at them now. - -### Basic Building Blocks - -#### Schema - -The core data type of ZIO Schema is a `Schema[A]` which is **invariant in `A`** by necessity, because a Schema allows you to -derive operations that produce an `A` but also operations that consume an `A` and that imposes limitations on the types of -**transformation operators** and **composition operators** that we can provide based on a `Schema`. - -It looks kind of like this (simplified): -```scala -sealed trait Schema[A] { - self => - type Accessors[Lens[_, _], Prism[_, _], Traversal[_, _]] - - /** - * A symbolic operator for [[optional]]. - */ - def ? : Schema[Option[A]] = self.optional - - def makeAccessors(b: AccessorBuilder): Accessors[b.Lens, b.Prism, b.Traversal] - - /** - * Transforms this `Schema[A]` into a `Schema[B]`, by supplying two functions that can transform - * between `A` and `B`, without possibility of failure. - */ - def transform[B](f: A => B, g: B => A): Schema[B] = - Schema.Transform[A, B](self, a => Right(f(a)), b => Right(g(b))) - - def transformOrFail[B](f: A => Either[String, B], g: B => Either[String, A]): Schema[B] = - Schema.Transform[A, B](self, f, g) - - def zip[B](that: Schema[B]): Schema[(A, B)] = Schema.Tuple(self, that) -} -``` - -#### Records - -Your data structures usually are composed from a lot of types. For example, you might have a `User` -type that has a `name` field, an `age` field, an `address` field, and a `friends` field. - -```scala -case class User(name: String, age: Int, address: Address, friends: List[User]) -``` - -This is called a "product type" in functional programming. -The equivalent of a product type in ZIO Schema is called a record. - -In ZIO Schema such a record would be represented using the `Record[R]` typeclass: - -```scala -object Schema { - sealed trait Record[R] extends Schema[R] { - def structure: Chunk[Field[_]] - def annotations: Chunk[Any] = Chunk.empty - def rawConstruct(values: Chunk[Any]): Either[String, R] - } -} - -``` - -#### Enumerations - -Other times, you might have a type that represents a list of different types. For example, you might -have a type, like this: -```scala - sealed trait PaymentMethod - - object PaymentMethod { - final case class CreditCard(number: String, expirationMonth: Int, expirationYear: Int) extends PaymentMethod - final case class WireTransfer(accountNumber: String, bankCode: String) extends PaymentMethod - } -``` - -In functional programming, this kind of type is called a "sum type". -In Scala2, this is called a "sealed trait". -In Scala3, this is called an "enum". - -In ZIO Schema we call these types `enumeration` types and they are -represented using the `Enum[A]` type class. - -```scala -object Schema ... { - sealed trait Enum[A] extends Schema[A] { - def annotations: Chunk[Any] - def structure: ListMap[String, Schema[_]] - } -} -``` - -#### Sequence - -Often you have a type that is a collection of elements. For example, you might have a `List[User]`. -This is called a `Sequence` and is represented using the `Sequence[Col, Elem]` type class: - -```scala -object Schema ... { - ... - - final case class Sequence[Col, Elem]( - elementSchema: Schema[Elem], - fromChunk: Chunk[Elem] => Col, - toChunk: Col => Chunk[Elem] - ) extends Schema[Col] { - self => - override type Accessors[Lens[_, _], Prism[_, _], Traversal[_, _]] = Traversal[Col, Elem] - override def makeAccessors(b: AccessorBuilder): b.Traversal[Col, Elem] = b.makeTraversal(self, schemaA) - override def toString: String = s"Sequence($elementSchema)" - } - ... -} -``` - -#### Optionals - -A special variant of a collection type is the `Optional[A]` type: - -```scala -object Schema ... { - - final case class Optional[A](codec: Schema[A]) extends Schema[Option[A]] { - self => - - private[schema] val someCodec: Schema[Some[A]] = codec.transform(a => Some(a), _.get) - - override type Accessors[Lens[_, _], Prism[_, _], Traversal[_, _]] = - (Prism[Option[A], Some[A]], Prism[Option[A], None.type]) - - val toEnum: Enum2[Some[A], None.type, Option[A]] = Enum2( - Case[Some[A], Option[A]]("Some", someCodec, _.asInstanceOf[Some[A]], Chunk.empty), - Case[None.type, Option[A]]("None", singleton(None), _.asInstanceOf[None.type], Chunk.empty), - Chunk.empty - ) - - override def makeAccessors(b: AccessorBuilder): (b.Prism[Option[A], Some[A]], b.Prism[Option[A], None.type]) = - b.makePrism(toEnum, toEnum.case1) -> b.makePrism(toEnum, toEnum.case2) - } - -} -``` - -#### Primitives - -Last but not least, we have primitive values. - -```scala -object Schema ... { - ... - final case class Primitive[A](standardType: StandardType[A]) extends Schema[A] { - type Accessors[Lens[_, _], Prism[_, _], Traversal[_, _]] = Unit - - override def makeAccessors(b: AccessorBuilder): Unit = () - } - ... -} -``` - -Primitive values are represented using the `Primitive[A]` type class and represent the elements, -that we cannot further define through other means. If you visualize your data structure as a tree, -primitives are the leaves. - -ZIO Schema provides a number of built-in primitive types, that you can use to represent your data. -These can be found in the `StandardType` companion-object. - -### Transforming Schemas - -Once we have a `Schema`, we can transform it into another `Schema` by applying a `Transformer`. -In normal Scala code this would be the equivalent of `map`. - -In ZIO Schema this is modelled by the `Transform` type class: - -```scala - final case class Transform[A, B](codec: Schema[A], f: A => Either[String, B], g: B => Either[String, A]) - extends Schema[B] { - override type Accessors[Lens[_, _], Prism[_, _], Traversal[_, _]] = codec.Accessors[Lens, Prism, Traversal] - - override def makeAccessors(b: AccessorBuilder): codec.Accessors[b.Lens, b.Prism, b.Traversal] = - codec.makeAccessors(b) - - override def serializable: Schema[Schema[_]] = Meta(SchemaAst.fromSchema(codec)) - override def toString: String = s"Transform($codec)" - } -``` - -In the example above, we can transform the `User` Schema into a `UserRecord` Schema, which is a record, -by using the `transform`-method, which has to be an "isomorphism" (= providing methods to transform A to B _and_ B to A): - -```scala - /** - * Transforms this `Schema[A]` into a `Schema[B]`, by supplying two functions that can transform - * between `A` and `B`, without possibility of failure. - */ - def transform[B](f: A => B, g: B => A): Schema[B] = - Schema.Transform[A, B](self, a => Right(f(a)), b => Right(g(b))) -``` - -#### Codecs - -Once you have your schema, you can combine it with a codec. -A codec is a combination of a schema and a serializer. -Unlike codecs in other libraries, a codec in ZIO Schema has no type parameter. - -```scala - -trait Codec { - def encoder[A](schema: Schema[A]): ZTransducer[Any, Nothing, A, Byte] - def decoder[A](schema: Schema[A]): ZTransducer[Any, String, Byte, A] - - def encode[A](schema: Schema[A]): A => Chunk[Byte] - def decode[A](schema: Schema[A]): Chunk[Byte] => Either[String, A] -} - -``` - -It basically says: -`encoder[A]`: Given a `Schema[A]` it is capable of generating an `Encoder[A]` ( `A => Chunk[Byte]`) for any Schema. -`decoder[A]`: Given a `Schema[A]` it is capable of generating a `Decoder[A]` ( `Chunk[Byte] => Either[String, A]`) for any Schema. - - -Example of possible codecs are: - - - CSV Codec - - JSON Codec (already available) - - Apache Avro Codec (in progress) - - Apache Thrift Codec (in progress) - - XML Codec - - YAML Codec - - Protobuf Codec (already available) - - QueryString Codec - - etc. diff --git a/docs/use-cases.md b/docs/use-cases.md index ba7f69df1..099585690 100644 --- a/docs/use-cases.md +++ b/docs/use-cases.md @@ -4,15 +4,15 @@ title: "ZIO Schema Use cases" sidebar_label: "Use cases" --- -ZIO Schema allows you to create representations of your data types as values. +ZIO Schema allows us to create representations of our data types as values. -Once you have a representation of your data types, you can use it to - - serialize and deserialize your types - - validate your types - - transform your types - - create instances of your types +Once we have a representation of our data types, we can use it to + - Serialize and deserialize our types + - Validate our types + - Transform our types + - Create instances of your types -You can then use one of the various codecs (or create your own) to serialize and deserialize your types. +We can then use one of the various codecs (or create our own) to serialize and deserialize your types. Example of possible codecs are: @@ -39,8 +39,8 @@ Example use cases that are possible: - Creating diffs from arbitrary data structures - Creating migrations / evolutions e.g. of Events used in Event-Sourcing - Transformation pipelines, e.g. - 1. convert from protobuf to object, e.g. `PersonDTO`, - 2. transform to another representation, e.g. `Person`, - 3. validate - 4. transform to JSON `JsonObject` - 5. serialize to `String` + 1. Convert from protobuf to object, e.g. `PersonDTO`, + 2. Transform to another representation, e.g. `Person`, + 3. Validate + 4. Transform to JSON `JsonObject` + 5. Serialize to `String` From 6482280cff818027565cc51931997a3394a23db4 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Wed, 5 Jul 2023 16:33:40 +0330 Subject: [PATCH 02/63] add manual and automatic derivation. --- build.sbt | 1 + docs/automatic-schema-derivation.md | 91 +++++++++++++ docs/manual-schema-construction.md | 192 ++++++++++++++++++++++++++++ docs/our-first-schema.md | 129 ------------------- docs/sidebars.js | 21 ++- 5 files changed, 299 insertions(+), 135 deletions(-) create mode 100644 docs/automatic-schema-derivation.md create mode 100644 docs/manual-schema-construction.md delete mode 100644 docs/our-first-schema.md diff --git a/build.sbt b/build.sbt index cb32d750c..b81d8a290 100644 --- a/build.sbt +++ b/build.sbt @@ -341,4 +341,5 @@ lazy val docs = project |sbt test |```""".stripMargin ) + .dependsOn(zioSchemaJVM, zioSchemaProtobufJVM) .enablePlugins(WebsitePlugin) diff --git a/docs/automatic-schema-derivation.md b/docs/automatic-schema-derivation.md new file mode 100644 index 000000000..b057ac814 --- /dev/null +++ b/docs/automatic-schema-derivation.md @@ -0,0 +1,91 @@ +--- +id: automatic-schema-derivation +title: "Automatic Schema Derivation" +--- + +Automatic schema derivation is the process of generating schema definitions for data types automatically, without the need to manually write them. It allows us to generate the schema for a data type based on its structure and annotations. + +Instead of manually specifying the schema for each data type, we can rely on automatic schema derivation to generate the schema for us. This approach can save time and reduce the potential for errors, especially when dealing with complex data models. + +By leveraging reflection and type introspection using macros, automatic schema derivation analyzes the structure of the data type and its fields, including their names, types, and annotations. It then generates the corresponding schema definition based on this analysis. + +ZIO streamlines schema derivation through its `zio-schema-derivation` package, which utilizes the capabilities of Scala macros to automatically derive schemas. In order to use automatic schema derivation, we neeed to add the following line to our `build.sbt` file: + +```scala +libraryDependencies += "dev.zio" %% "zio-schema-derivation" % @VERSION@ +``` + +Once again, let's revisit our domain models: + +```scala mdoc:compile-only +final case class Person(name: String, age: Int) + +sealed trait PaymentMethod + +object PaymentMethod { + final case class CreditCard(number: String, expirationMonth: Int, expirationYear: Int) extends PaymentMethod + final case class WireTransfer(accountNumber: String, bankCode: String) extends PaymentMethod +} + +final case class Customer(person: Person, paymentMethod: PaymentMethod) +``` + +We can easily use auto derivation to create schemas: + +```scala +import zio.schema._ +import zio.schema.codec._ + +final case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = DeriveSchema.gen[Person] +} + +sealed trait PaymentMethod + +object PaymentMethod { + + implicit val schema: Schema[PaymentMethod] = + DeriveSchema.gen[PaymentMethod] + + final case class CreditCard( + number: String, + expirationMonth: Int, + expirationYear: Int + ) extends PaymentMethod + + final case class WireTransfer(accountNumber: String, bankCode: String) + extends PaymentMethod +} + +final case class Customer(person: Person, paymentMethod: PaymentMethod) + +object Customer { + implicit val schema: Schema[Customer] = DeriveSchema.gen[Customer] +} +``` + +Now we can write an example that demonstrates a roundtrip test for protobuf codecs: + +```scala +// Create a customer instance +val customer = + Customer( + person = Person("John Doe", 42), + paymentMethod = PaymentMethod.CreditCard("1000100010001000", 6, 2024) + ) + +// Create binary codec from customer +val customerCodec: BinaryCodec[Customer] = + ProtobufCodec.protobufCodec[Customer] + +// Encode the customer object +val encodedCustomer: Chunk[Byte] = customerCodec.encode(customer) + +// Decode the byte array back to the person instance +val decodedCustomer: Either[DecodeError, Customer] = + customerCodec.decode(encodedCustomer) + +assert(Right(customer) == decodedCustomer) +``` diff --git a/docs/manual-schema-construction.md b/docs/manual-schema-construction.md new file mode 100644 index 000000000..29c0848fc --- /dev/null +++ b/docs/manual-schema-construction.md @@ -0,0 +1,192 @@ +--- +id: manual-schema-construction +title: "Manual Schema Construction" +--- + +Assume we have a domain containing following models: + +```scala +object Domain { + final case class Person(name: String, age: Int) + + sealed trait PaymentMethod + + object PaymentMethod { + final case class CreditCard(number: String, expirationMonth: Int, expirationYear: Int) extends PaymentMethod + final case class WireTransfer(accountNumber: String, bankCode: String) extends PaymentMethod + } + + final case class Customer(person: Person, paymentMethod: PaymentMethod) + +} +``` + +Let's begin by creating a schema for the `Person` data type: + +```scala mdoc:silent +import zio.schema._ + +final case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = + Schema.CaseClass2[String, Int, Person]( + id0 = TypeId.fromTypeName("Person"), + field01 = Schema.Field(name0 = "name", schema0 = Schema[String], get0 = _.name, set0 = (p, x) => p.copy(name = x)), + field02 = Schema.Field(name0 = "age", schema0 = Schema[Int], get0 = _.age, set0 = (person, age) => person.copy(age = age)), + construct0 = (name, age) => Person(name, age), + ) +} +``` + +The next step is writing schema for `PaymentMethod`: + +```scala mdoc:silent +import zio._ +import zio.schema._ + +sealed trait PaymentMethod + +object PaymentMethod { + implicit val schema: Schema[PaymentMethod] = + Schema.Enum2[CreditCard, WireTransfer, PaymentMethod]( + id = TypeId.fromTypeName("PaymentMethod"), + case1 = Schema.Case[PaymentMethod, CreditCard]( + id = "CreditCard", + schema = CreditCard.schema, + unsafeDeconstruct = pm => pm.asInstanceOf[PaymentMethod.CreditCard], + construct = cc => cc.asInstanceOf[PaymentMethod], + isCase = _.isInstanceOf[PaymentMethod.CreditCard], + annotations = Chunk.empty + ), + case2 = Schema.Case[PaymentMethod, WireTransfer]( + id = "WireTransfer", + schema = WireTransfer.schema, + unsafeDeconstruct = pm => pm.asInstanceOf[PaymentMethod.WireTransfer], + construct = wt => wt.asInstanceOf[PaymentMethod], + isCase = _.isInstanceOf[PaymentMethod.WireTransfer], + annotations = Chunk.empty + ) + ) + + final case class CreditCard( + number: String, + expirationMonth: Int, + expirationYear: Int + ) extends PaymentMethod + + object CreditCard { + implicit val schema: Schema[CreditCard] = + Schema.CaseClass3[String, Int, Int, CreditCard]( + id0 = TypeId.fromTypeName("CreditCard"), + field01 = Schema.Field[CreditCard, String]( + name0 = "number", + schema0 = Schema.primitive[String], + get0 = _.number, + set0 = (cc, n) => cc.copy(number = n) + ), + field02 = Schema.Field[CreditCard, Int]( + name0 = "expirationMonth", + schema0 = Schema.primitive[Int], + get0 = _.expirationMonth, + set0 = (cc, em) => cc.copy(expirationMonth = em) + ), + field03 = Schema.Field[CreditCard, Int]( + name0 = "expirationYear", + schema0 = Schema.primitive[Int], + get0 = _.expirationYear, + set0 = (cc, ey) => cc.copy(expirationYear = ey) + ), + construct0 = (n, em, ey) => CreditCard(n, em, ey) + ) + } + + final case class WireTransfer(accountNumber: String, bankCode: String) + extends PaymentMethod + + object WireTransfer { + implicit val schema: Schema[WireTransfer] = + Schema.CaseClass2[String, String, WireTransfer]( + id0 = TypeId.fromTypeName("WireTransfer"), + field01 = Schema.Field[WireTransfer, String]( + name0 = "accountNumber", + schema0 = Schema.primitive[String], + get0 = _.accountNumber, + set0 = (wt, an) => wt.copy(accountNumber = an) + ), + field02 = Schema.Field[WireTransfer, String]( + name0 = "bankCode", + schema0 = Schema.primitive[String], + get0 = _.bankCode, + set0 = (wt, bc) => wt.copy(bankCode = bc) + ), + construct0 = (ac, bc) => WireTransfer(ac, bc) + ) + } +} +``` + +And finally, we need to define the schema for the `Customer` data type: + +```scala mdoc:silent +import zio._ +import zio.schema._ + +final case class Customer(person: Person, paymentMethod: PaymentMethod) + +object Customer { + implicit val schema: Schema[Customer] = + Schema.CaseClass2[Person, PaymentMethod, Customer]( + id0 = TypeId.fromTypeName("Customer"), + field01 = Schema.Field[Customer, Person]( + name0 = "person", + schema0 = Person.schema, + get0 = _.person, + set0 = (c, p) => c.copy(person = p) + ), + field02 = Schema.Field[Customer, PaymentMethod]( + name0 = "paymentMethod", + schema0 = Schema[PaymentMethod], + get0 = _.paymentMethod, + set0 = (c, pm) => c.copy(paymentMethod = pm) + ), + construct0 = (p, pm) => Customer(p, pm) + ) +} +``` + +Now that we have written all the required schemas, we can proceed to create encoders and decoders (codecs) for each of our domain models. + +Let's start with writing protobuf codecs. We need to add the following line to our `build.sbt`: + +```scala +libraryDependencies += "dev.zio" %% "zio-schema-protobuf" % @VERSION@ +``` + +Here's an example that demonstrates a roundtrip test for protobuf codecs: + +```scala mdoc:silent +import zio.schema._ +import zio.schema.codec._ +import zio.schema.codec.ProtobufCodec._ + +// Create a customer instance +val customer = + Customer( + person = Person("John Doe", 42), + paymentMethod = PaymentMethod.CreditCard("1000100010001000", 6, 2024) + ) + +// Create binary codec from customer +val customerCodec: BinaryCodec[Customer] = + ProtobufCodec.protobufCodec[Customer] + +// Encode the customer object +val encodedCustomer: Chunk[Byte] = customerCodec.encode(customer) + +// Decode the byte array back to the person instance +val decodedCustomer: Either[DecodeError, Customer] = + customerCodec.decode(encodedCustomer) + +assert(Right(customer) == decodedCustomer) +``` diff --git a/docs/our-first-schema.md b/docs/our-first-schema.md deleted file mode 100644 index 6a2b7eb53..000000000 --- a/docs/our-first-schema.md +++ /dev/null @@ -1,129 +0,0 @@ ---- -id: our-first-schema -title: "Our First Schema" ---- - -ZIO Schema provides macros to help you create `Schema`s out of our data types. But before using the macros, we should take a look at how to do this the manual way. - -## The Domain - -As described in the [Overview](index.md) section, we define the example domain as follows: - -```scala -object Domain { - final case class Person(name: String, age: Int) - - sealed trait PaymentMethod - - object PaymentMethod { - final case class CreditCard(number: String, expirationMonth: Int, expirationYear: Int) extends PaymentMethod - final case class WireTransfer(accountNumber: String, bankCode: String) extends PaymentMethod - } - - final case class Customer(person: Person, paymentMethod: PaymentMethod) - -} -``` - -### Manual construction of a Schema - -This part is similar to other libraries that we might know, e.g. for JSON processing. Basically, we create a `Schema` for every data type in our domain: - -```scala -object ManualConstruction { - import zio.schema.Schema._ - import Domain._ - import Domain.PaymentMethod._ - - val schemaPerson: Schema[Person] = Schema.CaseClass2[String, Int, Person]( - field1 = Schema.Field[String]("name", Schema.primitive[String]), - field2 = Schema.Field[Int]("age", Schema.primitive[Int]), - construct = (name, age) => Person(name, age), - extractField1 = p => p.name, - extractField2 = p => p.age - ) - - val schemaPaymentMethodWireTransfer: Schema[WireTransfer] = Schema.CaseClass2[String, String, WireTransfer]( - field1 = Schema.Field[String]("accountNumber", Schema.primitive[String]), - field2 = Schema.Field[String]("bankCode", Schema.primitive[String]), - construct = (number, bankCode) => PaymentMethod.WireTransfer(number, bankCode), - extractField1 = p => p.accountNumber, - extractField2 = p => p.bankCode - ) - - val schemaPaymentMethodCreditCard: Schema[CreditCard] = Schema.CaseClass3[String, Int, Int, CreditCard]( - field1 = Schema.Field[String]("number", Schema.primitive[String]), - field2 = Schema.Field[Int]("expirationMonth", Schema.primitive[Int]), - field3 = Schema.Field[Int]("expirationYear", Schema.primitive[Int]), - construct = (number, expirationMonth, expirationYear) => PaymentMethod.CreditCard(number, expirationMonth, expirationYear), - extractField1 = p => p.number, - extractField2 = p => p.expirationMonth, - extractField3 = p => p.expirationYear - ) - - val schemaPaymentMethod: Schema[PaymentMethod] = Schema.Enum2( - case1 = Case[PaymentMethod.CreditCard, PaymentMethod]( - id = "CreditCard", - codec = schemaPaymentMethodCreditCard, - unsafeDeconstruct = pm => pm.asInstanceOf[PaymentMethod.CreditCard] - ), - case2 = Case[PaymentMethod.WireTransfer, PaymentMethod]( - id = "WireTransfer", - codec = schemaPaymentMethodWireTransfer, - unsafeDeconstruct = pm => pm.asInstanceOf[PaymentMethod.WireTransfer] - ) - ) - - val schemaCustomer: Schema[Customer] = Schema.CaseClass2[Person, PaymentMethod, Customer]( - field1 = Schema.Field[Person]("person", schemaPerson), - field2 = Schema.Field[PaymentMethod]("paymentMethod", schemaPaymentMethod), - construct = (person, paymentMethod) => Customer(person, paymentMethod), - extractField1 = c => c.person, - extractField2 = c => c.paymentMethod - ) -} -``` - -### Macro derivation - -Using macros, the above code gets reduced to this: - -```scala -object MacroConstruction { - import Domain._ - - val schemaPerson: Schema[Person] = DeriveSchema.gen[Person] - - val schemaPaymentMethod: Schema[PaymentMethod] = DeriveSchema.gen[PaymentMethod] - - val schemaCustomer: Schema[Customer] = DeriveSchema.gen[Customer] -} -``` - -## Applying it to Our Domain - -Let's put this all together in a small sample: - -```scala -object JsonSample extends zio.App { - import zio.schema.codec.JsonCodec - import ManualConstruction._ - import zio.stream.ZStream - - override def run(args: List[String]): UIO[ExitCode] = for { - _ <- ZIO.unit - person = Person("Michelle", 32) - personToJsonTransducer = JsonCodec.encoder[Person](schemaPerson) - _ <- ZStream(person) - .transduce(personToJsonTransducer) - .transduce(ZTransducer.utf8Decode) - .foreach(ZIO.debug) - } yield ExitCode.success -} -``` - -When we run this, we get our expected result printed out: - -```json -{"name":"Michelle","age":32} -``` diff --git a/docs/sidebars.js b/docs/sidebars.js index 3d3940693..bf99c66cc 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -5,18 +5,27 @@ const sidebars = { label: "ZIO Schema", collapsed: false, link: { type: "doc", id: "index" }, - items: [ + items: [ "use-cases", - "our-first-schema", + { + type: "category", + label: "Writing Schema", + collapsed: true, + link: { type: "doc", id: "index" }, + items: [ + "manual-schema-construction", + "automatic-schema-derivation" + ], + }, "motivation", "getting-started", "transforming-schemas", "codecs", "protobuf-example", - "combining-different-encoders" - ] - } - ] + "combining-different-encoders", + ], + }, + ], }; module.exports = sidebars; From 21ea28668bdf38abe298a137835da74d8ef6ac76 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Sat, 8 Jul 2023 21:26:20 +0330 Subject: [PATCH 03/63] integration with zio streams. --- build.sbt | 2 +- docs/integration-with-zio-streams.md | 37 ++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 docs/integration-with-zio-streams.md diff --git a/build.sbt b/build.sbt index b81d8a290..1f40907fe 100644 --- a/build.sbt +++ b/build.sbt @@ -341,5 +341,5 @@ lazy val docs = project |sbt test |```""".stripMargin ) - .dependsOn(zioSchemaJVM, zioSchemaProtobufJVM) + .dependsOn(zioSchemaJVM, zioSchemaProtobufJVM, zioSchemaJsonJVM) .enablePlugins(WebsitePlugin) diff --git a/docs/integration-with-zio-streams.md b/docs/integration-with-zio-streams.md new file mode 100644 index 000000000..7335a5463 --- /dev/null +++ b/docs/integration-with-zio-streams.md @@ -0,0 +1,37 @@ +--- +id: integration-with-zio-streams +title: "Integration with ZIO Streams" +--- + +In addition to the regular `encode` and `decode` functions, each codec also has a streaming version of these functions called `streamEncoder` and `streamDecoder`. By invoking these methods on codecs, we can obtain a `ZPipeline` where the encoder and decoder are integrated into the `ZPipeline` stream transformer. + +We can use the `ZPipline` to transform (encode/decode) a stream of values of type `A` into a stream of values of type `B`. + +For example, assume we have a stream of `Person` values, and we want to encode them into a stream of bytes and then convert back to `Person` values. We can do this as follows: + +```scala mdoc:compile-only +import zio._ +import zio.stream._ +import zio.schema._ +import zio.schema.codec.JsonCodec + +object Main extends ZIOAppDefault { + case class Person(name: String, age: Int) + + object Person { + implicit val schema: Schema[Person] = DeriveSchema.gen[Person] + } + + def run = + ZStream + .fromIterable(Seq(Person("John", 42))) + .debug("the input object is") + .via(JsonCodec.schemaBasedBinaryCodec[Person].streamEncoder) + .via(ZPipeline.utfDecode) + .debug("json string of person") + .via(ZPipeline.utf8Encode) + .via(JsonCodec.schemaBasedBinaryCodec[Person].streamDecoder) + .debug("person after roundtrip") + .runDrain +} +``` \ No newline at end of file From 0a3e44fbfe30c485116a433d34c0350f9cd77ade Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Sat, 8 Jul 2023 21:42:31 +0330 Subject: [PATCH 04/63] add more packages to installation section. --- docs/index.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index f55322e4d..d516b4930 100644 --- a/docs/index.md +++ b/docs/index.md @@ -31,12 +31,15 @@ In order to use this library, we need to add the following lines in our `build.s ```scala libraryDependencies += "dev.zio" %% "zio-schema" % "@VERSION@" +libraryDependencies += "dev.zio" %% "zio-schema-avro" % "@VERSION@" libraryDependencies += "dev.zio" %% "zio-schema-bson" % "@VERSION@" libraryDependencies += "dev.zio" %% "zio-schema-json" % "@VERSION@" +libraryDependencies += "dev.zio" %% "zio-schema-msg-pack" % "@VERSION@" libraryDependencies += "dev.zio" %% "zio-schema-protobuf" % "@VERSION@" +libraryDependencies += "dev.zio" %% "zio-schema-thrift" % "@VERSION@" // Required for automatic generic derivation of schemas -libraryDependencies += "dev.zio" %% "zio-schema-derivation" % "@VERSION@", +libraryDependencies += "dev.zio" %% "zio-schema-derivation" % "@VERSION@" libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value % "provided" ``` From 751e1197a3474dbc783f9bea17b49e1a28ebf5de Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Sun, 9 Jul 2023 17:00:02 +0330 Subject: [PATCH 05/63] add comment for fail constructor. --- zio-schema/shared/src/main/scala/zio/schema/Schema.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zio-schema/shared/src/main/scala/zio/schema/Schema.scala b/zio-schema/shared/src/main/scala/zio/schema/Schema.scala index 26eb6c168..fa275fc77 100644 --- a/zio-schema/shared/src/main/scala/zio/schema/Schema.scala +++ b/zio-schema/shared/src/main/scala/zio/schema/Schema.scala @@ -164,6 +164,9 @@ object Schema extends SchemaEquality { def enumeration[A, C <: CaseSet.Aux[A]](id: TypeId, caseSet: C): Schema[A] = EnumN(id, caseSet, Chunk.empty) + /** + * Represents the absence of schema information for the given `A` type. + */ def fail[A](message: String): Schema[A] = Fail(message) def first[A](schema: Schema[(A, Unit)]): Schema[A] = From 8e8e32bfe2f3eb086da1e8e02d89a4567b5235af Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Sun, 9 Jul 2023 18:26:08 +0330 Subject: [PATCH 06/63] primitives. --- docs/getting-started.md | 56 +++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 98503ef63..cdc7420f0 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -24,6 +24,43 @@ sealed trait Schema[A] { self => } ``` +### Primitives + +To describe scalar data type `A`, we use the `Primitive[A]` data type which basically is a wrapper around `StandardType`: + +```scala +case class Primitive[A](standardType: StandardType[A]) extends Schema[A] +``` + +Primitive values are represented using the `Primitive[A]` type class and represent the elements, that we cannot further define through other means. If we visualize our data structure as a tree, primitives are the leaves. + +ZIO Schema provides a number of built-in primitive types, that we can use to represent our data. These can be found in the [`StandardType`](https://github.com/zio/zio-schema/blob/main/zio-schema/shared/src/main/scala/zio/schema/StandardType.scala) companion-object: + +```scala +sealed trait StandardType[A] +object StandardType { + implicit object UnitType extends StandardType[Unit] + implicit object StringType extends StandardType[String] + implicit object BoolType extends StandardType[Boolean] + // ... +} +``` + +Inside `Schema`'s companion object, we have an implicit conversion from `StandardType[A]` to `Schema[A]`: + +```scala +object Schema { + implicit def primitive[A](implicit standardType: StandardType[A]): Schema[A] = ??? +} +``` + +So we can easily create a `Schema` for a primitive type `A` either by calling `Schema.primitive[A]` or by calling `Schema.apply[A]`: + +```scala +val intSchema1: Schema[Int] = Schema[Int] +val intSchema2: Schema[Int] = Schema.primitive[Int] +``` + ### Records Our data structures usually are composed of a lot of types. For example, we might have a `User` @@ -124,23 +161,4 @@ object Schema extends SchemaEquality { } ``` -### Primitives - -Last but not least, we have primitive values. - -```scala -object Schema extends SchemaEquality { - final case class Primitive[A](standardType: StandardType[A]) extends Schema[A] { - type Accessors[Lens[_, _], Prism[_, _], Traversal[_, _]] = Unit - - override def makeAccessors(b: AccessorBuilder): Unit = () - } -} -``` - -Primitive values are represented using the `Primitive[A]` type class and represent the elements, -that we cannot further define through other means. If we visualize our data structure as a tree, primitives are the leaves. - -ZIO Schema provides a number of built-in primitive types, that we can use to represent our data. -These can be found in the `StandardType` companion-object. From 47e3fcbc92a34c5811dd8d22d2493d92cd538707 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Sun, 9 Jul 2023 19:35:14 +0330 Subject: [PATCH 07/63] transforming schemas. --- docs/transforming-schemas.md | 55 +++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/docs/transforming-schemas.md b/docs/transforming-schemas.md index 40d1263e8..09faaa9bb 100644 --- a/docs/transforming-schemas.md +++ b/docs/transforming-schemas.md @@ -3,31 +3,48 @@ id: transforming-schemas title: "Transforming Schemas" --- -Once we have a `Schema`, we can transform it into another `Schema` by applying a `Transformer`. In normal Scala code this would be the equivalent of `map`. - -In ZIO Schema this is modelled by the `Transform` type class: +Using the `Schema#transform` method, we can transform a `Schema[A]` into a `Schema[B]` by supplying two functions that can transform between `A` and `B`. In normal Scala code this would be the equivalent of `map`, but with isomorphism property. ```scala - final case class Transform[A, B](codec: Schema[A], f: A => Either[String, B], g: B => Either[String, A]) - extends Schema[B] { - override type Accessors[Lens[_, _], Prism[_, _], Traversal[_, _]] = codec.Accessors[Lens, Prism, Traversal] +object Schema { + def transform[B](f: A => B, g: B => A): Schema[B] = ??? +} +``` - override def makeAccessors(b: AccessorBuilder): codec.Accessors[b.Lens, b.Prism, b.Traversal] = - codec.makeAccessors(b) +Therefore, if we have a schema for `A`, and isomorphism between `A` and `B`, we can derive a schema for `B` in terms of `Schema[A]. - override def serializable: Schema[Schema[_]] = Meta(SchemaAst.fromSchema(codec)) - override def toString: String = s"Transform($codec)" - } -``` +:::note +In type theory, isomorphism refers to a relationship between two types that have a bijective correspondence or mapping between their elements. More specifically, if two types, let's say Type `A` and Type `B`, are isomorphic, it means that there exists a pair of functions—one going from A to B (often called the forward function) and another going from B to A (often called the backward function)—that satisfy certain properties. +::: -In the previous example, we can transform the `User` Schema into a `UserRecord` Schema, which is a record, by using the `transform` method, which has to be an "isomorphism" (= providing methods to transform A to B _and_ B to A): +In ZIO Schema this is modelled by the `Transform` type class: ```scala -/** - * Transforms this `Schema[A]` into a `Schema[B]`, by supplying two functions that can transform - * between `A` and `B`, without possibility of failure. - */ -def transform[B](f: A => B, g: B => A): Schema[B] = - Schema.Transform[A, B](self, a => Right(f(a)), b => Right(g(b))) +object Schema { + final case class Transform[A, B]( + codec: Schema[A], + f: A => Either[String, B], + g: B => Either[String, A] + ) extends Schema[B] +} ``` +For example, assume we have a wrapper class `Age` that wraps an `Int` value, and it has some validation logic, e.g. the age must be between 0 and 120. We can define a `Schema[Age]` by using the `Schema.transform` method: + +```scala mdoc:compile-only +import zio.schema._ + +case class Age(i: Int) + +object Age { + implicit val schema: Schema[Age] = + Schema[Int].transformOrFail( + (i: Int) => + if (i >= 0 && i <= 120) + Right(Age(i)) + else + Left("Age must be between 1 and 120"), + (age: Age) => Right(age.i) + ) +} +``` From d3bf9b98ae71549b0264bd91ebd0d517bcab69d9 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Sun, 9 Jul 2023 20:06:26 +0330 Subject: [PATCH 08/63] sequence section. --- docs/getting-started.md | 45 +++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index cdc7420f0..e8f000657 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -115,22 +115,45 @@ object Schema extends SchemaEquality { ### Sequence -Often you have a type that is a collection of elements. For example, you might have a `List[User]`. This is called a `Sequence` and is represented using the `Sequence[Col, Elem]` type class: +Often we have a type that is a collection of elements. For example, we might have a `List[User]`. This is called a `Sequence` and is represented using the `Sequence[Col, Elem, I]` type class: ```scala -object Schema extends SchemaEquality { - - final case class Sequence[Col, Elem]( +object Schema { + sealed trait Collection[Col, Elem] extends Schema[Col] + + final case class Sequence[Col, Elem, I]( elementSchema: Schema[Elem], fromChunk: Chunk[Elem] => Col, - toChunk: Col => Chunk[Elem] - ) extends Schema[Col] { - self => - override type Accessors[Lens[_, _], Prism[_, _], Traversal[_, _]] = Traversal[Col, Elem] - override def makeAccessors(b: AccessorBuilder): b.Traversal[Col, Elem] = b.makeTraversal(self, schemaA) - override def toString: String = s"Sequence($elementSchema)" - } + toChunk: Col => Chunk[Elem], + override val annotations: Chunk[Any] = Chunk.empty, + identity: I + ) extends Collection[Col, Elem] +} +``` + +The `Sequence` can be anything that can be isomorphic to a list. + +Here is an example schema for list of `Person`s: + +```scala mdoc:compile-only +import zio._ +import zio.schema._ +import zio.schema.Schema._ + +case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = DeriveSchema.gen[Person] } + +val personListSchema: Schema[List[Person]] = + Sequence[List[Person], Person, String]( + elementSchema = Schema[Person], + fromChunk = _.toList, + toChunk = i => Chunk.fromIterable(i), + annotations = Chunk.empty, + identity = "List" + ) ``` ### Optionals From 88fdf3c9d8f32fbfa00feb54cf7c542b79b27021 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 10 Jul 2023 11:42:54 +0330 Subject: [PATCH 09/63] map collection. --- docs/getting-started.md | 136 +++++++++++++++++++++++++++------------- 1 file changed, 94 insertions(+), 42 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index e8f000657..fc516230d 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -61,6 +61,100 @@ val intSchema1: Schema[Int] = Schema[Int] val intSchema2: Schema[Int] = Schema.primitive[Int] ``` +### Collections + +#### Sequence + +Often we have a type that is a collection of elements. For example, we might have a `List[User]`. This is called a `Sequence` and is represented using the `Sequence[Col, Elem, I]` type class: + +```scala +object Schema { + sealed trait Collection[Col, Elem] extends Schema[Col] + + final case class Sequence[Col, Elem, I]( + elementSchema: Schema[Elem], + fromChunk: Chunk[Elem] => Col, + toChunk: Col => Chunk[Elem], + override val annotations: Chunk[Any] = Chunk.empty, + identity: I + ) extends Collection[Col, Elem] +} +``` + +The `Sequence` can be anything that can be isomorphic to a list. + +Here is an example schema for list of `Person`s: + +```scala mdoc:compile-only +import zio._ +import zio.schema._ +import zio.schema.Schema._ + +case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = DeriveSchema.gen[Person] +} + +val personListSchema: Schema[List[Person]] = + Sequence[List[Person], Person, String]( + elementSchema = Schema[Person], + fromChunk = _.toList, + toChunk = i => Chunk.fromIterable(i), + annotations = Chunk.empty, + identity = "List" + ) +``` + +ZIO Schema has a helper method `Schema.list[A]` that creates a `Schema[List[A]]` for us: + +```scala mdoc:compile-only +import zio._ +import zio.schema._ +import zio.schema.Schema._ + +case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = DeriveSchema.gen[Person] + + implicit val listSchema: Schema[List[Person]] = Schema.list[Person] +} + +``` + +#### Map + +Likewise, we can have a type that is a map of keys to values. ZIO Schema represents this using the following type class: + +```scala +object Schema { + sealed trait Collection[Col, Elem] extends Schema[Col] + + case class Map[K, V]( + keySchema: Schema[K], + valueSchema: Schema[V], + override val annotations: Chunk[Any] = Chunk.empty + ) extends Collection[scala.collection.immutable.Map[K, V], (K, V)] +} +``` + +It stores the key and value schemas. Like `Sequence`, instead of using `Map` directly, we can use the helper method `Schema.map[K, V]`: + +```scala mdoc:compile-only +import zio._ +import zio.schema._ +import zio.schema.Schema._ + +case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = DeriveSchema.gen[Person] + + implicit val mapSchema: Schema[Map[String, Person]] = Schema.map[String, Person] +} +``` + ### Records Our data structures usually are composed of a lot of types. For example, we might have a `User` @@ -113,48 +207,6 @@ object Schema extends SchemaEquality { } ``` -### Sequence - -Often we have a type that is a collection of elements. For example, we might have a `List[User]`. This is called a `Sequence` and is represented using the `Sequence[Col, Elem, I]` type class: - -```scala -object Schema { - sealed trait Collection[Col, Elem] extends Schema[Col] - - final case class Sequence[Col, Elem, I]( - elementSchema: Schema[Elem], - fromChunk: Chunk[Elem] => Col, - toChunk: Col => Chunk[Elem], - override val annotations: Chunk[Any] = Chunk.empty, - identity: I - ) extends Collection[Col, Elem] -} -``` - -The `Sequence` can be anything that can be isomorphic to a list. - -Here is an example schema for list of `Person`s: - -```scala mdoc:compile-only -import zio._ -import zio.schema._ -import zio.schema.Schema._ - -case class Person(name: String, age: Int) - -object Person { - implicit val schema: Schema[Person] = DeriveSchema.gen[Person] -} - -val personListSchema: Schema[List[Person]] = - Sequence[List[Person], Person, String]( - elementSchema = Schema[Person], - fromChunk = _.toList, - toChunk = i => Chunk.fromIterable(i), - annotations = Chunk.empty, - identity = "List" - ) -``` ### Optionals From 6c91333fbacae2f1411e73d073d2d6937f08d9d0 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 10 Jul 2023 12:00:37 +0330 Subject: [PATCH 10/63] schema for set collection. --- docs/getting-started.md | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index fc516230d..134932311 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -120,7 +120,6 @@ object Person { implicit val listSchema: Schema[List[Person]] = Schema.list[Person] } - ``` #### Map @@ -155,6 +154,37 @@ object Person { } ``` +#### Set + +The `Set` type class is similar to `Sequence` and `Map`. It is used to represent a schema for a set of elements: + +```scala +object Schema { + sealed trait Collection[Col, Elem] extends Schema[Col] + + case class Set[A]( + elementSchema: Schema[A], + override val annotations: Chunk[Any] = Chunk.empty + ) extends Collection[scala.collection.immutable.Set[A], A] +} +``` + +To create a `Schema` for a `Set[A]`, we can use the above type class directly or use the helper method `Schema.set[A]`: + +```scala mdoc:compile-only +import zio._ +import zio.schema._ +import zio.schema.Schema._ + +case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = DeriveSchema.gen[Person] + + implicit val setSchema: Schema[Set[Person]] = Schema.set[Person] +} +``` + ### Records Our data structures usually are composed of a lot of types. For example, we might have a `User` From 1bd2befcaea250472f960ed2bc2a6f5bb5c3a21d Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 10 Jul 2023 12:14:12 +0330 Subject: [PATCH 11/63] record section. --- docs/getting-started.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 134932311..f2a45b086 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -187,8 +187,7 @@ object Person { ### Records -Our data structures usually are composed of a lot of types. For example, we might have a `User` -type that has a `name` field, an `age` field, an `address` field, and a `friends` field. +Our data structures usually are composed of a lot of types. For example, we might have a `User` type that has a `name` field, an `age` field, an `address` field, and a `friends` field. ```scala case class User(name: String, age: Int, address: Address, friends: List[User]) @@ -201,9 +200,9 @@ In ZIO Schema such a record would be represented using the `Record[R]` typeclass ```scala object Schema { sealed trait Record[R] extends Schema[R] { + def id: TypeId def fields: Chunk[Field[_]] - def construct(value: R): Chunk[Any] - def defaultValue: Either[String, R] + def construct(fieldValues: Chunk[Any]): Either[String, R] } } @@ -229,7 +228,7 @@ In functional programming, this kind of type is called a **sum type**: In ZIO Schema we call these types `enumeration` types, and they are represented using the `Enum[A]` type class. ```scala -object Schema extends SchemaEquality { +object Schema { sealed trait Enum[A] extends Schema[A] { def annotations: Chunk[Any] def structure: ListMap[String, Schema[_]] From 751fa38fa9b3b8c31da3b7c4cc8568b2a66df04c Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 10 Jul 2023 12:51:45 +0330 Subject: [PATCH 12/63] add case class section. --- docs/getting-started.md | 56 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index f2a45b086..144ce8068 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -150,7 +150,8 @@ case class Person(name: String, age: Int) object Person { implicit val schema: Schema[Person] = DeriveSchema.gen[Person] - implicit val mapSchema: Schema[Map[String, Person]] = Schema.map[String, Person] + implicit val mapSchema: Schema[scala.collection.immutable.Map[String, Person]] = + Schema.map[String, Person] } ``` @@ -181,7 +182,8 @@ case class Person(name: String, age: Int) object Person { implicit val schema: Schema[Person] = DeriveSchema.gen[Person] - implicit val setSchema: Schema[Set[Person]] = Schema.set[Person] + implicit val setSchema: Schema[scala.collection.immutable.Set[Person]] = + Schema.set[Person] } ``` @@ -199,13 +201,63 @@ In ZIO Schema such a record would be represented using the `Record[R]` typeclass ```scala object Schema { + sealed trait Field[R, A] { + type Field <: Singleton with String + def name: Field + def schema: Schema[A] + } + sealed trait Record[R] extends Schema[R] { def id: TypeId def fields: Chunk[Field[_]] def construct(fieldValues: Chunk[Any]): Either[String, R] } } +``` + +ZIO Schema has specialized record types for case classes, called `CaseClass1[A, Z]`, `CaseClass2[A1, A2, Z]`, and so on. Here is the definition of `apply` method of `CaseClass1` and `CaseClass2`: + +```scala +sealed trait CaseClass1[A, Z] extends Record[Z] + +object CaseClass1 { + def apply[A, Z]( + id0: TypeId, + field0: Field[Z, A], + defaultConstruct0: A => Z, + annotations0: Chunk[Any] = Chunk.empty + ): CaseClass1[A, Z] = ??? +} + +object CaseClass2 { + def apply[A1, A2, Z]( + id0: TypeId, + field01: Field[Z, A1], + field02: Field[Z, A2], + construct0: (A1, A2) => Z, + annotations0: Chunk[Any] = Chunk.empty + ): CaseClass2[A1, A2, Z] = ??? +} +``` +As we can see, they take a `TypeId`, a number of fields of type `Field`, and a construct function. The `TypeId` is used to uniquely identify the type. The `Field` is used to store the name of the field and the schema of the field. The `construct` is used to construct the type from the field values. + +Here is an example of defining schema for `Person` data type: + +```scala mdoc:compile-only +import zio.schema._ + +final case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = + Schema.CaseClass2[String, Int, Person]( + id0 = TypeId.fromTypeName("Person"), + field01 = Schema.Field(name0 = "name", schema0 = Schema[String], get0 = _.name, set0 = (p, x) => p.copy(name = x)), + field02 = Schema.Field(name0 = "age", schema0 = Schema[Int], get0 = _.age, set0 = (person, age) => person.copy(age = age)), + construct0 = (name, age) => Person(name, age), + ) +} ``` ### Enumerations From dfc8367c1005fc82c8dbc0eef7f4daa23c4dd393 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 10 Jul 2023 12:57:38 +0330 Subject: [PATCH 13/63] refactor. --- docs/getting-started.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 144ce8068..429bc1624 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -253,8 +253,20 @@ object Person { implicit val schema: Schema[Person] = Schema.CaseClass2[String, Int, Person]( id0 = TypeId.fromTypeName("Person"), - field01 = Schema.Field(name0 = "name", schema0 = Schema[String], get0 = _.name, set0 = (p, x) => p.copy(name = x)), - field02 = Schema.Field(name0 = "age", schema0 = Schema[Int], get0 = _.age, set0 = (person, age) => person.copy(age = age)), + field01 = + Schema.Field( + name0 = "name", + schema0 = Schema[String], + get0 = _.name, + set0 = (p, x) => p.copy(name = x) + ), + field02 = + Schema.Field( + name0 = "age", + schema0 = Schema[Int], + get0 = _.age, + set0 = (person, age) => person.copy(age = age) + ), construct0 = (name, age) => Person(name, age), ) } From 5e6deedbb527bcc472e7ef9772466889ee98bfca Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 10 Jul 2023 14:35:17 +0330 Subject: [PATCH 14/63] generic record. --- docs/getting-started.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 429bc1624..1a706e964 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -215,7 +215,7 @@ object Schema { } ``` -ZIO Schema has specialized record types for case classes, called `CaseClass1[A, Z]`, `CaseClass2[A1, A2, Z]`, and so on. Here is the definition of `apply` method of `CaseClass1` and `CaseClass2`: +ZIO Schema has specialized record types for case classes, called `CaseClass1[A, Z]`, `CaseClass2[A1, A2, Z]`, ..., `CaseClass22`. Here is the definition of `apply` method of `CaseClass1` and `CaseClass2`: ```scala sealed trait CaseClass1[A, Z] extends Record[Z] @@ -272,6 +272,18 @@ object Person { } ``` +There is also the `GenericRecord` which is used to either ad-hoc records or records that have more than 22 fields: + +```scala +object Schema { + sealed case class GenericRecord( + id: TypeId, + fieldSet: FieldSet, + override val annotations: Chunk[Any] = Chunk.empty + ) extends Record[ListMap[String, _]] +} +``` + ### Enumerations Other times, you might have a type that represents a list of different types. For example, we might have a type, like this: From 243b69e3df83b937ad10e4ae5572bb89fb337c37 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 10 Jul 2023 14:57:30 +0330 Subject: [PATCH 15/63] either. --- docs/getting-started.md | 49 ++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 1a706e964..70fa6511c 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -106,7 +106,7 @@ val personListSchema: Schema[List[Person]] = ) ``` -ZIO Schema has a helper method `Schema.list[A]` that creates a `Schema[List[A]]` for us: +ZIO Schema has a `Schema.list[A]` constructor that creates a `Schema[List[A]]` for us: ```scala mdoc:compile-only import zio._ @@ -138,7 +138,7 @@ object Schema { } ``` -It stores the key and value schemas. Like `Sequence`, instead of using `Map` directly, we can use the helper method `Schema.map[K, V]`: +It stores the key and value schemas. Like `Sequence`, instead of using `Map` directly, we can use the `Schema.map[K, V]` constructor: ```scala mdoc:compile-only import zio._ @@ -170,7 +170,7 @@ object Schema { } ``` -To create a `Schema` for a `Set[A]`, we can use the above type class directly or use the helper method `Schema.set[A]`: +To create a `Schema` for a `Set[A]`, we can use the above type class directly or use the `Schema.set[A]` constructor: ```scala mdoc:compile-only import zio._ @@ -312,33 +312,42 @@ object Schema { } ``` - ### Optionals -A special variant of a collection type is the `Optional[A]` type: +To create a `Schema` for optional values, we can use the `Optional` type class: ```scala -object Schema extends SchemaEquality { +object Schema { + case class Optional[A]( + schema: Schema[A], + annotations: Chunk[Any] = Chunk.empty + ) extends Schema[Option[A]] +} +``` - final case class Optional[A](codec: Schema[A]) extends Schema[Option[A]] { - self => +Using the `Schema.option[A]` constructor, makes it easier to do so: - private[schema] val someCodec: Schema[Some[A]] = codec.transform(a => Some(a), _.get) +```scala +val option: Schema[Option[Person]] = Schema.option[Person] +``` - override type Accessors[Lens[_, _], Prism[_, _], Traversal[_, _]] = - (Prism[Option[A], Some[A]], Prism[Option[A], None.type]) +### Either - val toEnum: Enum2[Some[A], None.type, Option[A]] = Enum2( - Case[Some[A], Option[A]]("Some", someCodec, _.asInstanceOf[Some[A]], Chunk.empty), - Case[None.type, Option[A]]("None", singleton(None), _.asInstanceOf[None.type], Chunk.empty), - Chunk.empty - ) - - override def makeAccessors(b: AccessorBuilder): (b.Prism[Option[A], Some[A]], b.Prism[Option[A], None.type]) = - b.makePrism(toEnum, toEnum.case1) -> b.makePrism(toEnum, toEnum.case2) - } +Here is the same but for `Either`: +```scala +object Schema { + case class Either[A, B]( + left: Schema[A], + right: Schema[B], + annotations: Chunk[Any] = Chunk.empty + ) extends Schema[scala.util.Either[A, B]] } ``` +We can use `Schema.either[A, B]` to create a `Schema` for `scala.util.Either[A, B]`: +```scala +val eitherPersonSchema: Schema[scala.util.Either[String, Person]] = + Schema.either[String, Person] +``` From c0c3d330053ffb4b87a3bd74038c4a751506e0d2 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 10 Jul 2023 15:43:38 +0330 Subject: [PATCH 16/63] enumeration. --- docs/getting-started.md | 45 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 70fa6511c..4584bfe62 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -305,10 +305,47 @@ In ZIO Schema we call these types `enumeration` types, and they are represented ```scala object Schema { - sealed trait Enum[A] extends Schema[A] { - def annotations: Chunk[Any] - def structure: ListMap[String, Schema[_]] - } + sealed trait Enum[Z] extends Schema[Z] +} +``` + +It has specialized types `Enum1[A, Z]`, `Enum2[A1, A2, Z]`, ..., `Enum22[A1, A2, ..., A22, Z]` for enumerations with 1, 2, ..., 22 cases. Here is the definition of `Enum1` and `Enum2`: + +```scala + sealed case class Enum1[A, Z]( + id: TypeId, + case1: Case[Z, A], + annotations: Chunk[Any] = Chunk.empty + ) extends Enum[Z] + + sealed case class Enum2[A1, A2, Z]( + id: TypeId, + case1: Case[Z, A1], + case2: Case[Z, A2], + annotations: Chunk[Any] = Chunk.empty + ) extends Enum[Z] + + // Enum3, Enum4, ..., Enum22 +} +``` + +If the enumeration has more than 22 cases, we can use the `EnumN` type class: + +```scala +object Schema { + sealed case class EnumN[Z, C <: CaseSet.Aux[Z]]( + id: TypeId, + caseSet: C, + annotations: Chunk[Any] = Chunk.empty + ) extends Enum[Z] +} +``` + +It has a simple constructor called `Schema.enumeration`: + +```scala +object Schema { + def enumeration[A, C <: CaseSet.Aux[A]](id: TypeId, caseSet: C): Schema[A] = ??? } ``` From e640822d820ec3c8ba4414210439565410686461 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 10 Jul 2023 16:00:35 +0330 Subject: [PATCH 17/63] chunk and vector. --- docs/getting-started.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 4584bfe62..b00c50f91 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -106,7 +106,7 @@ val personListSchema: Schema[List[Person]] = ) ``` -ZIO Schema has a `Schema.list[A]` constructor that creates a `Schema[List[A]]` for us: +ZIO Schema has `Schema.list[A]`, `Schema.chunk[A]` and `Schema.vector[A]` constructors that create `Schema[List[A]]`, `Schema[Chunk[A]]` and `Schema[Vector[A]]` for us: ```scala mdoc:compile-only import zio._ @@ -118,7 +118,9 @@ case class Person(name: String, age: Int) object Person { implicit val schema: Schema[Person] = DeriveSchema.gen[Person] - implicit val listSchema: Schema[List[Person]] = Schema.list[Person] + implicit val listSchema: Schema[List[Person]] = Schema.list[Person] + implicit val chunkSchema: Schema[Chunk[Person]] = Schema.chunk[Person] + implicit val vectorSchema: Schema[Vector[Person]] = Schema.vector[Person] } ``` From fc0bccd26a5dd3725631c859ebd59de16a75081d Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 10 Jul 2023 16:28:45 +0330 Subject: [PATCH 18/63] fail. --- docs/getting-started.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index b00c50f91..08190e172 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -61,6 +61,19 @@ val intSchema1: Schema[Int] = Schema[Int] val intSchema2: Schema[Int] = Schema.primitive[Int] ``` +### Fail + +To represents the absence of schema information for the given `A` type, we can use `Schema.fail` constructor, which creates the following schema: + +```scala +object Schema { + case class Fail[A]( + message: String, + annotations: Chunk[Any] = Chunk.empty + ) extends Schema[A] +} +``` + ### Collections #### Sequence From 4b1b2dfdd68cb879bf9cf7956fcebc2136e69c4b Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 10 Jul 2023 17:36:27 +0330 Subject: [PATCH 19/63] tuples. --- docs/getting-started.md | 48 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index 08190e172..f97c0c403 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -403,3 +403,51 @@ We can use `Schema.either[A, B]` to create a `Schema` for `scala.util.Either[A, val eitherPersonSchema: Schema[scala.util.Either[String, Person]] = Schema.either[String, Person] ``` + +### Tuple + +Each schema has a `Schema#zip` operator that allows us to combine two schemas and create a schema for a tuple of the two types: + +```scala +object Schema { + def zip[B](that: Schema[B]): Schema[(A, B)] = + Schema.Tuple2(self, that) +} +``` + +It is implemented using the `Schema.Tuple2` type class: + +```scala +object Schema { + final case class Tuple2[A, B]( + left: Schema[A], + right: Schema[B], + annotations: Chunk[Any] = Chunk.em + pty + ) extends Schema[(A, B)] +} +``` + +ZIO Schema also provides implicit conversions for tuples of arity 2, 3, ..., 22: + +```scala +object Schema { + implicit def tuple2[A, B](implicit c1: Schema[A], c2: Schema[B]): Schema[(A, B)] = + c1.zip(c2) + + implicit def tuple3[A, B, C](implicit c1: Schema[A], c2: Schema[B], c3: Schema[C]): Schema[(A, B, C)] = + c1.zip(c2).zip(c3).transform({ case ((a, b), c) => (a, b, c) }, { case (a, b, c) => ((a, b), c) }) + + // tuple3, tuple4, ..., tuple22 +} +``` + +So we can easily create a `Schema` for a tuple of n elements, just by calling `Schema[(A1, A2, ..., An)]`: + +```scala mdoc:compile-only +import zio.schema._ + +val tuple2: Schema[(String, Int)] = Schema[(String, Int)] +val tuple3: Schema[(String, Int, Boolean)] = Schema[(String, Int, Boolean)] +// ... +``` From a5dd59ca1d350a88d80780f72af710e7883a8b81 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 10 Jul 2023 17:52:55 +0330 Subject: [PATCH 20/63] update readme. --- README.md | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 243bf44ee..6439bc41b 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,31 @@ ## Introduction +ZIO Schema helps us to solve some of the most common problems in distributed computing, such as serialization, deserialization, and data migration. + +```scala +trait Schema[A] { + def schema: Schema[A] +} +``` + +The trait `Schema[A]` is the core data type of this library. The `Schema[A]` is a description of the structure of a data type `A`, it is a data type that describes other data types. It is a first-class value that can be passed around, composed, and manipulated. + +It turns a compiled-time construct (the type of a data structure) into a runtime construct (a value that can be read, manipulated, and composed at runtime). + +## What Problems Does ZIO Schema Solve? + +1. Metaprogramming without macros, reflection or complicated implicit derivations. + 1. Creating serialization and deserialization codecs for any supported protocol (JSON, protobuf, etc.) + 2. Deriving standard type classes (`Eq`, `Show`, `Ordering`, etc.) from the structure of the data + 4. Default values for data types +2. Automate ETL (Extract, Transform, Load) pipelines + 1. Diffing: diffing between two values of the same type + 2. Patching: applying a diff to a value to update it + 3. Migration: migrating values from one type to another +3. Computations as data: Not only we can turn types into values, but we can also turn computations into values. This opens up a whole new world of possibilities in respect to distributed computing. + 1. Optics + Schema is a structure of a data type. ZIO Schema reifies the concept of structure for data types. It makes a high-level description of any data type and makes them as first-class values. Creating a schema for a data type helps us to write codecs for that data type. So this library can be a host of functionalities useful for writing codecs and protocols like JSON, Protobuf, CSV, and so forth. @@ -30,13 +55,16 @@ _ZIO Schema_ is used by a growing number of ZIO libraries, including _ZIO Flow_, In order to use this library, we need to add the following lines in our `build.sbt` file: ```scala -libraryDependencies += "dev.zio" %% "zio-schema" % "0.4.11" -libraryDependencies += "dev.zio" %% "zio-schema-bson" % "0.4.11" -libraryDependencies += "dev.zio" %% "zio-schema-json" % "0.4.11" -libraryDependencies += "dev.zio" %% "zio-schema-protobuf" % "0.4.11" +libraryDependencies += "dev.zio" %% "zio-schema" % "0.4.12" +libraryDependencies += "dev.zio" %% "zio-schema-avro" % "0.4.12" +libraryDependencies += "dev.zio" %% "zio-schema-bson" % "0.4.12" +libraryDependencies += "dev.zio" %% "zio-schema-json" % "0.4.12" +libraryDependencies += "dev.zio" %% "zio-schema-msg-pack" % "0.4.12" +libraryDependencies += "dev.zio" %% "zio-schema-protobuf" % "0.4.12" +libraryDependencies += "dev.zio" %% "zio-schema-thrift" % "0.4.12" // Required for automatic generic derivation of schemas -libraryDependencies += "dev.zio" %% "zio-schema-derivation" % "0.4.11", +libraryDependencies += "dev.zio" %% "zio-schema-derivation" % "0.4.12" libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value % "provided" ``` From da4a61e3ee724331864253d6ffcab0bc8f24c298 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 10 Jul 2023 18:32:01 +0330 Subject: [PATCH 21/63] refactor sidebars. --- ...ng-started.md => basic-building-blocks.md} | 32 ++++++++----------- docs/motivation.md | 3 +- docs/sidebars.js | 5 ++- 3 files changed, 17 insertions(+), 23 deletions(-) rename docs/{getting-started.md => basic-building-blocks.md} (98%) diff --git a/docs/getting-started.md b/docs/basic-building-blocks.md similarity index 98% rename from docs/getting-started.md rename to docs/basic-building-blocks.md index f97c0c403..c75099a92 100644 --- a/docs/getting-started.md +++ b/docs/basic-building-blocks.md @@ -1,17 +1,11 @@ --- -id: getting-started -title: "Getting Started" +id: basic-building-blocks +title: "Basic Building Blocks" --- To get started, first we need to understand that a ZIO Schema is basically built-up from these three sealed traits: `Record[R]`, `Enum[A]` and `Sequence[Col, Elem]`, along with the case class `Primitive[A]`. Every other type is just a specialisation of one of these (or not relevant to get you started). -We will take a look at them now. - -## Basic Building Blocks - -### Schema - The core data type of ZIO Schema is a `Schema[A]` which is **invariant in `A`** by necessity, because a Schema allows us to derive operations that produce an `A` but also operations that consume an `A` and that imposes limitations on the types of **transformation operators** and **composition operators** that we can provide based on a `Schema`. It looks kind of like this (simplified): @@ -24,7 +18,7 @@ sealed trait Schema[A] { self => } ``` -### Primitives +## Primitives To describe scalar data type `A`, we use the `Primitive[A]` data type which basically is a wrapper around `StandardType`: @@ -61,7 +55,7 @@ val intSchema1: Schema[Int] = Schema[Int] val intSchema2: Schema[Int] = Schema.primitive[Int] ``` -### Fail +## Fail To represents the absence of schema information for the given `A` type, we can use `Schema.fail` constructor, which creates the following schema: @@ -74,9 +68,9 @@ object Schema { } ``` -### Collections +## Collections -#### Sequence +### Sequence Often we have a type that is a collection of elements. For example, we might have a `List[User]`. This is called a `Sequence` and is represented using the `Sequence[Col, Elem, I]` type class: @@ -137,7 +131,7 @@ object Person { } ``` -#### Map +### Map Likewise, we can have a type that is a map of keys to values. ZIO Schema represents this using the following type class: @@ -170,7 +164,7 @@ object Person { } ``` -#### Set +### Set The `Set` type class is similar to `Sequence` and `Map`. It is used to represent a schema for a set of elements: @@ -202,7 +196,7 @@ object Person { } ``` -### Records +## Records Our data structures usually are composed of a lot of types. For example, we might have a `User` type that has a `name` field, an `age` field, an `address` field, and a `friends` field. @@ -299,7 +293,7 @@ object Schema { } ``` -### Enumerations +## Enumerations Other times, you might have a type that represents a list of different types. For example, we might have a type, like this: @@ -364,7 +358,7 @@ object Schema { } ``` -### Optionals +## Optionals To create a `Schema` for optional values, we can use the `Optional` type class: @@ -383,7 +377,7 @@ Using the `Schema.option[A]` constructor, makes it easier to do so: val option: Schema[Option[Person]] = Schema.option[Person] ``` -### Either +## Either Here is the same but for `Either`: @@ -404,7 +398,7 @@ val eitherPersonSchema: Schema[scala.util.Either[String, Person]] = Schema.either[String, Person] ``` -### Tuple +## Tuple Each schema has a `Schema#zip` operator that allows us to combine two schemas and create a schema for a tuple of the two types: diff --git a/docs/motivation.md b/docs/motivation.md index 77470f3da..dc09df9e5 100644 --- a/docs/motivation.md +++ b/docs/motivation.md @@ -1,6 +1,7 @@ --- id: motivation -title: "Understanding The Motivation Behind ZIO Schema" +title: "The Motivation Behind ZIO Schema" +sidebar_label: "Motivation" --- ZIO Schema is a library used in many ZIO projects such as _ZIO Flow_, _ZIO Redis_, _ZIO Web_, _ZIO SQL_ and _ZIO DynamoDB_. It is all about reification of our types. Reification means transforming something abstract (e.g. side effects, accessing fields, structure) into something "real" (values). diff --git a/docs/sidebars.js b/docs/sidebars.js index bf99c66cc..9b700ffc3 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -6,19 +6,18 @@ const sidebars = { collapsed: false, link: { type: "doc", id: "index" }, items: [ + "motivation", "use-cases", + "basic-building-blocks", { type: "category", label: "Writing Schema", collapsed: true, - link: { type: "doc", id: "index" }, items: [ "manual-schema-construction", "automatic-schema-derivation" ], }, - "motivation", - "getting-started", "transforming-schemas", "codecs", "protobuf-example", From 377d55539dee015200e1b96f04fe85d2c947f237 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Tue, 11 Jul 2023 11:36:23 +0330 Subject: [PATCH 22/63] add more resources for zio schema. --- docs/index.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index d516b4930..f9957ff70 100644 --- a/docs/index.md +++ b/docs/index.md @@ -79,7 +79,6 @@ zio.Runtime.default.unsafe.run( ).getOrThrowFiberFailure() ``` - ## Resources - [Zymposium - ZIO Schema](https://www.youtube.com/watch?v=GfNiDaL5aIM) by John A. De Goes, Adam Fraser and Kit Langton (May 2021) From 6dbd47cebb227a397815f45e51bc601d9c875bbf Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Tue, 11 Jul 2023 12:23:35 +0330 Subject: [PATCH 23/63] getting the default value of a schema. --- docs/operations.md | 18 ++++++++++++++++++ docs/sidebars.js | 1 + 2 files changed, 19 insertions(+) create mode 100644 docs/operations.md diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 000000000..b68bf5817 --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,18 @@ +--- +id: operations +title: "ZIO Schema Operations" +--- + +Once we have defined our schemas, we can use them to perform a variety of operations. In this section, we will explore some of the most common operations that we can perform on schemas. + +## Getting the Default Value of a Schema + +ZIO Schema provides a method called `defaultValue` that returns the default value of the underlying type described by the schema. This method returns a `scala.util.Either[String, A]` value, where `A` is the type described by the schema. If the schema does not have a default value, the method returns a `Left` value containing an error message. Otherwise, it returns a `Right` value containing the default value: + +```scala +sealed trait Schema[A] { + def defaultValue: scala.util.Either[String, A] +} +``` + +ZIO Schema have out of the box default values for all standard types, such as `String`, `Int`, `Boolean`, ..., `LocalDateTime` and `UUID`. For example, the default value of a schema for `String` is the empty string, and the default value of a schema for `Int` is `0`. diff --git a/docs/sidebars.js b/docs/sidebars.js index 9b700ffc3..ffb4c8559 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -9,6 +9,7 @@ const sidebars = { "motivation", "use-cases", "basic-building-blocks", + "operations", { type: "category", label: "Writing Schema", From 538a7788e9cc468114be554b4c49e684fb877abc Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Tue, 11 Jul 2023 12:25:19 +0330 Subject: [PATCH 24/63] sidebar label for operations. --- docs/operations.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/operations.md b/docs/operations.md index b68bf5817..b8a0c7fa8 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -1,6 +1,7 @@ --- id: operations title: "ZIO Schema Operations" +sidebar_label: "Operations" --- Once we have defined our schemas, we can use them to perform a variety of operations. In this section, we will explore some of the most common operations that we can perform on schemas. From a5969c8bbc87fc3484c19f8bb11512dcc0e7c9fa Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Tue, 11 Jul 2023 12:27:32 +0330 Subject: [PATCH 25/63] reorder sidebar items. --- docs/sidebars.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sidebars.js b/docs/sidebars.js index ffb4c8559..a57c0472e 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -9,7 +9,6 @@ const sidebars = { "motivation", "use-cases", "basic-building-blocks", - "operations", { type: "category", label: "Writing Schema", @@ -19,6 +18,7 @@ const sidebars = { "automatic-schema-derivation" ], }, + "operations", "transforming-schemas", "codecs", "protobuf-example", From 86f4154a6f6ecc710c8f03fd77898b5396364008 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Tue, 11 Jul 2023 12:57:09 +0330 Subject: [PATCH 26/63] generate ordering for schemas. --- docs/operations.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/operations.md b/docs/operations.md index b8a0c7fa8..2d2bcb1af 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -17,3 +17,29 @@ sealed trait Schema[A] { ``` ZIO Schema have out of the box default values for all standard types, such as `String`, `Int`, `Boolean`, ..., `LocalDateTime` and `UUID`. For example, the default value of a schema for `String` is the empty string, and the default value of a schema for `Int` is `0`. + +## Generate Orderings for Schemas + +Standard Scala library provides a type class called `Ordering[A]` that allows us to compare values of type `A`. ZIO Schema provides a method called `ordering` that generates an `Ordering[A]` instance for the underlying type described by the schema: + +```scala +sealed trait Schema[A] { + def ordering: Ordering[A] +} +``` + +Here is an example, where it helps us to sort the list of `Person`: + +```scala mdoc:compile-only +case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = DeriveSchema.gen[Person] +} + +val sortedList: Seq[Person] = + List( + Person("John", 42), + Person("Jane", 34) + ).sorted(Person.schema.ordering) +``` From 43395937917488069ccaaaf9f481d0308aff4990 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Tue, 11 Jul 2023 12:57:59 +0330 Subject: [PATCH 27/63] fix mdoc errors. --- docs/operations.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/operations.md b/docs/operations.md index 2d2bcb1af..97a2f6f8b 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -31,6 +31,8 @@ sealed trait Schema[A] { Here is an example, where it helps us to sort the list of `Person`: ```scala mdoc:compile-only +import zio.schema._ + case class Person(name: String, age: Int) object Person { From a1a1868bd3403a861044b333aef4a22dfd0f01ed Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Tue, 11 Jul 2023 13:32:58 +0330 Subject: [PATCH 28/63] diffing and patching. --- docs/operations.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/operations.md b/docs/operations.md index 97a2f6f8b..7b0f42d26 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -45,3 +45,39 @@ val sortedList: Seq[Person] = Person("Jane", 34) ).sorted(Person.schema.ordering) ``` + +## Diffing and Patching + +ZIO Schema provides two methods called `diff` and `patch`: + +```scala +sealed trait Schema[A] { + def diff(thisValue: A, thatValue: A): Patch[A] + + def patch(oldValue: A, diff: Patch[A]): scala.util.Either[String, A] +} +``` + +The `diff` method takes two values of the same type `A` and returns a `Patch[A]` value that describes the differences between the two values. conversely, the `patch` method takes a value of type `A` and a `Patch[A]` value and returns a new value of type `A` that is the result of applying the patch to the original value. + +Here is a simple example that demonstrate the how to use `diff` and `patch`: + +```scala mdoc:compile-only +import zio.schema._ + +case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = DeriveSchema.gen[Person] +} + +val oldValue = Person("John", 42) +val newValue = Person("John", 43) + +val patch: Patch[Person] = + Person.schema.diff(oldValue, newValue) + +assert( + Person.schema.patch(oldValue, patch) == Right(newValue) +) +``` From 175e04a29d4a2383ba94637fc2935cfbdf69bd33 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Tue, 11 Jul 2023 16:48:58 +0330 Subject: [PATCH 29/63] automatic migration. --- docs/operations.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/operations.md b/docs/operations.md index 7b0f42d26..ca2a281a5 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -81,3 +81,21 @@ assert( Person.schema.patch(oldValue, patch) == Right(newValue) ) ``` + +## Automatic Migrations + +With ZIO Schema, we can automatically migrate data from one version of a schema to another. As software evolves, we often need to add, change or remove old fields. ZIO Schema provides two methods called `migrate` and `coerce` which help migrate the old schema to the new one: + +```scala +selaed trait Schema[A] { + def migrate[B](newSchema: Schema[B]): Either[String, A => scala.util.Either[String, B]] + + def coerce[B](newSchema: Schema[B]): Either[String, Schema[B]] = + for { + f <- self.migrate(newSchema) + g <- newSchema.migrate(self) + } yield self.transformOrFail(f, g) +} +``` + +The `migrate` method takes a new schema and returns a function that can migrate values of the old schema to values of the new schema as a `Right` value of `Either`. If the schemas have unambiguous transformations or are incompatible, the method returns a `Left` value containing an error message. From 814a25f77cc585fd0a6774a8836216264a611afe Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Tue, 11 Jul 2023 17:17:15 +0330 Subject: [PATCH 30/63] schema serialization. --- docs/operations.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/operations.md b/docs/operations.md index ca2a281a5..80995302b 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -99,3 +99,17 @@ selaed trait Schema[A] { ``` The `migrate` method takes a new schema and returns a function that can migrate values of the old schema to values of the new schema as a `Right` value of `Either`. If the schemas have unambiguous transformations or are incompatible, the method returns a `Left` value containing an error message. + +## Schema Serialization + +In distributed systems, we often need to move computations to data instead of moving data to computations. The data is big and the network is slow, so moving it is expensive and sometimes impossible due to the volume of data. So in distributed systems, we would like to move our functions to the data and apply the data to the functions and gather the results back. + +So we need a way to serialize our computations and send them through the network. In ZIO Schema, each schema itself has a schema, so we can treat the structure as pure data! we can serialize our schemas by calling the `serializable` method: + +```scala +selead trait Schema[A] { + def serializable: Schema[Schema[A]] +} +``` + +By calling this method, we can get the schema of a schema. So we can serialize it and send it across the wire, and it can be deserialized on the other side. After deserializing it, we have a schema that is isomorphic to the original schema. So all the operations that we can perform on the original type `A`, we can perform on any value that is isomorphic to `A` on the other side. From b6cacacc301f9d949b8e82d33fd819ffda566dfb Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Tue, 11 Jul 2023 17:34:37 +0330 Subject: [PATCH 31/63] fix typo. --- docs/operations.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/operations.md b/docs/operations.md index 80995302b..257e2912c 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -87,7 +87,7 @@ assert( With ZIO Schema, we can automatically migrate data from one version of a schema to another. As software evolves, we often need to add, change or remove old fields. ZIO Schema provides two methods called `migrate` and `coerce` which help migrate the old schema to the new one: ```scala -selaed trait Schema[A] { +sealed trait Schema[A] { def migrate[B](newSchema: Schema[B]): Either[String, A => scala.util.Either[String, B]] def coerce[B](newSchema: Schema[B]): Either[String, Schema[B]] = @@ -107,7 +107,7 @@ In distributed systems, we often need to move computations to data instead of mo So we need a way to serialize our computations and send them through the network. In ZIO Schema, each schema itself has a schema, so we can treat the structure as pure data! we can serialize our schemas by calling the `serializable` method: ```scala -selead trait Schema[A] { +sealed trait Schema[A] { def serializable: Schema[Schema[A]] } ``` From e4a74e4eee846bed7009825b3be8617ba7f115f3 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Wed, 12 Jul 2023 18:31:51 +0330 Subject: [PATCH 32/63] add more resources. --- docs/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/index.md b/docs/index.md index f9957ff70..3816e90eb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -82,3 +82,5 @@ zio.Runtime.default.unsafe.run( ## Resources - [Zymposium - ZIO Schema](https://www.youtube.com/watch?v=GfNiDaL5aIM) by John A. De Goes, Adam Fraser and Kit Langton (May 2021) +- [ZIO SCHEMA: A Toolkit For Functional Distributed Computing](https://www.youtube.com/watch?v=lJziseYKvHo&t=481s) by Dan Harris (Functional Scala 2021) +- [Creating Declarative Query Plans With ZIO Schema](https://www.youtube.com/watch?v=ClePN4P9_pg) by Dan Harris (ZIO World 2022) From 60a42db981e90ebd120feaa9a916ae50c5fed684 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Thu, 13 Jul 2023 10:03:27 +0330 Subject: [PATCH 33/63] mapping dto to domain object. --- docs/mapping-dto-to-domain-object.md | 95 ++++++++++++++++++++++++++++ docs/sidebars.js | 8 +++ 2 files changed, 103 insertions(+) create mode 100644 docs/mapping-dto-to-domain-object.md diff --git a/docs/mapping-dto-to-domain-object.md b/docs/mapping-dto-to-domain-object.md new file mode 100644 index 000000000..f51b137fa --- /dev/null +++ b/docs/mapping-dto-to-domain-object.md @@ -0,0 +1,95 @@ +--- +id: mapping-dto-to-domain-object +title: "Mapping DTO to Domain Object" +--- + +When we write layered applications, where different layers are decoupled from each other, we need to transfer data between layers. For example, assume we have a layer that has `Person` data type and it receives JSON string of type `PersonDTO` from another layer. We need to convert `PersonDTO` to `Person` and maybe vice versa. + +One way to do this is to write codec for `PersonDTO` and convert the JSON String to the `PersonDTO` and then convert `PersonDTO` to `Person`. This approach is not very convenient and we need to write some boilerplate code. With ZIO Schema we can simplify this process and write a codec for `Person` that uses a specialized schema for `Person`, i.e. `personDTOMapperSchema`, which describes `Person` data type in terms of transformation from `PersonDTO` to `Person` and vice versa. With this approach, we can directly convert the JSON string to `Person` in one step: + +```scala mdoc:compile-only +import zio._ +import zio.json.JsonCodec +import zio.schema.codec.JsonCodec._ +import zio.schema.{DeriveSchema, Schema} + +import java.time.LocalDate + +object MainApp extends ZIOAppDefault { + + case class PersonDTO( + firstName: String, + lastName: String, + birthday: (Int, Int, Int) + ) + + object PersonDTO { + implicit val schema: Schema[PersonDTO] = DeriveSchema.gen[PersonDTO] + + implicit val codec: JsonCodec[PersonDTO] = jsonCodec[PersonDTO](schema) + } + + case class Person(name: String, birthdate: LocalDate) + + object Person { + implicit val schema: Schema[Person] = DeriveSchema.gen[Person] + + val personDTOMapperSchema: Schema[Person] = + PersonDTO.schema.transform( + f = dto => { + val (year, month, day) = dto.birthday + Person( + dto.firstName + " " + dto.lastName, + birthdate = LocalDate.of(year, month, day) + ) + }, + g = (person: Person) => { + val fullNameArray = person.name.split(" ") + PersonDTO( + fullNameArray.head, + fullNameArray.last, + ( + person.birthdate.getYear, + person.birthdate.getMonthValue, + person.birthdate.getDayOfMonth + ) + ) + } + ) + implicit val codec: JsonCodec[Person] = jsonCodec[Person](schema) + + val personDTOJsonMapperCodec: JsonCodec[Person] = + jsonCodec[Person](personDTOMapperSchema) + } + + val json: String = + """ + |{ + | "firstName": "John", + | "lastName": "Doe", + | "birthday": [[1981, 07], 13] + |} + |""".stripMargin + + def run = for { + // Approach 1: Decode JSON String to PersonDTO and then Transform it into the Person object + personDTO <- ZIO.fromEither(JsonCodec[PersonDTO].decodeJson(json)) + (year, month, day) = personDTO.birthday + person1 = Person( + name = personDTO.firstName + " " + personDTO.lastName, + LocalDate.of(year, month, day) + ) + _ <- ZIO.debug( + s"person: $person1" + ) + + // Approach 2: Decode JSON string in one step into the Person object + person2 <- ZIO.fromEither( + JsonCodec[Person](Person.personDTOJsonMapperCodec).decodeJson(json) + ) + _ <- ZIO.debug( + s"person: $person2" + ) + } yield assert(person1 == person2) +} +``` diff --git a/docs/sidebars.js b/docs/sidebars.js index a57c0472e..d8dcf247f 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -23,6 +23,14 @@ const sidebars = { "codecs", "protobuf-example", "combining-different-encoders", + { + type: "category", + label: "Examples", + collapsed: true, + items: [ + "mapping-dto-to-domain-object" + ], + } ], }, ], From 1b605983a04bd75a5d78b7705affaf0c277d5f9f Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Thu, 13 Jul 2023 12:11:51 +0330 Subject: [PATCH 34/63] example for Schema#migrate method. --- docs/mapping-dto-to-domain-object.md | 72 ++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/docs/mapping-dto-to-domain-object.md b/docs/mapping-dto-to-domain-object.md index f51b137fa..6edd76059 100644 --- a/docs/mapping-dto-to-domain-object.md +++ b/docs/mapping-dto-to-domain-object.md @@ -93,3 +93,75 @@ object MainApp extends ZIOAppDefault { } yield assert(person1 == person2) } ``` + +As we can see in the example above, the second approach is much simpler and more convenient than the first one. + +The problem we solved in previous example is common in microservices architecture, where we transfer DTOs across the network. So we need to serialize and deserialize the data transfer objects. + +In the next example, we will see how we can use schema migration, when we need to map data transfer object to domain object and vice versa. In this example, we do not require to serialize/deserialize any object, but the problem of mapping DTO to domain object persists. + +In this example, as the same as in the previous one, we will define the schema for `Person` in terms of schema transformation from `PersonDTO` to `Person` and vice versa. The only difference is that to map `PersonDTO` to `Person` we will use `Schema#migrate` method which returns `Either[String, PersonDTO => Either[String, Person]]`. If the migration is successful, we will get `Right` with a function that converts `PersonDTO` to `Either[String, Person]`, otherwise we will get `Left` with an error message: + +```scala mdoc:compile-only +import zio._ +import zio.schema.{DeriveSchema, Schema} + +import java.time.LocalDate + +object MainApp extends ZIOAppDefault { + + case class PersonDTO( + firstName: String, + lastName: String, + birthday: (Int, Int, Int) + ) + + object PersonDTO { + implicit val schema: Schema[PersonDTO] = DeriveSchema.gen[PersonDTO] + } + + case class Person(name: String, birthdate: LocalDate) + + object Person { + implicit val schema: Schema[Person] = DeriveSchema.gen[Person] + + val personDTOMapperSchema: Schema[Person] = + PersonDTO.schema.transform( + f = dto => { + val (year, month, day) = dto.birthday + Person( + dto.firstName + " " + dto.lastName, + birthdate = LocalDate.of(year, month, day) + ) + }, + g = (person: Person) => { + val fullNameArray = person.name.split(" ") + PersonDTO( + fullNameArray.head, + fullNameArray.last, + ( + person.birthdate.getYear, + person.birthdate.getMonthValue, + person.birthdate.getDayOfMonth + ) + ) + } + ) + + def fromPersonDTO(p: PersonDTO): IO[String, Person] = + ZIO.fromEither( + PersonDTO.schema + .migrate(personDTOMapperSchema) + .flatMap(_ (p)) + ) + } + + + def run = for { + personDTO <- ZIO.succeed(PersonDTO("John", "Doe", (1981, 7, 13))) + person <- Person.fromPersonDTO(personDTO) + _ <- ZIO.debug(s"person: $person") + } yield () + +} +``` From 82a13e21633db26a45ee682ad0c8855c7036ea37 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Thu, 13 Jul 2023 12:18:32 +0330 Subject: [PATCH 35/63] improve sentences. --- docs/mapping-dto-to-domain-object.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/mapping-dto-to-domain-object.md b/docs/mapping-dto-to-domain-object.md index 6edd76059..11635c2de 100644 --- a/docs/mapping-dto-to-domain-object.md +++ b/docs/mapping-dto-to-domain-object.md @@ -100,7 +100,7 @@ The problem we solved in previous example is common in microservices architectur In the next example, we will see how we can use schema migration, when we need to map data transfer object to domain object and vice versa. In this example, we do not require to serialize/deserialize any object, but the problem of mapping DTO to domain object persists. -In this example, as the same as in the previous one, we will define the schema for `Person` in terms of schema transformation from `PersonDTO` to `Person` and vice versa. The only difference is that to map `PersonDTO` to `Person` we will use `Schema#migrate` method which returns `Either[String, PersonDTO => Either[String, Person]]`. If the migration is successful, we will get `Right` with a function that converts `PersonDTO` to `Either[String, Person]`, otherwise we will get `Left` with an error message: +In this example, similar to the previous one, we will define the schema for `Person` in terms of schema transformation from `PersonDTO` to `Person` and vice versa. The only difference is that we will utilize the `Schema#migrate` method to map `PersonDTO` to `Person`. This method returns `Either[String, PersonDTO => Either[String, Person]]`. If the migration is successful, we will receive `Right` with a function that converts `PersonDTO` to `Either[String, Person]`. Otherwise, if there is an error, we will receive `Left` along with an error message: ```scala mdoc:compile-only import zio._ From eabb44f3e53dbc8b6207ce35c9994b4359a9669a Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Sat, 15 Jul 2023 12:33:52 +0330 Subject: [PATCH 36/63] add nuttycombe talk to resource section. --- docs/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.md b/docs/index.md index 3816e90eb..a80a98615 100644 --- a/docs/index.md +++ b/docs/index.md @@ -84,3 +84,4 @@ zio.Runtime.default.unsafe.run( - [Zymposium - ZIO Schema](https://www.youtube.com/watch?v=GfNiDaL5aIM) by John A. De Goes, Adam Fraser and Kit Langton (May 2021) - [ZIO SCHEMA: A Toolkit For Functional Distributed Computing](https://www.youtube.com/watch?v=lJziseYKvHo&t=481s) by Dan Harris (Functional Scala 2021) - [Creating Declarative Query Plans With ZIO Schema](https://www.youtube.com/watch?v=ClePN4P9_pg) by Dan Harris (ZIO World 2022) +- [Describing Data...with free applicative functors (and more)](https://www.youtube.com/watch?v=oRLkb6mqvVM) by Kris Nuttycombe (Scala World) on the idea behind the [xenomorph](https://github.com/nuttycom/xenomorph) library \ No newline at end of file From caf01ecce907916abf61a8fb3bd0dc6cbfa81a23 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Sun, 16 Jul 2023 13:20:52 +0330 Subject: [PATCH 37/63] dynamic data representation. --- docs/dynamic-data-representation.md | 55 +++++++++++++++++++++++++++++ docs/sidebars.js | 1 + 2 files changed, 56 insertions(+) create mode 100644 docs/dynamic-data-representation.md diff --git a/docs/dynamic-data-representation.md b/docs/dynamic-data-representation.md new file mode 100644 index 000000000..b5d872cbd --- /dev/null +++ b/docs/dynamic-data-representation.md @@ -0,0 +1,55 @@ +--- +id: dynamic-data-representation +title: "Dynamic Data Representation" +--- + +DynamicValue is a way to describe the entire universe of possibilities for schema values. It does that in a way that we can interact with and introspect the data with its structure (type information). The structure of the data is baked into the data itself. + +Let's create a simple instance of `Person("John Doe", 42)` and convert it to `DynamicValue`: + +```scala mdoc:compile-only +import zio.schema._ + +case class Person(name: String, age: Int) + +object Person { + implicit val schema = DeriveSchema.gen[Person] +} + +val person = Person("John Doe", 42) +val dynamicPerson = DynamicValue.fromSchemaAndValue(Person.schema, person) +// or we can call `toDynamic` on the schema directly: +// val dynamicPerson = Person.schema.toDynamic(person) +println(dynamicPerson) +``` + +As we can see, the dynamic value of `person` is the mixure of the data and its structure: + +```scala +// The output pretty printed manually +Record( + Nominal(Chunk(dev,zio,quickstart),Chunk(),Person), + ListMap( + name -> Primitive(John Doe,string), + age -> Primitive(42,int) + ) +) +``` + +This is in contrast to the relational database model, where the data structure is stored in the database schema and the data itself is stored in a separate location. + +However, when we switch to document-based database models, such as JSON or XML, we can store both the data and its structure together. The JSON data model serves as a good example of self-describing data, as it allows us not only to include the data itself but also to add type information within the JSON. In this way, there is no need for a separate schema and data; everything is combined into a single entity. + +## Converting to/from DynamicValue + +With a `Schema[A]`, we can convert any value of type `A` to a `DynamicValue` and conversely we can convert it back to `A`: + +```scala +sealed trait Schema[A] { + def toDynamic(value: A): DynamicValue + + def fromDynamic(value: DynamicValue): scala.util.Either[String, A] +} +``` + +The `toDynamic` operation erases the type information of the value and places it into the value (the dynamic value) itself. The `fromDynamic` operation does the opposite: it takes the type information from the dynamic value and uses it to reconstruct the original value. diff --git a/docs/sidebars.js b/docs/sidebars.js index d8dcf247f..1c8b90a0e 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -21,6 +21,7 @@ const sidebars = { "operations", "transforming-schemas", "codecs", + "dynamic-data-representation", "protobuf-example", "combining-different-encoders", { From 400b620776c32b6bb47470c76f90e351099e2ca2 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Sun, 16 Jul 2023 13:33:01 +0330 Subject: [PATCH 38/63] dynamic value migration. --- docs/dynamic-data-representation.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/dynamic-data-representation.md b/docs/dynamic-data-representation.md index b5d872cbd..b5156bb10 100644 --- a/docs/dynamic-data-representation.md +++ b/docs/dynamic-data-representation.md @@ -53,3 +53,15 @@ sealed trait Schema[A] { ``` The `toDynamic` operation erases the type information of the value and places it into the value (the dynamic value) itself. The `fromDynamic` operation does the opposite: it takes the type information from the dynamic value and uses it to reconstruct the original value. + +## DynamicValue Migration + +By having the type information embedded in the data itself, we can perform migrations of the data easily by applying sequence of migration steps to the data. + +```scala +trait DynamicValue { + def transform(transforms: Chunk[Migration]): Either[String, DynamicValue] +} +``` + +We will discuss migrations in more detail in the next section. \ No newline at end of file From f8b8e6dc89373aceb9cfca1433268a33081e47f0 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 17 Jul 2023 14:54:35 +0330 Subject: [PATCH 39/63] schema migration. --- docs/dynamic-data-representation.md | 12 --- docs/operations.md | 17 ---- docs/schema-migration.md | 130 ++++++++++++++++++++++++++++ docs/sidebars.js | 1 + 4 files changed, 131 insertions(+), 29 deletions(-) create mode 100644 docs/schema-migration.md diff --git a/docs/dynamic-data-representation.md b/docs/dynamic-data-representation.md index b5156bb10..b5d872cbd 100644 --- a/docs/dynamic-data-representation.md +++ b/docs/dynamic-data-representation.md @@ -53,15 +53,3 @@ sealed trait Schema[A] { ``` The `toDynamic` operation erases the type information of the value and places it into the value (the dynamic value) itself. The `fromDynamic` operation does the opposite: it takes the type information from the dynamic value and uses it to reconstruct the original value. - -## DynamicValue Migration - -By having the type information embedded in the data itself, we can perform migrations of the data easily by applying sequence of migration steps to the data. - -```scala -trait DynamicValue { - def transform(transforms: Chunk[Migration]): Either[String, DynamicValue] -} -``` - -We will discuss migrations in more detail in the next section. \ No newline at end of file diff --git a/docs/operations.md b/docs/operations.md index 257e2912c..c3b7ab683 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -82,23 +82,6 @@ assert( ) ``` -## Automatic Migrations - -With ZIO Schema, we can automatically migrate data from one version of a schema to another. As software evolves, we often need to add, change or remove old fields. ZIO Schema provides two methods called `migrate` and `coerce` which help migrate the old schema to the new one: - -```scala -sealed trait Schema[A] { - def migrate[B](newSchema: Schema[B]): Either[String, A => scala.util.Either[String, B]] - - def coerce[B](newSchema: Schema[B]): Either[String, Schema[B]] = - for { - f <- self.migrate(newSchema) - g <- newSchema.migrate(self) - } yield self.transformOrFail(f, g) -} -``` - -The `migrate` method takes a new schema and returns a function that can migrate values of the old schema to values of the new schema as a `Right` value of `Either`. If the schemas have unambiguous transformations or are incompatible, the method returns a `Left` value containing an error message. ## Schema Serialization diff --git a/docs/schema-migration.md b/docs/schema-migration.md new file mode 100644 index 000000000..5ab49e479 --- /dev/null +++ b/docs/schema-migration.md @@ -0,0 +1,130 @@ +--- +id: schema-migration +title: "Schema Migration" +--- + +## Automatic Migration + +With ZIO Schema, we can automatically migrate data from one version of a schema to another. As software evolves, we often need to add, change or remove old fields. ZIO Schema provides two methods called `migrate` and `coerce` which help migrate the old schema to the new one: + +```scala +sealed trait Schema[A] { + def migrate[B](newSchema: Schema[B]): Either[String, A => scala.util.Either[String, B]] + + def coerce[B](newSchema: Schema[B]): Either[String, Schema[B]] +} +``` + +The `migrate` method takes a new schema and returns a function that can migrate values of the old schema to values of the new schema as a `Right` value of `Either`. If the schemas have unambiguous transformations or are incompatible, the method returns a `Left` value containing an error message. + +## Manual Migration + +By having `DynamicValue` which its type information embedded in the data itself, we can perform migrations of the data easily by applying a sequence of migration steps to the data. + +```scala +trait DynamicValue { + def transform(transforms: Chunk[Migration]): Either[String, DynamicValue] +} +``` + +The `Migration` is a sealed trait with several subtypes: + +```scala +sealed trait Migration +object Migration { + final case class AddNode(override val path: NodePath, node: MetaSchema) extends Migration + + final case class DeleteNode(override val path: NodePath) extends Migration + + final case class AddCase(override val path: NodePath, node: MetaSchema) extends Migration + + // ... +} +``` + +Using the `Migration` ADT we can describe the migration steps and then we can apply them to the `DynamicValue`. Let's try a simple example: + +```scala mdoc:compile-only +import zio.Chunk +import zio.schema.meta.Migration.DeleteNode +import zio.schema.meta.{Migration, NodePath} +import zio.schema.{DeriveSchema, Schema} + +case class Person1(name: String, age: Int) + +object Person1 { + implicit val schema: Schema[Person1] = DeriveSchema.gen +} + +case class Person2(name: String) + +object Person2 { + implicit val schema: Schema[Person2] = DeriveSchema.gen +} + +val person1: Person1 = Person1("John Doe", 42) + +val migrations: Chunk[Migration] = Chunk(DeleteNode(NodePath.root / "age")) + +val person2 = DeriveSchema + .gen[Person1] + .toDynamic(person1) + .transform(migrations) + .flatMap(_.toTypedValue[Person2]) + +assert(person2 == Right(Person2("John Doe"))) +``` + +## Deriving Migrations + +ZIO Schema provides a way to derive migrations automatically using the `Migration.derive` operation: + +```scala mdoc:compile-only +object Migration { + def derive(from: MetaSchema, to: MetaSchema): Either[String, Chunk[Migration]] +} +``` + +It takes two `MetaSchema` values, the old and the new schema, and returns a `Chunk[Migration]` that describes the migrations steps. Let's try a simple example: + +```scala mdoc:compile-only +import zio.schema._ +import zio.schema.meta._ + +case class Person1(name: String, age: Int, language: String, height: Int) + +object Person1 { + implicit val schema: Schema[Person1] = DeriveSchema.gen +} + +case class Person2( + name: String, + role: String, + language: Set[String], + height: Double +) + +object Person2 { + implicit val schema: Schema[Person2] = DeriveSchema.gen +} + +val migrations = Migration.derive( + MetaSchema.fromSchema(Person1.schema), + MetaSchema.fromSchema(Person2.schema) +) + +println(migrations) +``` + +The output of the above code is: + +```scala +Right( + Chunk(IncrementDimensions(Chunk(language,item),1), + ChangeType(Chunk(height),double), + AddNode(Chunk(role),string), + DeleteNode(Chunk(age))) +) +``` + +This output describes a series of migration steps that should be applied to the old schema to be transformed into the new schema. diff --git a/docs/sidebars.js b/docs/sidebars.js index 1c8b90a0e..80bdbcd32 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -19,6 +19,7 @@ const sidebars = { ], }, "operations", + "schema-migration", "transforming-schemas", "codecs", "dynamic-data-representation", From de2767c72c2940cb9886a89ca70426a99bc60a94 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 17 Jul 2023 15:08:28 +0330 Subject: [PATCH 40/63] derive ordering. --- docs/derive-ordering.md | 30 ++++++++++++++++++++++++++++++ docs/operations.md | 27 --------------------------- docs/schema-migration.md | 2 +- docs/sidebars.js | 1 + 4 files changed, 32 insertions(+), 28 deletions(-) create mode 100644 docs/derive-ordering.md diff --git a/docs/derive-ordering.md b/docs/derive-ordering.md new file mode 100644 index 000000000..734395551 --- /dev/null +++ b/docs/derive-ordering.md @@ -0,0 +1,30 @@ +--- +id: deriving-ordering +title: "Deriving Ordering" +--- + +Standard Scala library provides a type class called `Ordering[A]` that allows us to compare values of type `A`. ZIO Schema provides a method called `ordering` that generates an `Ordering[A]` instance for the underlying type described by the schema: + +```scala +sealed trait Schema[A] { + def ordering: Ordering[A] +} +``` + +Here is an example, where it helps us to sort the list of `Person`: + +```scala mdoc:compile-only +import zio.schema._ + +case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = DeriveSchema.gen[Person] +} + +val sortedList: Seq[Person] = + List( + Person("John", 42), + Person("Jane", 34) + ).sorted(Person.schema.ordering) +``` diff --git a/docs/operations.md b/docs/operations.md index c3b7ab683..1c75455ef 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -18,33 +18,6 @@ sealed trait Schema[A] { ZIO Schema have out of the box default values for all standard types, such as `String`, `Int`, `Boolean`, ..., `LocalDateTime` and `UUID`. For example, the default value of a schema for `String` is the empty string, and the default value of a schema for `Int` is `0`. -## Generate Orderings for Schemas - -Standard Scala library provides a type class called `Ordering[A]` that allows us to compare values of type `A`. ZIO Schema provides a method called `ordering` that generates an `Ordering[A]` instance for the underlying type described by the schema: - -```scala -sealed trait Schema[A] { - def ordering: Ordering[A] -} -``` - -Here is an example, where it helps us to sort the list of `Person`: - -```scala mdoc:compile-only -import zio.schema._ - -case class Person(name: String, age: Int) - -object Person { - implicit val schema: Schema[Person] = DeriveSchema.gen[Person] -} - -val sortedList: Seq[Person] = - List( - Person("John", 42), - Person("Jane", 34) - ).sorted(Person.schema.ordering) -``` ## Diffing and Patching diff --git a/docs/schema-migration.md b/docs/schema-migration.md index 5ab49e479..e92374542 100644 --- a/docs/schema-migration.md +++ b/docs/schema-migration.md @@ -79,7 +79,7 @@ assert(person2 == Right(Person2("John Doe"))) ZIO Schema provides a way to derive migrations automatically using the `Migration.derive` operation: -```scala mdoc:compile-only +```scala object Migration { def derive(from: MetaSchema, to: MetaSchema): Either[String, Chunk[Migration]] } diff --git a/docs/sidebars.js b/docs/sidebars.js index 80bdbcd32..6107df4d7 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -19,6 +19,7 @@ const sidebars = { ], }, "operations", + "deriving-ordering", "schema-migration", "transforming-schemas", "codecs", From 2c0ab5220f0794b34be6851734450f52c9e39602 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 17 Jul 2023 15:29:04 +0330 Subject: [PATCH 41/63] separate article for getting the default value. --- docs/getting-the-default-value.md | 14 ++++++++++++++ docs/operations.md | 11 ----------- docs/sidebars.js | 1 + 3 files changed, 15 insertions(+), 11 deletions(-) create mode 100644 docs/getting-the-default-value.md diff --git a/docs/getting-the-default-value.md b/docs/getting-the-default-value.md new file mode 100644 index 000000000..6c24c5f6d --- /dev/null +++ b/docs/getting-the-default-value.md @@ -0,0 +1,14 @@ +--- +id: getting-the-default-value +title: "Getting The Default Value" +--- + +ZIO Schema provides a method called `defaultValue` that returns the default value of the underlying type described by the schema. This method returns a `scala.util.Either[String, A]` value, where `A` is the type described by the schema. If the schema does not have a default value, the method returns a `Left` value containing an error message. Otherwise, it returns a `Right` value containing the default value: + +```scala +sealed trait Schema[A] { + def defaultValue: scala.util.Either[String, A] +} +``` + +ZIO Schema have out of the box default values for all standard types, such as `String`, `Int`, `Boolean`, ..., `LocalDateTime` and `UUID`. For example, the default value of a schema for `String` is the empty string, and the default value of a schema for `Int` is `0`. diff --git a/docs/operations.md b/docs/operations.md index 1c75455ef..c71c8b9de 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -6,17 +6,6 @@ sidebar_label: "Operations" Once we have defined our schemas, we can use them to perform a variety of operations. In this section, we will explore some of the most common operations that we can perform on schemas. -## Getting the Default Value of a Schema - -ZIO Schema provides a method called `defaultValue` that returns the default value of the underlying type described by the schema. This method returns a `scala.util.Either[String, A]` value, where `A` is the type described by the schema. If the schema does not have a default value, the method returns a `Left` value containing an error message. Otherwise, it returns a `Right` value containing the default value: - -```scala -sealed trait Schema[A] { - def defaultValue: scala.util.Either[String, A] -} -``` - -ZIO Schema have out of the box default values for all standard types, such as `String`, `Int`, `Boolean`, ..., `LocalDateTime` and `UUID`. For example, the default value of a schema for `String` is the empty string, and the default value of a schema for `Int` is `0`. ## Diffing and Patching diff --git a/docs/sidebars.js b/docs/sidebars.js index 6107df4d7..84bb2a160 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -19,6 +19,7 @@ const sidebars = { ], }, "operations", + "getting-the-default-value", "deriving-ordering", "schema-migration", "transforming-schemas", From 2ca42c5f94b6db58cabc3710a5033d1c6aa1e524 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 17 Jul 2023 15:31:23 +0330 Subject: [PATCH 42/63] separate article for diffing and patching. --- docs/diffing-and-patching.md | 38 +++++++++++++++++++++++++++++++++++ docs/operations.md | 39 ------------------------------------ docs/sidebars.js | 1 + 3 files changed, 39 insertions(+), 39 deletions(-) create mode 100644 docs/diffing-and-patching.md diff --git a/docs/diffing-and-patching.md b/docs/diffing-and-patching.md new file mode 100644 index 000000000..a747304b0 --- /dev/null +++ b/docs/diffing-and-patching.md @@ -0,0 +1,38 @@ +--- +id: diffing-and-patching +title: "Diffing and Patching" +--- + +ZIO Schema provides two methods called `diff` and `patch`: + +```scala +sealed trait Schema[A] { + def diff(thisValue: A, thatValue: A): Patch[A] + + def patch(oldValue: A, diff: Patch[A]): scala.util.Either[String, A] +} +``` + +The `diff` method takes two values of the same type `A` and returns a `Patch[A]` value that describes the differences between the two values. conversely, the `patch` method takes a value of type `A` and a `Patch[A]` value and returns a new value of type `A` that is the result of applying the patch to the original value. + +Here is a simple example that demonstrate the how to use `diff` and `patch`: + +```scala mdoc:compile-only +import zio.schema._ + +case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = DeriveSchema.gen[Person] +} + +val oldValue = Person("John", 42) +val newValue = Person("John", 43) + +val patch: Patch[Person] = + Person.schema.diff(oldValue, newValue) + +assert( + Person.schema.patch(oldValue, patch) == Right(newValue) +) +``` diff --git a/docs/operations.md b/docs/operations.md index c71c8b9de..dbd3a2a21 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -6,45 +6,6 @@ sidebar_label: "Operations" Once we have defined our schemas, we can use them to perform a variety of operations. In this section, we will explore some of the most common operations that we can perform on schemas. - - -## Diffing and Patching - -ZIO Schema provides two methods called `diff` and `patch`: - -```scala -sealed trait Schema[A] { - def diff(thisValue: A, thatValue: A): Patch[A] - - def patch(oldValue: A, diff: Patch[A]): scala.util.Either[String, A] -} -``` - -The `diff` method takes two values of the same type `A` and returns a `Patch[A]` value that describes the differences between the two values. conversely, the `patch` method takes a value of type `A` and a `Patch[A]` value and returns a new value of type `A` that is the result of applying the patch to the original value. - -Here is a simple example that demonstrate the how to use `diff` and `patch`: - -```scala mdoc:compile-only -import zio.schema._ - -case class Person(name: String, age: Int) - -object Person { - implicit val schema: Schema[Person] = DeriveSchema.gen[Person] -} - -val oldValue = Person("John", 42) -val newValue = Person("John", 43) - -val patch: Patch[Person] = - Person.schema.diff(oldValue, newValue) - -assert( - Person.schema.patch(oldValue, patch) == Right(newValue) -) -``` - - ## Schema Serialization In distributed systems, we often need to move computations to data instead of moving data to computations. The data is big and the network is slow, so moving it is expensive and sometimes impossible due to the volume of data. So in distributed systems, we would like to move our functions to the data and apply the data to the functions and gather the results back. diff --git a/docs/sidebars.js b/docs/sidebars.js index 84bb2a160..a145be8aa 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -20,6 +20,7 @@ const sidebars = { }, "operations", "getting-the-default-value", + "diffing-and-patching", "deriving-ordering", "schema-migration", "transforming-schemas", From bff228812eb9c26832ddec639c7c681118e7afc1 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 17 Jul 2023 15:56:19 +0330 Subject: [PATCH 43/63] separate other articles. --- docs/operations.md | 14 -------------- docs/serialization-of-the-schema-itself.md | 17 +++++++++++++++++ docs/sidebars.js | 3 ++- ...he-default-value.md => the-default-value.md} | 3 ++- 4 files changed, 21 insertions(+), 16 deletions(-) create mode 100644 docs/serialization-of-the-schema-itself.md rename docs/{getting-the-default-value.md => the-default-value.md} (93%) diff --git a/docs/operations.md b/docs/operations.md index dbd3a2a21..ece308c1e 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -5,17 +5,3 @@ sidebar_label: "Operations" --- Once we have defined our schemas, we can use them to perform a variety of operations. In this section, we will explore some of the most common operations that we can perform on schemas. - -## Schema Serialization - -In distributed systems, we often need to move computations to data instead of moving data to computations. The data is big and the network is slow, so moving it is expensive and sometimes impossible due to the volume of data. So in distributed systems, we would like to move our functions to the data and apply the data to the functions and gather the results back. - -So we need a way to serialize our computations and send them through the network. In ZIO Schema, each schema itself has a schema, so we can treat the structure as pure data! we can serialize our schemas by calling the `serializable` method: - -```scala -sealed trait Schema[A] { - def serializable: Schema[Schema[A]] -} -``` - -By calling this method, we can get the schema of a schema. So we can serialize it and send it across the wire, and it can be deserialized on the other side. After deserializing it, we have a schema that is isomorphic to the original schema. So all the operations that we can perform on the original type `A`, we can perform on any value that is isomorphic to `A` on the other side. diff --git a/docs/serialization-of-the-schema-itself.md b/docs/serialization-of-the-schema-itself.md new file mode 100644 index 000000000..6edd2df8a --- /dev/null +++ b/docs/serialization-of-the-schema-itself.md @@ -0,0 +1,17 @@ +--- +id: schema-serialization +title: "Serialization of the Schema Itself" +sidebar_label: "Schema Serialization" +--- + +In distributed systems, we often need to move computations to data instead of moving data to computations. The data is big and the network is slow, so moving it is expensive and sometimes impossible due to the volume of data. So in distributed systems, we would like to move our functions to the data and apply the data to the functions and gather the results back. + +So we need a way to serialize our computations and send them through the network. In ZIO Schema, each schema itself has a schema, so we can treat the structure as pure data! we can serialize our schemas by calling the `serializable` method: + +```scala +sealed trait Schema[A] { + def serializable: Schema[Schema[A]] +} +``` + +By calling this method, we can get the schema of a schema. So we can serialize it and send it across the wire, and it can be deserialized on the other side. After deserializing it, we have a schema that is isomorphic to the original schema. So all the operations that we can perform on the original type `A`, we can perform on any value that is isomorphic to `A` on the other side. diff --git a/docs/sidebars.js b/docs/sidebars.js index a145be8aa..29ae878cf 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -19,10 +19,11 @@ const sidebars = { ], }, "operations", - "getting-the-default-value", + "the-default-value", "diffing-and-patching", "deriving-ordering", "schema-migration", + "schema-serialization", "transforming-schemas", "codecs", "dynamic-data-representation", diff --git a/docs/getting-the-default-value.md b/docs/the-default-value.md similarity index 93% rename from docs/getting-the-default-value.md rename to docs/the-default-value.md index 6c24c5f6d..654c32fd6 100644 --- a/docs/getting-the-default-value.md +++ b/docs/the-default-value.md @@ -1,6 +1,7 @@ --- -id: getting-the-default-value +id: the-default-value title: "Getting The Default Value" +sidebar_label: "The Default Value" --- ZIO Schema provides a method called `defaultValue` that returns the default value of the underlying type described by the schema. This method returns a `scala.util.Either[String, A]` value, where `A` is the type described by the schema. If the schema does not have a default value, the method returns a `Left` value containing an error message. Otherwise, it returns a `Right` value containing the default value: From 745c8eb27608a0fee9023984173179d1966fbe3f Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 17 Jul 2023 20:34:46 +0330 Subject: [PATCH 44/63] validation section. --- docs/sidebars.js | 1 + docs/validating-types.md | 61 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 docs/validating-types.md diff --git a/docs/sidebars.js b/docs/sidebars.js index 29ae878cf..a0a0670b1 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -25,6 +25,7 @@ const sidebars = { "schema-migration", "schema-serialization", "transforming-schemas", + "validation", "codecs", "dynamic-data-representation", "protobuf-example", diff --git a/docs/validating-types.md b/docs/validating-types.md new file mode 100644 index 000000000..8498ff13c --- /dev/null +++ b/docs/validating-types.md @@ -0,0 +1,61 @@ +--- +id: validation +title: "Validation" +--- + +When we create a schema for a type, we can also specify validation rules for the type. Validations are a way to ensure that the data conforms to certain rules. + +Using `Schema#validate` we can validate a value against the validation rules of its schema: + +```scala +trait Schema[A] { + def validate(value: A)(implicit schema: Schema[A]): Chunk[ValidationError] +} +``` + +Let's write a schema for the `Person` case class and add validation rules to it. For example, we can specify that the `age` field must be greater than 0 and less than 120 and the `name` field must be non-empty: + +```scala mdoc:silent +import zio.Chunk +import zio.schema._ +import zio.schema.Schema._ +import zio.schema.validation.Validation + +case class Person(name: String, age: Int) + +object Person { + implicit val schema = CaseClass2( + id0 = TypeId.fromTypeName("Person"), + field01 = Schema.Field( + name0 = "name", + schema0 = Schema[String], + validation0 = Validation.minLength(1), + get0 = (p: Person) => p.name, + set0 = { (p: Person, s: String) => p.copy(name = s) } + ), + field02 = Schema.Field( + name0 = "age", + schema0 = Schema[Int], + validation0 = Validation.between(0, 120), + get0 = (p: Person) => p.age, + set0 = { (p: Person, age: Int) => p.copy(age = age) } + ), + construct0 = (name, age) => Person(name, age), + annotations0 = Chunk.empty + ) +} +``` + +Both fields of the `Person` case class have validation rules. Let's see what happens when we try to validate a `Person` value that does not conform to the validation rules: + +```scala mdoc:compile-only +import zio._ +import zio.schema.validation._ + +val result: Chunk[ValidationError] = Person.schema.validate(Person("John Doe", 130)) +println(result) +// Output: +// Chunk(EqualTo(130,120),LessThan(130,120)) +``` + +Due to the failed validation rules, a list of the specific rules that were not met is generated. In this case, it indicates that the age is not equal, or less than 120. From 40d6dacd2e07607f23b3552497eb1be285b9d22e Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Tue, 18 Jul 2023 13:28:35 +0330 Subject: [PATCH 45/63] update dynamic data representation article. --- docs/dynamic-data-representation.md | 50 ++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/docs/dynamic-data-representation.md b/docs/dynamic-data-representation.md index b5d872cbd..395f99978 100644 --- a/docs/dynamic-data-representation.md +++ b/docs/dynamic-data-representation.md @@ -5,6 +5,18 @@ title: "Dynamic Data Representation" DynamicValue is a way to describe the entire universe of possibilities for schema values. It does that in a way that we can interact with and introspect the data with its structure (type information). The structure of the data is baked into the data itself. +We can create a `DynamicValue` from a schema and a value using `DynamicValue.fromSchemaAndValue` (or `Schema#toDynamic`). We can turn it back into a typed value using `DynamicValue#toTypedValue`: + +```scala +trait DynamicValue { + def toTypedValue[A](implicit schema: Schema[A]): Either[String, A] = +} + +object DynamicValue { + def fromSchemaAndValue[A](schema: Schema[A], value: A): DynamicValue +} +``` + Let's create a simple instance of `Person("John Doe", 42)` and convert it to `DynamicValue`: ```scala mdoc:compile-only @@ -40,7 +52,7 @@ This is in contrast to the relational database model, where the data structure i However, when we switch to document-based database models, such as JSON or XML, we can store both the data and its structure together. The JSON data model serves as a good example of self-describing data, as it allows us not only to include the data itself but also to add type information within the JSON. In this way, there is no need for a separate schema and data; everything is combined into a single entity. -## Converting to/from DynamicValue +## Schema: Converting to/from DynamicValue With a `Schema[A]`, we can convert any value of type `A` to a `DynamicValue` and conversely we can convert it back to `A`: @@ -53,3 +65,39 @@ sealed trait Schema[A] { ``` The `toDynamic` operation erases the type information of the value and places it into the value (the dynamic value) itself. The `fromDynamic` operation does the opposite: it takes the type information from the dynamic value and uses it to reconstruct the original value. + +Please note that, if we have two types `A` and `B` that are isomorphic, we can convert a dynamic value of type `A` to a typed value of type `B` and vice versa: + +```scala mdoc:compile-only +import zio.schema._ + +case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = DeriveSchema.gen +} + +case class User(name: String, age: Int) + +object User { + implicit val schema: Schema[User] = DeriveSchema.gen +} + +val johnPerson = Person("John Doe", 42) +val johnUser = User("John Doe", 42) + +val dynamicJohnPerson = Person.schema.toDynamic(johnPerson) +val dynamicJohnUser = User.schema.toDynamic(johnUser) + +println(dynamicJohnPerson) +// Output: Record(Nominal(Chunk(dev,zio,quickstart),Chunk(Main),Person),ListMap(name -> Primitive(John Doe,string), age -> Primitive(42,int))) +println(dynamicJohnUser) +// Output: Record(Nominal(Chunk(dev,zio,quickstart),Chunk(Main),User),ListMap(name -> Primitive(John Doe,string), age -> Primitive(42,int))) + +assert(dynamicJohnPerson.toTypedValue[User] == Right(johnUser)) +assert(dynamicJohnUser.toTypedValue[Person] == Right(johnPerson)) +``` + +## Manipulating Dynamic Values + +When we turn a typed value `A` into a `DynamicValue`, we can manipulate its structure and data dynamically. For example, we can add a new field to a record or change the type of a field. This process is called dynamic value migration, which we will discuss in the [schema migration](schema-migration.md) section. From 1555fc8e591c87b945f064a7fdc2665c2988a9ea Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Wed, 19 Jul 2023 14:34:17 +0330 Subject: [PATCH 46/63] reified optics. --- docs/reified-optics.md | 257 +++++++++++++++++++++++++++++++++++++++++ docs/sidebars.js | 1 + 2 files changed, 258 insertions(+) create mode 100644 docs/reified-optics.md diff --git a/docs/reified-optics.md b/docs/reified-optics.md new file mode 100644 index 000000000..a41e84900 --- /dev/null +++ b/docs/reified-optics.md @@ -0,0 +1,257 @@ +--- +id: reified-optics +title: "Reified Optics" +--- + +Reified optics is a technique in functional programming that allows you to treat optics as first-class values. This means that we can pass them around, compose them, and store optics in data structures. Reified optics is one of the solutions to the problem of making computations as first-class values. + +Optics are a way of accessing and manipulating data in a functional way. They can be used to get, set, and update values in data structures, as well as to traverse and explore data. + +## Pure Optics (Manual Derivation) + +Before we dive into reified optics and how we can have an automatic derivation of optics, let's take a look at the pure optics and how should we create them manually. + +First, we should add `zio-schema-optics` to our `build.sbt` file: + +```scala +libraryDependencies += "dev.zio" %% "zio-schema-optics" % @VERSION@ +``` + +Now let's define a simple data type called `User` and create two optics for its `name` and `age` fields: + +```scala mdoc:silent +import zio.optics._ + +case class User(name: String, age: Int) + +val nameLens = Lens[User, String]( + user => Right(user.name), + name => user => Right(user.copy(name = name)) +) + +val ageLens = Lens[User, Int]( + user => Right(user.age), + age => user => Right(user.copy(age = age)) +) + +val ageAndNameLens = nameLens.zip(ageLens) +``` + +Now we can use these optics to get, set, and update values in the `Person` data structure: + +```scala mdoc:silent +import zio._ + +object Main extends ZIOAppDefault { + def run = + for { + _ <- ZIO.debug("Pure Optics") + user = User("John", 34) + + updatedUser1 <- ZIO.fromEither(nameLens.setOptic("Jane")(user)) + _ <- ZIO.debug(s"Name of user updated: $updatedUser1") + + updatedUser2 <- ZIO.fromEither(ageLens.setOptic(32)(user)) + _ <- ZIO.debug(s"Age of user updated: $updatedUser2") + + updatedUser3 <- ZIO.fromEither( + ageAndNameLens.set(("Jane", 32))(User("John", 34)) + ) + _ <- ZIO.debug(s"Name and age of the user updated: $updatedUser3") + } yield () +} +``` + +## Reified Optics (Automatic Derivation) + +With reified optics, we can derive optics automatically from a schema. This means that we don't have to write the optics manually, but instead, we can use the `Schema#makeAccessors` method which will derive the optics for us: + +```scala +trait Schema[A] { + def makeAccessors(b: AccessorBuilder): Accessors[b.Lens, b.Prism, b.Traversal] +} +``` + +It takes an `AccessorBuilder` which is an interface of the creation of optics: + +```scala +trait AccessorBuilder { + type Lens[F, S, A] + type Prism[F, S, A] + type Traversal[S, A] + + def makeLens[F, S, A]( + product: Schema.Record[S], + term: Schema.Field[S, A] + ): Lens[F, S, A] + + def makePrism[F, S, A]( + sum: Schema.Enum[S], + term: Schema.Case[S, A] + ): Prism[F, S, A] + + def makeTraversal[S, A]( + collection: Schema.Collection[S, A], + element: Schema[A] + ): Traversal[S, A] +} +``` + +It has three methods for creating three types of optics: + +- Lens is an optic used to get and update values in a product type. +- Prism is an optic used to get and update values in a sum type. +- Traversal is an optic used to get and update values in a collection type. + +Let's take a look at how we can derive optics using ZIO Schema. + +First we should add `zio-schema-optics` to our `build.sbt` file: + +```scala +libraryDependencies += "dev.zio" %% "zio-schema-optics" % @VERSION@ +``` + +This package contains a `ZioOpticsBuilder` which is an implementation of the `AccessorBuilder` interface based on ZIO Optics library. + +Now we are ready to try any of the following examples: + +### Lens + +Now we can derive the schema for our `User` data type in its companion object, and then derive optics using `Schema#makeAccessors` method: + +```scala mdoc:silent:reset +import zio._ +import zio.schema.DeriveSchema +import zio.schema.Schema.CaseClass2 +import zio.schema.optics.ZioOpticsBuilder + +case class User(name: String, age: Int) + +object User { + implicit val schema: CaseClass2[String, Int, User] = + DeriveSchema.gen[User].asInstanceOf[CaseClass2[String, Int, User]] + + val (nameLens, ageLens) = schema.makeAccessors(ZioOpticsBuilder) +} +``` + +Based on the type of the schema, the `makeAccessors` method will derive the proper optics for us. + +Now we can use these optics to update values in the `User` data structure: + +```scala mdoc:compile-only +object MainApp extends ZIOAppDefault { + def run = for { + _ <- ZIO.debug("Auto-derivation of Optics") + user = User("John", 42) + + updatedUser1 = User.nameLens.set("Jane")(user) + _ <- ZIO.debug(s"Name of user updated: $updatedUser1") + + updatedUser2 = User.ageLens.set(32)(user) + _ <- ZIO.debug(s"Age of user updated: $updatedUser2") + + nameAndAgeLens = User.nameLens.zip(User.ageLens) + updatedUser3 = nameAndAgeLens.set(("Jane", 32))(user) + _ <- ZIO.debug(s"Name and age of the user updated: $updatedUser3") + } yield () +} +``` + +Output: + +```scala +Auto-derivation of Lens Optics: +Name of user updated: Right(User(Jane,42)) +Age of user updated: Right(User(John,32)) +Name and age of the user updated: Right(User(Jane,32)) +``` + +### Prism + +```scala mdoc:compile-only +import zio._ +import zio.schema.Schema._ + +sealed trait Shape { + def area: Double +} + +case class Circle(radius: Double) extends Shape { + val area: Double = Math.PI * radius * radius +} + +case class Rectangle(width: Double, height: Double) extends Shape { + val area: Double = width * height +} + +object Shape { + implicit val schema: Enum2[Circle, Rectangle, Shape] = + DeriveSchema.gen[Shape].asInstanceOf[Enum2[Circle, Rectangle, Shape]] + + val (circlePrism, rectanglePrism) = + schema.makeAccessors(ZioOpticsBuilder) +} + +object MainApp extends ZIOAppDefault { + def run = for { + _ <- ZIO.debug("Auto-derivation of Prism Optics") + shape = Circle(1.2) + _ <- ZIO.debug(s"Original shape: $shape") + updatedShape <- ZIO.fromEither( + Shape.rectanglePrism.setOptic(Rectangle(2.0, 3.0))(shape) + ) + _ <- ZIO.debug(s"Updated shape: $updatedShape") + } yield () + +} +``` + +Output: + +```scala +Auto-derivation of Prism Optics: +Original shape: Circle(1.2) +Updated shape: Rectangle(2.0,3.0) +``` + +### Traversal + +```scala mdoc:compile-only +import zio._ +import zio.optics._ +import zio.schema.Schema._ +import zio.schema._ + +object IntList { + implicit val listschema = + Sequence[List[Int], Int, String]( + elementSchema = Schema[Int], + fromChunk = _.toList, + toChunk = i => Chunk.fromIterable(i), + annotations = Chunk.empty, + identity = "List" + ) + + val traversal: ZTraversal[List[Int], List[Int], Int, Int] = + listschema.makeAccessors(ZioOpticsBuilder) +} + +object MainApp extends ZIOAppDefault { + def run = for { + _ <- ZIO.debug("Auto-derivation of Traversal Optic:") + list = List(1, 2, 3, 4, 5) + _ <- ZIO.debug(s"Original list: $list") + updatedList <- ZIO.fromEither(IntList.traversal.set(Chunk(1, 5, 7))(list)) + _ <- ZIO.debug(s"Updated list: $updatedList") + } yield () +} +``` + +Output: + +```scala +Auto-derivation of Traversal Optic: +Original list: List(1, 2, 3, 4, 5) +Updated list: List(1, 5, 7, 4, 5) +``` diff --git a/docs/sidebars.js b/docs/sidebars.js index a0a0670b1..5e9099c35 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -28,6 +28,7 @@ const sidebars = { "validation", "codecs", "dynamic-data-representation", + "reified-optics", "protobuf-example", "combining-different-encoders", { From 78812e0d4f2e8e3653e19991690c1e439c3dd09e Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Sun, 30 Jul 2023 17:00:47 +0330 Subject: [PATCH 47/63] avro codecs. --- build.sbt | 2 +- docs/codecs.md | 212 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 213 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 1f40907fe..99bf8b3b3 100644 --- a/build.sbt +++ b/build.sbt @@ -341,5 +341,5 @@ lazy val docs = project |sbt test |```""".stripMargin ) - .dependsOn(zioSchemaJVM, zioSchemaProtobufJVM, zioSchemaJsonJVM) + .dependsOn(zioSchemaJVM, zioSchemaProtobufJVM, zioSchemaJsonJVM, zioSchemaOpticsJVM, zioSchemaAvroJVM) .enablePlugins(WebsitePlugin) diff --git a/docs/codecs.md b/docs/codecs.md index f6e5d25a6..a6f7c1a5e 100644 --- a/docs/codecs.md +++ b/docs/codecs.md @@ -19,6 +19,44 @@ It basically says: - `encoder[A]`: Given a `Schema[A]` it is capable of generating an `Encoder[A]` ( `A => Chunk[Byte]`) for any Schema. - `decoder[A]`: Given a `Schema[A]` it is capable of generating a `Decoder[A]` ( `Chunk[Byte] => Either[String, A]`) for any Schema. +## Binary Codecs + +The binary codecs are codecs that can encode/decode a value of some type `A` to/from binary format (e.g. `Chunk[Byte]`). In ZIO Schema, by having a `BinaryCodec[A]` instance, other than being able to encode/decode a value of type `A` to/from binary format, we can also encode/decode a stream of values of type `A` to/from a stream of binary format. + +```scala +import zio.Chunk +import zio.stream.ZPipeline + +trait Decoder[Whole, Element, +A] { + def decode(whole: Whole): Either[DecodeError, A] + def streamDecoder: ZPipeline[Any, DecodeError, Element, A] +} + +trait Encoder[Whole, Element, -A] { + def encode(value: A): Whole + def streamEncoder: ZPipeline[Any, Nothing, A, Element] +} + +trait Codec[Whole, Element, A] extends Encoder[Whole, Element, A] with Decoder[Whole, Element, A] + +trait BinaryCodec[A] extends Codec[Chunk[Byte], Byte, A] +``` + +To make it simpler, we can think of a `BinaryCodec[A]` as the following trait: + +```scala +import zio.Chunk +import zio.stream.ZPipeline + +trait BinaryCodec[A] { + def encode(value: A): Chunk[Byte] + def decode(whole: Chunk[Byte]): Either[DecodeError, A] + + def streamEncoder: ZPipeline[Any, Nothing, A, Byte] + def streamDecoder: ZPipeline[Any, DecodeError, Byte, A] +} +``` + Example of possible codecs are: - CSV Codec @@ -30,3 +68,177 @@ Example of possible codecs are: - Protobuf Codec (already available) - QueryString Codec - etc. + +## Avro + +To use the Avro codecs, we need to add the following dependency to our `build.sbt` file: + +```scala +libraryDependencies += "dev.zio" %% "zio-schema-avro" % @VERSION@ +``` + +It has two codecs: + +- An **AvroSchemaCodec** to serialize a `Schema[A]` to Avro JSON schema and deserialize an Avro JSON schema to a `Schema.GenericRecord`. +- An **AvroCodec** to serialize/deserialize the Avro binary serialization format. + +### AvroSchemaCodec + +Here is the definition of the `AvroSchemaCodec`: + +```scala +trait AvroSchemaCodec { + def encode(schema: Schema[_]): scala.util.Either[String, String] + def decode(bytes: Chunk[Byte]): scala.util.Either[String, Schema[_]] +} +``` + +The `encode` method takes a `Schema[_]` and returns an `Either[String, String]` where the `Right` side contains the Avro schema in JSON‌ format. + +The `decode` method takes a `Chunk[Byte]` which contains the Avro JSON Schema in binary format and returns an `Either[String, Schema[_]]` where the `Right` side contains the ZIO Schema in `GenericRecord` format. + +Here is an example of how to use it: + +```scala mdoc:compile-only +import zio._ +import zio.schema.Schema +import zio.schema.DeriveSchema +import zio.schema.codec.AvroSchemaCodec + +case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = DeriveSchema.gen +} + +object Main extends ZIOAppDefault { + def run = + for { + _ <- ZIO.debug("AvroSchemaCodec Example:") + avroSchema <- ZIO.fromEither(AvroSchemaCodec.encode(Person.schema)) + _ <- ZIO.debug(s"The person schema in Avro Schema JSON format: $avroSchema") + avroSchemaBinary = Chunk.fromArray(avroSchema.getBytes) + zioSchema <- ZIO.fromEither(AvroSchemaCodec.decode(avroSchemaBinary)) + _ <- ZIO.debug(s"The person schema in ZIO Schema GenericRecord format: $zioSchema") + } yield () +} +``` + +The output: + +```scala +AvroSchemaCodec Example: +The person schema in Avro Schema JSON format: {"type":"record","name":"Person","fields":[{"name":"name","type":"string"},{"name":"age","type":"int"}]} +The person schema in ZIO Schema GenericRecord format: GenericRecord(Nominal(Chunk(),Chunk(),Person),Field(name,Primitive(string,Chunk())) :*: Field(age,Primitive(int,Chunk())) :*: Empty,Chunk(name(Person))) +``` + +As we can see, we converted the `Schema[Person]` to Avro schema JSON format, and then we converted it back to the ZIO Schema `GenericRecord` format. + +### AvroCodec + +We can create a `BinaryCodec[A]` for any type `A` that has a `Schema[A]` instance using `AvroCodec.schemaBasedBinaryCodec`: + +```scala +object AvroCodec { + implicit def schemaBasedBinaryCodec[A](implicit schema: Schema[A]): BinaryCodec[A] = ??? +} +``` + +Now, let's write an example and see how it works: + +```scala mdoc:compile-only +import zio._ +import zio.schema.Schema +import zio.schema.DeriveSchema +import zio.schema.codec.{AvroCodec, BinaryCodec} + +case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = DeriveSchema.gen + implicit val binaryCodec: BinaryCodec[Person] = + AvroCodec.schemaBasedBinaryCodec[Person] +} + +object Main extends ZIOAppDefault { + def run = + for { + _ <- ZIO.debug("AvroCodec Example:") + encodedPerson = Person.binaryCodec.encode(Person("John", 42)) + _ <- ZIO.debug(s"encoded person object: ${toHex(encodedPerson)}") + decodedPerson <- ZIO.fromEither( + Person.binaryCodec.decode(encodedPerson) + ) + _ <- ZIO.debug(s"decoded person object: $decodedPerson") + } yield () + + def toHex(bytes: Chunk[Byte]): String = + bytes.map("%02x".format(_)).mkString(" ") +} +``` + +The output: + +```scala +AvroCodec Example: +encoded person object: 08 4a 6f 68 6e 54 +decoded person object: Person(John,42) +``` + +### Annotations + +The Apache Avro specification supports some attributes for describing the data which are not part of the default ZIO Schema. To support these extra metadata, we can use annotations defined in the `zio.schema.codec.AvroAnnotations` object. + +There tons of annotations that we can use. Let's introduce some of them: + +- `name(name: String)`: To change the name of a field or a record. +- `namespace(namespace: String)`: To add the namespace for a field or a record. +- `doc(doc: String)`: To add documentation to a field or a record. +- `aliases(aliases: Set[String])`: To add aliases to a field or a record. +- `avroEnum`: To treat a sealed trait as an Avro enum. +- `scale(scale: Int = 24)` and `precision(precision: Int = 48)`: To describe the scale and precision of a decimal field. +- `decimal(decimalType: DecimalType)`: Used to annotate a `BigInteger` or `BigDecimal` type to indicate the logical type encoding (avro bytes or avro fixed). +- `bytes(bytesType: BytesType)`: Used to annotate a Byte type to indicate the avro type encoding (avro bytes or avro fixed). +- `formatToString`: Used to annotate fields of type `LocalDate`, `LocalTime`, `LocalDateTime` or `Instant` in order to render them as a string using the given formatter instead of rendering them as avro logical types. +- `timeprecision(timeprecisionType: TimePrecisionType)`: Used to indicate the precision (millisecond precision or microsecond precision) of avro logical types `Time`, `Timestamp` and `Local timestamp` +- `error`: Used to annotate a record in order to render it as a avro error record +- `fieldOrder(fieldOrderType: FieldOrderType)`: Used to indicate the avro field order of a record + +For example, to change the name of a field in the Avro schema, we can use the `AvroAnnotations.name` annotation: + +```scala mdoc:compile-only +import zio.schema.Schema +import zio.schema.DeriveSchema +import zio.schema.codec.AvroAnnotations + +@AvroAnnotations.name("User") +case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = DeriveSchema.gen +} +``` + +Now, if we generate the Avro schema for the `Person` class, we will see that the name of the record is `User` instead of `Person`: + +```scala +import zio._ +import zio.schema.Schema +import zio.schema.DeriveSchema +import zio.schema.codec.AvroSchemaCodec + +object Main extends ZIOAppDefault { + def run = + for { + _ <- ZIO.debug("AvroSchemaCodec Example with annotations:") + avroSchema <- ZIO.fromEither(AvroSchemaCodec.encode(Person.schema)) + _ <- ZIO.debug(s"The person schema in Avro Schema JSON format: $avroSchema") + } yield () +} +``` + +The output: + +```scala +The person schema in Avro Schema JSON format: {"type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"age","type":{"type":"bytes","logicalType":"decimal","precision":48,"scale":24}}]} +``` From 5e465a819e676b866c6aba9cfe3f3acb7d13c834 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 31 Jul 2023 11:11:42 +0330 Subject: [PATCH 48/63] add bson codecs. --- docs/{codecs.md => codecs/apache-avro.md} | 77 ++---------------- docs/codecs/bson.md | 96 +++++++++++++++++++++++ docs/codecs/index.md | 78 ++++++++++++++++++ docs/sidebars.js | 11 ++- 4 files changed, 191 insertions(+), 71 deletions(-) rename docs/{codecs.md => codecs/apache-avro.md} (74%) create mode 100644 docs/codecs/bson.md create mode 100644 docs/codecs/index.md diff --git a/docs/codecs.md b/docs/codecs/apache-avro.md similarity index 74% rename from docs/codecs.md rename to docs/codecs/apache-avro.md index a6f7c1a5e..20c5ded39 100644 --- a/docs/codecs.md +++ b/docs/codecs/apache-avro.md @@ -1,75 +1,10 @@ --- -id: codecs -title: "Codecs" +id: apache-avro +title: "Apache Avro Codecs" +sidebar_label: "Apache Avro" --- -Once we have our schema, we can combine it with a codec. A codec is a combination of a schema and a serializer. Unlike codecs in other libraries, a codec in ZIO Schema has no type parameter: - -```scala -trait Codec { - def encoder[A](schema: Schema[A]): ZTransducer[Any, Nothing, A, Byte] - def decoder[A](schema: Schema[A]): ZTransducer[Any, String, Byte, A] - - def encode[A](schema: Schema[A]): A => Chunk[Byte] - def decode[A](schema: Schema[A]): Chunk[Byte] => Either[String, A] -} -``` - -It basically says: -- `encoder[A]`: Given a `Schema[A]` it is capable of generating an `Encoder[A]` ( `A => Chunk[Byte]`) for any Schema. -- `decoder[A]`: Given a `Schema[A]` it is capable of generating a `Decoder[A]` ( `Chunk[Byte] => Either[String, A]`) for any Schema. - -## Binary Codecs - -The binary codecs are codecs that can encode/decode a value of some type `A` to/from binary format (e.g. `Chunk[Byte]`). In ZIO Schema, by having a `BinaryCodec[A]` instance, other than being able to encode/decode a value of type `A` to/from binary format, we can also encode/decode a stream of values of type `A` to/from a stream of binary format. - -```scala -import zio.Chunk -import zio.stream.ZPipeline - -trait Decoder[Whole, Element, +A] { - def decode(whole: Whole): Either[DecodeError, A] - def streamDecoder: ZPipeline[Any, DecodeError, Element, A] -} - -trait Encoder[Whole, Element, -A] { - def encode(value: A): Whole - def streamEncoder: ZPipeline[Any, Nothing, A, Element] -} - -trait Codec[Whole, Element, A] extends Encoder[Whole, Element, A] with Decoder[Whole, Element, A] - -trait BinaryCodec[A] extends Codec[Chunk[Byte], Byte, A] -``` - -To make it simpler, we can think of a `BinaryCodec[A]` as the following trait: - -```scala -import zio.Chunk -import zio.stream.ZPipeline - -trait BinaryCodec[A] { - def encode(value: A): Chunk[Byte] - def decode(whole: Chunk[Byte]): Either[DecodeError, A] - - def streamEncoder: ZPipeline[Any, Nothing, A, Byte] - def streamDecoder: ZPipeline[Any, DecodeError, Byte, A] -} -``` - -Example of possible codecs are: - -- CSV Codec -- JSON Codec (already available) -- Apache Avro Codec (in progress) -- Apache Thrift Codec (in progress) -- XML Codec -- YAML Codec -- Protobuf Codec (already available) -- QueryString Codec -- etc. - -## Avro +## Installation To use the Avro codecs, we need to add the following dependency to our `build.sbt` file: @@ -77,6 +12,8 @@ To use the Avro codecs, we need to add the following dependency to our `build.sb libraryDependencies += "dev.zio" %% "zio-schema-avro" % @VERSION@ ``` +## Codecs + It has two codecs: - An **AvroSchemaCodec** to serialize a `Schema[A]` to Avro JSON schema and deserialize an Avro JSON schema to a `Schema.GenericRecord`. @@ -185,7 +122,7 @@ encoded person object: 08 4a 6f 68 6e 54 decoded person object: Person(John,42) ``` -### Annotations +## Annotations The Apache Avro specification supports some attributes for describing the data which are not part of the default ZIO Schema. To support these extra metadata, we can use annotations defined in the `zio.schema.codec.AvroAnnotations` object. diff --git a/docs/codecs/bson.md b/docs/codecs/bson.md new file mode 100644 index 000000000..31102d71b --- /dev/null +++ b/docs/codecs/bson.md @@ -0,0 +1,96 @@ +--- +id: bson +title: "Bson Codecs" +--- + +## Introduction + +BSON (Binary JSON) is a binary serialization format used to store and exchange data efficiently. In this article, we will explore how to derive BSON codecs from a ZIO Schema. The `zio-schema-bson` module, provides support for deriving codecs from ZIO Schema, and makes it easy to communicate data in BSON format. + +## Installation + +To use BSON codecs, you need to add the following dependency to your Scala project: + +```scala +libraryDependencies += "dev.zio" %% "zio-schema-bson" % @VERSION@ +``` + +## BsonSchemaCodec + +The ZIO Schema library provides the `BsonSchemaCodec` object in the `zio.schema.codec` package. This object offers three methods that allow us to derive BSON encoders, decoders, and codecs from ZIO Schemas: + +```scala +object BsonSchemaCodec { + def bsonEncoder[A](schema: Schema[A]): BsonEncoder[A] + def bsonDecoder[A](schema: Schema[A]): BsonDecoder[A] + def bsonCodec[A](schema: Schema[A]): BsonCodec[A] +} +``` + +It has three methods, by calling each of them, we can get a `BsonEncoder[A]`, `BsonDecoder[A]`, or `BsonCodec[A]` from a `Schema[A]`. Let's see a simplified version of each of these traits: + +### 1. BsonEncoder + +The `BsonEncoder` trait defines a type class for encoding a value of type `A` into a BSON value. The `toBsonValue` method accomplishes this conversion: + +```scala +trait BsonEncoder[A] { + def toBsonValue(value: A): BsonValue +} +``` + +### 2. BsonDecoder + +The BsonDecoder trait defines a type class for decoding a BSON value into a value of type A. The fromBsonValue method handles this conversion and returns an Either indicating success or an error: + +```scala +trait BsonDecoder[A] { + def fromBsonValue(value: BsonValue): Either[BsonDecoder.Error, A] +} +``` + +### 3. BsonCodec + +The `BsonCodec` case class combines both the BSON encoder and decoder for a specific type `A`: + +```scala +final case class BsonCodec[A]( + encoder: BsonEncoder[A], + decoder: BsonDecoder[A] +) +``` + +## Example: Deriving a Bson Codec for a Case Class + +Let's see an example of how to derive a BSON codec for a case class using ZIO Schema: + +```scala mdoc:compile-only +import org.bson.BsonValue +import zio._ +import zio.bson._ +import zio.schema.codec._ +import zio.schema.{DeriveSchema, Schema} + +case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = DeriveSchema.gen + implicit val bsonCodec: BsonCodec[Person] = + BsonSchemaCodec.bsonCodec(Person.schema) +} + +object Main extends ZIOAppDefault { + def run = for { + _ <- ZIO.debug("Bson Example:") + person: Person = Person("John", 42) + encoded: BsonValue = person.toBsonValue + _ <- ZIO.debug(s"person object encoded to BsonValue: $encoded") + decoded <- ZIO.fromEither(encoded.as[Person]) + _ <- ZIO.debug(s"BsonValue of person object decoded to Person: $decoded") + } yield () +} +``` + +In the example above, we defined a case class `Person` with fields `name` and `age`. We then derived a ZIO Schema for the `Person` case class using `DeriveSchema.gen`. + +The `BsonSchemaCodec.bsonCodec` method allowed us to create a BSON codec for the `Person` case class by passing its corresponding ZIO Schema. Now, we can effortlessly encode `Person` objects to BSON and decode BSON values back to Person instances. diff --git a/docs/codecs/index.md b/docs/codecs/index.md new file mode 100644 index 000000000..e969bb29d --- /dev/null +++ b/docs/codecs/index.md @@ -0,0 +1,78 @@ +--- +id: index +title: "Introduction to ZIO Schema Codecs" +sidebar_label: "Codecs" +--- + +Once we generate a schema for a type, we can derive a codec for that type. + +A codec is a utility that can encode/decode a value of some type `A` to/from some format (e.g. binary format, JSON, etc.) + +## Codec + +Unlike codecs in other libraries, a codec in ZIO Schema has no type parameter: + +```scala +trait Codec { + def encoder[A](schema: Schema[A]): ZTransducer[Any, Nothing, A, Byte] + def decoder[A](schema: Schema[A]): ZTransducer[Any, String, Byte, A] + + def encode[A](schema: Schema[A]): A => Chunk[Byte] + def decode[A](schema: Schema[A]): Chunk[Byte] => Either[String, A] +} +``` + +The `Codec` trait has two basic methods: + +- `encode[A]`: Given a `Schema[A]` it is capable of generating an `Encoder[A]` ( `A => Chunk[Byte]`) for any Schema. +- `decode[A]`: Given a `Schema[A]` it is capable of generating a `Decoder[A]` ( `Chunk[Byte] => Either[String, A]`) for any Schema. + +## Binary Codecs + +The binary codecs are codecs that can encode/decode a value of some type `A` to/from binary format (e.g. `Chunk[Byte]`). In ZIO Schema, by having a `BinaryCodec[A]` instance, other than being able to encode/decode a value of type `A` to/from binary format, we can also encode/decode a stream of values of type `A` to/from a stream of binary format. + +```scala +import zio.Chunk +import zio.stream.ZPipeline + +trait Decoder[Whole, Element, +A] { + def decode(whole: Whole): Either[DecodeError, A] + def streamDecoder: ZPipeline[Any, DecodeError, Element, A] +} + +trait Encoder[Whole, Element, -A] { + def encode(value: A): Whole + def streamEncoder: ZPipeline[Any, Nothing, A, Element] +} + +trait Codec[Whole, Element, A] extends Encoder[Whole, Element, A] with Decoder[Whole, Element, A] + +trait BinaryCodec[A] extends Codec[Chunk[Byte], Byte, A] +``` + +To make it simpler, we can think of a `BinaryCodec[A]` as the following trait: + +```scala +import zio.Chunk +import zio.stream.ZPipeline + +trait BinaryCodec[A] { + def encode(value: A): Chunk[Byte] + def decode(whole: Chunk[Byte]): Either[DecodeError, A] + + def streamEncoder: ZPipeline[Any, Nothing, A, Byte] + def streamDecoder: ZPipeline[Any, DecodeError, Byte, A] +} +``` + +Example of possible codecs are: + +- CSV Codec +- JSON Codec (already available) +- Apache Avro Codec (in progress) +- Apache Thrift Codec (in progress) +- XML Codec +- YAML Codec +- Protobuf Codec (already available) +- QueryString Codec +- etc. diff --git a/docs/sidebars.js b/docs/sidebars.js index 5e9099c35..86e263726 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -26,7 +26,16 @@ const sidebars = { "schema-serialization", "transforming-schemas", "validation", - "codecs", + { + type: "category", + label: "Codecs", + collapsed: true, + link: { type: "doc", id: "codecs/index" }, + items: [ + "codecs/apache-avro", + "codecs/bson" + ], + }, "dynamic-data-representation", "reified-optics", "protobuf-example", From 13d0ae6fee0bfa81db558de7abff04246578e7ab Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 31 Jul 2023 11:26:08 +0330 Subject: [PATCH 49/63] improve apache avro doc. --- docs/codecs/apache-avro.md | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/docs/codecs/apache-avro.md b/docs/codecs/apache-avro.md index 20c5ded39..1c8a82381 100644 --- a/docs/codecs/apache-avro.md +++ b/docs/codecs/apache-avro.md @@ -4,6 +4,10 @@ title: "Apache Avro Codecs" sidebar_label: "Apache Avro" --- +## Introduction + +Apache Avro is a popular data serialization format used in distributed systems, particularly in the Apache Hadoop ecosystem. In this article, we will explore how to work with Apache Avro codecs in Scala using the ZIO Schema. Avro codecs allow us to easily serialize and deserialize data in Avro's binary and JSON formats. + ## Installation To use the Avro codecs, we need to add the following dependency to our `build.sbt` file: @@ -21,7 +25,7 @@ It has two codecs: ### AvroSchemaCodec -Here is the definition of the `AvroSchemaCodec`: +The `AvroSchemaCodec` provides methods to encode a `Schema[_]` to Avro JSON schema and decode an Avro JSON schema to a `Schema[_]` ([`Schema.GenericRecord`](../dynamic-data-representation.md)): ```scala trait AvroSchemaCodec { @@ -128,18 +132,18 @@ The Apache Avro specification supports some attributes for describing the data w There tons of annotations that we can use. Let's introduce some of them: -- `name(name: String)`: To change the name of a field or a record. -- `namespace(namespace: String)`: To add the namespace for a field or a record. -- `doc(doc: String)`: To add documentation to a field or a record. -- `aliases(aliases: Set[String])`: To add aliases to a field or a record. -- `avroEnum`: To treat a sealed trait as an Avro enum. -- `scale(scale: Int = 24)` and `precision(precision: Int = 48)`: To describe the scale and precision of a decimal field. -- `decimal(decimalType: DecimalType)`: Used to annotate a `BigInteger` or `BigDecimal` type to indicate the logical type encoding (avro bytes or avro fixed). -- `bytes(bytesType: BytesType)`: Used to annotate a Byte type to indicate the avro type encoding (avro bytes or avro fixed). -- `formatToString`: Used to annotate fields of type `LocalDate`, `LocalTime`, `LocalDateTime` or `Instant` in order to render them as a string using the given formatter instead of rendering them as avro logical types. -- `timeprecision(timeprecisionType: TimePrecisionType)`: Used to indicate the precision (millisecond precision or microsecond precision) of avro logical types `Time`, `Timestamp` and `Local timestamp` -- `error`: Used to annotate a record in order to render it as a avro error record -- `fieldOrder(fieldOrderType: FieldOrderType)`: Used to indicate the avro field order of a record +- `@AvroAnnotations.name(name: String)`: To change the name of a field or a record. +- `@AvroAnnotations.namespace(namespace: String)`: To add the namespace for a field or a record. +- `@AvroAnnotations.doc(doc: String)`: To add documentation to a field or a record. +- `@AvroAnnotations.aliases(aliases: Set[String])`: To add aliases to a field or a record. +- `@AvroAnnotations.avroEnum`: To treat a sealed trait as an Avro enum. +- `@AvroAnnotations.scale(scale: Int = 24)` and `@AvroAnnotations.precision(precision: Int = 48)`: To describe the scale and precision of a decimal field. +- `@AvroAnnotations.decimal(decimalType: DecimalType)`: Used to annotate a `BigInteger` or `BigDecimal` type to indicate the logical type encoding (avro bytes or avro fixed). +- `@AvroAnnotations.bytes(bytesType: BytesType)`: Used to annotate a Byte type to indicate the avro type encoding (avro bytes or avro fixed). +- `@AvroAnnotations.formatToString`: Used to annotate fields of type `LocalDate`, `LocalTime`, `LocalDateTime` or `Instant` in order to render them as a string using the given formatter instead of rendering them as avro logical types. +- `@AvroAnnotations.timeprecision(timeprecisionType: TimePrecisionType)`: Used to indicate the precision (millisecond precision or microsecond precision) of avro logical types `Time`, `Timestamp` and `Local timestamp` +- `@AvroAnnotations.error`: Used to annotate a record in order to render it as a avro error record +- `@AvroAnnotations.fieldOrder(fieldOrderType: FieldOrderType)`: Used to indicate the avro field order of a record For example, to change the name of a field in the Avro schema, we can use the `AvroAnnotations.name` annotation: @@ -179,3 +183,7 @@ The output: ```scala The person schema in Avro Schema JSON format: {"type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"age","type":{"type":"bytes","logicalType":"decimal","precision":48,"scale":24}}]} ``` + +## Conclusion + +In this article, we explored how to work with Apache Avro codecs in Scala using the ZIO Schema library. We saw how to use `AvroSchemaCodec` to encode and decode Avro JSON schemas to and from ZIO Schemas. Additionally, we created a binary codec using `AvroCodec.schemaBasedBinaryCodec` to encode and decode various data types to and from Avro binary format. We learned about using annotations to extend the default behavior of ZIO Schema for Apache Avro serialization. \ No newline at end of file From 4ee9b382f58f18209115d757355d866fa3a131e4 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 31 Jul 2023 11:26:34 +0330 Subject: [PATCH 50/63] add zio-schema-bson to doc's dependencies. --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 99bf8b3b3..477d31263 100644 --- a/build.sbt +++ b/build.sbt @@ -341,5 +341,5 @@ lazy val docs = project |sbt test |```""".stripMargin ) - .dependsOn(zioSchemaJVM, zioSchemaProtobufJVM, zioSchemaJsonJVM, zioSchemaOpticsJVM, zioSchemaAvroJVM) + .dependsOn(zioSchemaJVM, zioSchemaProtobufJVM, zioSchemaJsonJVM, zioSchemaOpticsJVM, zioSchemaAvroJVM, zioSchemaBsonJVM) .enablePlugins(WebsitePlugin) From 45abefc382f1bb1d17ffe08f4d2f678aba08079e Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 31 Jul 2023 15:45:07 +0330 Subject: [PATCH 51/63] add json codec article. --- docs/codecs/bson.md | 1 + docs/codecs/json.md | 110 ++++++++++++++++++++++++++++++++++++++++++++ docs/sidebars.js | 3 +- 3 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 docs/codecs/json.md diff --git a/docs/codecs/bson.md b/docs/codecs/bson.md index 31102d71b..284ce88d1 100644 --- a/docs/codecs/bson.md +++ b/docs/codecs/bson.md @@ -1,6 +1,7 @@ --- id: bson title: "Bson Codecs" +sidebar_label: "BSON" --- ## Introduction diff --git a/docs/codecs/json.md b/docs/codecs/json.md new file mode 100644 index 000000000..d45d8174b --- /dev/null +++ b/docs/codecs/json.md @@ -0,0 +1,110 @@ +--- +id: json +title: "JSON Codecs" +sidebar_label: "JSON" +--- + +## Introduction + +JSON (JavaScript Object Notation) is a widely used data interchange format for transmitting and storing data. ZIO Schema provides `zio-schema-json` module which has functionality to derive JSON codecs from a ZIO Schema. JSON codecs allow us to easily serialize and deserialize data in JSON format. In this article, we will explore how derive JSON codecs using the ZIO Schema. + +## Installation + +To derive JSON codecs from a ZIO Schema, we need to add the following dependency to our `build.sbt` file: + +```scala +libraryDependencies += "dev.zio" %% "zio-schema-json" % @VERSION@ +``` + +## JsonCodec + +The `JsonCodec` object inside the `zio.schema.codec` package provides the `jsonCodec` operator which allows us to derive JSON codecs from a ZIO Schema: + +```scala +object JsonCodec { + def jsonCodec[A](schema: Schema[A]): zio.json.JsonCodec[A] = ??? +} +``` + +Let's try an example to see how it works: + +```scala mdoc:compile-only +import zio._ +import zio.json._ +import zio.schema.{DeriveSchema, Schema} + +case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = + DeriveSchema.gen + implicit val jsonCodec: zio.json.JsonCodec[Person] = + zio.schema.codec.JsonCodec.jsonCodec(schema) +} + +object Main extends ZIOAppDefault { + def run = for { + _ <- ZIO.debug("JSON Codec Example:") + person: Person = Person("John", 42) + encoded: String = person.toJson + _ <- ZIO.debug(s"person object encoded to JSON string: $encoded") + decoded <- ZIO.fromEither(Person.jsonCodec.decodeJson(encoded)) + _ <- ZIO.debug(s"JSON object decoded to Person class: $decoded") + } yield () +} +``` + +## BinaryCodec + +We can also derive a binary codec from a ZIO Schema using the `schemaBasedBinaryCodec`: + +```scala +object JsonCodec { + implicit def schemaBasedBinaryCodec[A](implicit schema: Schema[A]): BinaryCodec[A] = ??? +} +``` + +Let's try an example: + +```scala mdoc:compile-only +import zio._ +import zio.schema.codec.BinaryCodec +import zio.schema.{DeriveSchema, Schema} + +case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = + DeriveSchema.gen + implicit val jsonBinaryCodec: BinaryCodec[Person] = + zio.schema.codec.JsonCodec.schemaBasedBinaryCodec(schema) +} + +object Main extends ZIOAppDefault { + def run = for { + _ <- ZIO.debug("JSON Codec Example:") + person: Person = Person("John", 42) + encoded: Chunk[Byte] = Person.jsonBinaryCodec.encode(person) + _ <- ZIO.debug(s"person object encoded to Binary JSON: ${toHex(encoded)}") + decoded <- ZIO.fromEither(Person.jsonBinaryCodec.decode(encoded)) + _ <- ZIO.debug(s"JSON object decoded to Person class: $decoded") + } yield () + + def toHex(bytes: Chunk[Byte]): String = + bytes.map("%02x".format(_)).mkString(" ") +} +``` + +The output of the above program is: + +```scala +JSON Codec Example: +person object encoded to JSON string: 7b 22 6e 61 6d 65 22 3a 22 4a 6f 68 6e 22 2c 22 61 67 65 22 3a 34 32 7d +JSON object decoded to Person class: Person(John,42) +``` + +By utilizing JSON codecs derived from ZIO Schema, developers can easily serialize and deserialize data in JSON format without writing boilerplate code. This enhances productivity and simplifies data handling in Scala applications. + + + + diff --git a/docs/sidebars.js b/docs/sidebars.js index 86e263726..ddad391e8 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -33,7 +33,8 @@ const sidebars = { link: { type: "doc", id: "codecs/index" }, items: [ "codecs/apache-avro", - "codecs/bson" + "codecs/bson", + "codecs/json" ], }, "dynamic-data-representation", From 9ac473210d80ecd7e13233e34cceeda31bc0c6bf Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 31 Jul 2023 16:50:54 +0330 Subject: [PATCH 52/63] add message pack section. --- build.sbt | 10 +++++- docs/codecs/messsage-pack.md | 66 ++++++++++++++++++++++++++++++++++++ docs/sidebars.js | 3 +- 3 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 docs/codecs/messsage-pack.md diff --git a/build.sbt b/build.sbt index 477d31263..5d4d423ba 100644 --- a/build.sbt +++ b/build.sbt @@ -341,5 +341,13 @@ lazy val docs = project |sbt test |```""".stripMargin ) - .dependsOn(zioSchemaJVM, zioSchemaProtobufJVM, zioSchemaJsonJVM, zioSchemaOpticsJVM, zioSchemaAvroJVM, zioSchemaBsonJVM) + .dependsOn( + zioSchemaJVM, + zioSchemaProtobufJVM, + zioSchemaJsonJVM, + zioSchemaOpticsJVM, + zioSchemaAvroJVM, + zioSchemaBsonJVM, + zioSchemaMsgPackJVM + ) .enablePlugins(WebsitePlugin) diff --git a/docs/codecs/messsage-pack.md b/docs/codecs/messsage-pack.md new file mode 100644 index 000000000..e2dfae642 --- /dev/null +++ b/docs/codecs/messsage-pack.md @@ -0,0 +1,66 @@ +--- +id: message-pack +title: "MessagePack Codecs" +sidebar_label: "Message Pack" +--- + +## Introduction + +MessagePack is a binary serialization format designed for efficient data exchange between different systems and languages. In this section, we will explore how to derive MessagePack codecs from a ZIO Schema. MessagePack codecs allow us to easily serialize and deserialize data in MessagePack format. + +## Installation + +To use MessagePack codecs, you need to add the following dependency to your build.sbt file: + +```scala +libraryDependencies += "dev.zio" %% "zio-schema-msg-pack" % "@VERSION@" +``` + +## BinaryCodec + +The `MessagePackCodec` object inside the `zio.schema.codec` package provides the `messagePackCodec` operator which allows us to derive MessagePack codecs from a ZIO Schema: + +```scala +object MessagePackCodec { + implicit def messagePackCodec[A](implicit schema: Schema[A]): BinaryCodec[A] = ??? +} +``` + +Let's try an example to see how it works: + +```scala mdoc:compile-only +import zio._ +import zio.schema.codec._ +import zio.schema.{DeriveSchema, Schema} + +case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = + DeriveSchema.gen + implicit val msgPackCodec: BinaryCodec[Person] = + MessagePackCodec.messagePackCodec(schema) +} + +object Main extends ZIOAppDefault { + def run = for { + _ <- ZIO.debug("MessagePack Codec Example:") + person: Person = Person("John", 42) + encoded: Chunk[Byte] = Person.msgPackCodec.encode(person) + _ <- ZIO.debug(s"person object encoded to MessagePack's binary format: ${toHex(encoded)}") + decoded <- ZIO.fromEither(Person.msgPackCodec.decode(encoded)) + _ <- ZIO.debug(s"MessagePack object decoded to Person class: $decoded") + } yield () + + def toHex(bytes: Chunk[Byte]): String = + bytes.map("%02x".format(_)).mkString(" ") +} +``` + +The output of the above program is: + +```scala +MessagePack Codec Example: +person object encoded to MessagePack's binary format: 82 a4 6e 61 6d 65 a4 4a 6f 68 6e a3 61 67 65 2a +MessagePack object decoded to Person class: Person(John,42) +``` \ No newline at end of file diff --git a/docs/sidebars.js b/docs/sidebars.js index ddad391e8..302e3c4e6 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -34,7 +34,8 @@ const sidebars = { items: [ "codecs/apache-avro", "codecs/bson", - "codecs/json" + "codecs/json", + "codecs/message-pack" ], }, "dynamic-data-representation", From 1409e5f305bb1d5b1f79ec98a283a2e2849a79a6 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 31 Jul 2023 18:16:47 +0330 Subject: [PATCH 53/63] add protobuf section. --- docs/codecs/json.md | 4 --- docs/codecs/messsage-pack.md | 2 +- docs/codecs/protobuf.md | 68 ++++++++++++++++++++++++++++++++++++ docs/sidebars.js | 3 +- 4 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 docs/codecs/protobuf.md diff --git a/docs/codecs/json.md b/docs/codecs/json.md index d45d8174b..6dcf356c3 100644 --- a/docs/codecs/json.md +++ b/docs/codecs/json.md @@ -104,7 +104,3 @@ JSON object decoded to Person class: Person(John,42) ``` By utilizing JSON codecs derived from ZIO Schema, developers can easily serialize and deserialize data in JSON format without writing boilerplate code. This enhances productivity and simplifies data handling in Scala applications. - - - - diff --git a/docs/codecs/messsage-pack.md b/docs/codecs/messsage-pack.md index e2dfae642..aa61f1b0e 100644 --- a/docs/codecs/messsage-pack.md +++ b/docs/codecs/messsage-pack.md @@ -63,4 +63,4 @@ The output of the above program is: MessagePack Codec Example: person object encoded to MessagePack's binary format: 82 a4 6e 61 6d 65 a4 4a 6f 68 6e a3 61 67 65 2a MessagePack object decoded to Person class: Person(John,42) -``` \ No newline at end of file +``` diff --git a/docs/codecs/protobuf.md b/docs/codecs/protobuf.md new file mode 100644 index 000000000..f6ed6375b --- /dev/null +++ b/docs/codecs/protobuf.md @@ -0,0 +1,68 @@ +--- +id: protobuf +title: "Protobuf Codecs" +sidebar_label: "Protobuf" +--- + +## Introduction + +Protocol Buffers (protobuf) is a binary serialization format developed by Google. It is designed for efficient data exchange between different systems and languages. In this article, we will explore how to derive Protobuf codecs from a ZIO Schema. Protobuf codecs allow us to easily serialize and deserialize data in Protobuf format, making it simple to interact with APIs and data sources that use Protobuf as their data format. + +## Installation + +To start using Protobuf codecs in ZIO, you need to add the following dependency to your build.sbt file: + +```scala +libraryDependencies += "dev.zio" %% "zio-schema-protobuf" % "@VERSION@" +``` + +## BinaryCodec + +The `ProtobufCodec` object inside the `zio.schema.codec` package provides the `protobufCodec` operator which allows us to derive Protobuf codecs from a ZIO Schema: + +```scala +object ProtobufCodec { + implicit def protobufCodec[A](implicit schema: Schema[A]): BinaryCodec[A] = ??? +} +``` + +Let's try an example: + +```scala mdoc:compile-only +import zio._ +import zio.schema.codec._ +import zio.schema.{DeriveSchema, Schema} + +case class Person(name: String, age: Int) + +object Person { + implicit val schema : Schema[Person] = + DeriveSchema.gen + implicit val protobufCodec: BinaryCodec[Person] = + ProtobufCodec.protobufCodec(schema) +} + +object Main extends ZIOAppDefault { + def run = for { + _ <- ZIO.debug("Protobuf Codec Example:") + person: Person = Person("John", 42) + encoded: Chunk[Byte] = Person.protobufCodec.encode(person) + _ <- ZIO.debug( + s"person object encoded to Protobuf's binary format: ${toHex(encoded)}" + ) + decoded <- ZIO.fromEither(Person.protobufCodec.decode(encoded)) + _ <- ZIO.debug(s"Protobuf object decoded to Person class: $decoded") + } yield () + + def toHex(bytes: Chunk[Byte]): String = + bytes.map("%02x".format(_)).mkString(" ") +} +``` + +Here is the output of running the above program: + +```scala +Protobuf Codec Example: +person object encoded to Protobuf's binary format: 0a 04 4a 6f 68 6e 10 2a +Protobuf object decoded to Person class: Person(John,42) +``` diff --git a/docs/sidebars.js b/docs/sidebars.js index 302e3c4e6..70f6968bf 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -35,7 +35,8 @@ const sidebars = { "codecs/apache-avro", "codecs/bson", "codecs/json", - "codecs/message-pack" + "codecs/message-pack", + "codecs/protobuf" ], }, "dynamic-data-representation", From 4bbcce424d53064ef9d20f752e40f498b2b5e39d Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Mon, 31 Jul 2023 18:18:19 +0330 Subject: [PATCH 54/63] update message pack. --- docs/codecs/messsage-pack.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/codecs/messsage-pack.md b/docs/codecs/messsage-pack.md index aa61f1b0e..9fd5571ec 100644 --- a/docs/codecs/messsage-pack.md +++ b/docs/codecs/messsage-pack.md @@ -1,7 +1,7 @@ --- id: message-pack title: "MessagePack Codecs" -sidebar_label: "Message Pack" +sidebar_label: "MessagePack" --- ## Introduction From d6e598546663df4effd6c1d86f10d35f9091de70 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Tue, 1 Aug 2023 11:33:47 +0330 Subject: [PATCH 55/63] add apache thrift section to codecs. --- build.sbt | 3 +- docs/codecs/thrift.md | 68 +++++++++++++++++++++++++++++++++++++++++++ docs/sidebars.js | 3 +- 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 docs/codecs/thrift.md diff --git a/build.sbt b/build.sbt index 5d4d423ba..c17b2c545 100644 --- a/build.sbt +++ b/build.sbt @@ -348,6 +348,7 @@ lazy val docs = project zioSchemaOpticsJVM, zioSchemaAvroJVM, zioSchemaBsonJVM, - zioSchemaMsgPackJVM + zioSchemaMsgPackJVM, + zioSchemaThriftJVM ) .enablePlugins(WebsitePlugin) diff --git a/docs/codecs/thrift.md b/docs/codecs/thrift.md new file mode 100644 index 000000000..298580945 --- /dev/null +++ b/docs/codecs/thrift.md @@ -0,0 +1,68 @@ +--- +id: thrift +title: "Apache Thrift Codecs" +sidebar_label: "Apache Thrift" +--- + +## Introduction + +Apache Thrift is an open-source framework that allows seamless communication and data sharing between different programming languages and platforms. In this section, we will explore how to derive Apache Thrift codecs from a ZIO Schema. + +## Installation + +To derive Apache Thrift codecs from a ZIO Schema, we need to add the following dependency to our `build.sbt` file: + +```scala +libraryDependencies += "dev.zio" %% "zio-schema-thrift" % "@VERSION@" +``` + +## BinaryCodec + +The `ThriftCodec` object inside the `zio.schema.codec` package provides the `thriftCodec` operator which allows us to derive Protobuf codecs from a ZIO Schema: + +```scala +object ThriftCodec { + implicit def thriftCodec[A](implicit schema: Schema[A]): BinaryCodec[A] = ??? +} +``` + +## Example + +Let's try an example: + +```scala mdoc:compile-only +import zio._ +import zio.schema.codec._ +import zio.schema.{DeriveSchema, Schema} + +case class Person(name: String, age: Int) + +object Person { + implicit val schema : Schema[Person] = + DeriveSchema.gen + implicit val thriftCodec: BinaryCodec[Person] = + ThriftCodec.thriftCodec(schema) +} + +object Main extends ZIOAppDefault { + def run = for { + _ <- ZIO.debug("Apache Thrift Codec Example:") + person: Person = Person("John", 42) + encoded: Chunk[Byte] = Person.thriftCodec.encode(person) + _ <- ZIO.debug(s"person object encoded to Thrift's binary format: ${toHex(encoded)}") + decoded <- ZIO.fromEither(Person.thriftCodec.decode(encoded)) + _ <- ZIO.debug(s"Thrift object decoded to Person class: $decoded") + } yield () + + def toHex(bytes: Chunk[Byte]): String = + bytes.map("%02x".format(_)).mkString(" ") +} +``` + +Here is the output of running the above program: + +```scala +Apache Thrift Codec Example: +person object encoded to Thrift's binary format: 0b 00 01 00 00 00 04 4a 6f 68 6e 08 00 02 00 00 00 2a 00 +Thrift object decoded to Person class: Person(John,42) +``` diff --git a/docs/sidebars.js b/docs/sidebars.js index 70f6968bf..d5db775ae 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -36,7 +36,8 @@ const sidebars = { "codecs/bson", "codecs/json", "codecs/message-pack", - "codecs/protobuf" + "codecs/protobuf", + "codecs/thrift" ], }, "dynamic-data-representation", From 3cfef9c98a7678c33c1909718d0d5c2ace3c75cf Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Tue, 1 Aug 2023 14:39:53 +0330 Subject: [PATCH 56/63] refactor. --- docs/codecs/{apache-avro.md => avro.md} | 6 +-- docs/codecs/bson.md | 39 +----------------- docs/codecs/messsage-pack.md | 2 + docs/codecs/protobuf.md | 54 +++++++++++++++++++++++++ docs/codecs/thrift.md | 3 +- docs/protobuf-example.md | 31 -------------- docs/sidebars.js | 7 ++-- 7 files changed, 64 insertions(+), 78 deletions(-) rename docs/codecs/{apache-avro.md => avro.md} (93%) delete mode 100644 docs/protobuf-example.md diff --git a/docs/codecs/apache-avro.md b/docs/codecs/avro.md similarity index 93% rename from docs/codecs/apache-avro.md rename to docs/codecs/avro.md index 1c8a82381..c32e28cce 100644 --- a/docs/codecs/apache-avro.md +++ b/docs/codecs/avro.md @@ -1,5 +1,5 @@ --- -id: apache-avro +id: avro title: "Apache Avro Codecs" sidebar_label: "Apache Avro" --- @@ -183,7 +183,3 @@ The output: ```scala The person schema in Avro Schema JSON format: {"type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"age","type":{"type":"bytes","logicalType":"decimal","precision":48,"scale":24}}]} ``` - -## Conclusion - -In this article, we explored how to work with Apache Avro codecs in Scala using the ZIO Schema library. We saw how to use `AvroSchemaCodec` to encode and decode Avro JSON schemas to and from ZIO Schemas. Additionally, we created a binary codec using `AvroCodec.schemaBasedBinaryCodec` to encode and decode various data types to and from Avro binary format. We learned about using annotations to extend the default behavior of ZIO Schema for Apache Avro serialization. \ No newline at end of file diff --git a/docs/codecs/bson.md b/docs/codecs/bson.md index 284ce88d1..786a0227a 100644 --- a/docs/codecs/bson.md +++ b/docs/codecs/bson.md @@ -18,50 +18,15 @@ libraryDependencies += "dev.zio" %% "zio-schema-bson" % @VERSION@ ## BsonSchemaCodec -The ZIO Schema library provides the `BsonSchemaCodec` object in the `zio.schema.codec` package. This object offers three methods that allow us to derive BSON encoders, decoders, and codecs from ZIO Schemas: +The `BsonSchemaCodec` object inside the `zio.schema.codec` package provides the `bsonCodec` operator which allows us to derive Protobuf codecs from a ZIO Schema: ```scala object BsonSchemaCodec { - def bsonEncoder[A](schema: Schema[A]): BsonEncoder[A] - def bsonDecoder[A](schema: Schema[A]): BsonDecoder[A] def bsonCodec[A](schema: Schema[A]): BsonCodec[A] } ``` -It has three methods, by calling each of them, we can get a `BsonEncoder[A]`, `BsonDecoder[A]`, or `BsonCodec[A]` from a `Schema[A]`. Let's see a simplified version of each of these traits: - -### 1. BsonEncoder - -The `BsonEncoder` trait defines a type class for encoding a value of type `A` into a BSON value. The `toBsonValue` method accomplishes this conversion: - -```scala -trait BsonEncoder[A] { - def toBsonValue(value: A): BsonValue -} -``` - -### 2. BsonDecoder - -The BsonDecoder trait defines a type class for decoding a BSON value into a value of type A. The fromBsonValue method handles this conversion and returns an Either indicating success or an error: - -```scala -trait BsonDecoder[A] { - def fromBsonValue(value: BsonValue): Either[BsonDecoder.Error, A] -} -``` - -### 3. BsonCodec - -The `BsonCodec` case class combines both the BSON encoder and decoder for a specific type `A`: - -```scala -final case class BsonCodec[A]( - encoder: BsonEncoder[A], - decoder: BsonDecoder[A] -) -``` - -## Example: Deriving a Bson Codec for a Case Class +## Example Let's see an example of how to derive a BSON codec for a case class using ZIO Schema: diff --git a/docs/codecs/messsage-pack.md b/docs/codecs/messsage-pack.md index 9fd5571ec..e6d22a81b 100644 --- a/docs/codecs/messsage-pack.md +++ b/docs/codecs/messsage-pack.md @@ -26,6 +26,8 @@ object MessagePackCodec { } ``` +## Example + Let's try an example to see how it works: ```scala mdoc:compile-only diff --git a/docs/codecs/protobuf.md b/docs/codecs/protobuf.md index f6ed6375b..f734d8eb8 100644 --- a/docs/codecs/protobuf.md +++ b/docs/codecs/protobuf.md @@ -26,6 +26,8 @@ object ProtobufCodec { } ``` +## Example: BinaryCodec + Let's try an example: ```scala mdoc:compile-only @@ -66,3 +68,55 @@ Protobuf Codec Example: person object encoded to Protobuf's binary format: 0a 04 4a 6f 68 6e 10 2a Protobuf object decoded to Person class: Person(John,42) ``` + +## Example: Streaming Codecs + +The following example shows how to use Protobuf codecs to encode and decode streams of data: + +```scala mdoc:compile-only +import zio._ +import zio.schema.codec.{BinaryCodec, ProtobufCodec} +import zio.schema.{DeriveSchema, Schema} +import zio.stream.ZStream + +case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = + DeriveSchema.gen + implicit val protobufCodec: BinaryCodec[Person] = + ProtobufCodec.protobufCodec(schema) +} + +object Main extends ZIOAppDefault { + + def run = for { + _ <- ZIO.debug("Protobuf Stream Codecs Example:") + person = Person("John", 42) + + personToProto = Person.protobufCodec.streamEncoder + protoToPerson = Person.protobufCodec.streamDecoder + + newPerson <- ZStream(person) + .via(personToProto) + .via(protoToPerson) + .runHead + .some + .catchAll(error => ZIO.debug(error)) + _ <- ZIO.debug( + "is old person the new person? " + (person == newPerson).toString + ) + _ <- ZIO.debug("old person: " + person) + _ <- ZIO.debug("new person: " + newPerson) + } yield () +} +``` + +The output of running the above program is: + +```scala +Protobuf Stream Codecs Example: +is old person the new person? true +old person: Person(John,42) +new person: Person(John,42) +``` diff --git a/docs/codecs/thrift.md b/docs/codecs/thrift.md index 298580945..e1acd9f94 100644 --- a/docs/codecs/thrift.md +++ b/docs/codecs/thrift.md @@ -38,8 +38,9 @@ import zio.schema.{DeriveSchema, Schema} case class Person(name: String, age: Int) object Person { - implicit val schema : Schema[Person] = + implicit val schema: Schema[Person] = DeriveSchema.gen + implicit val thriftCodec: BinaryCodec[Person] = ThriftCodec.thriftCodec(schema) } diff --git a/docs/protobuf-example.md b/docs/protobuf-example.md deleted file mode 100644 index 5e7591aa5..000000000 --- a/docs/protobuf-example.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -id: protobuf-example -title: "Protobuf Example" ---- - -```scala -object ProtobufExample extends zio.App { - import zio.schema.codec.ProtobufCodec - import ManualConstruction._ - import zio.stream.ZStream - - override def run(args: List[String]): UIO[ExitCode] = for { - _ <- ZIO.unit - _ <- ZIO.debug("protobuf roundtrip") - person = Person("Michelle", 32) - - personToProto = ProtobufCodec.encoder[Person](schemaPerson) - protoToPerson = ProtobufCodec.decoder[Person](schemaPerson) - - newPerson <- ZStream(person) - .transduce(personToProto) - .transduce(protoToPerson) - .runHead - .some - .catchAll(error => ZIO.debug(error)) - _ <- ZIO.debug("is old person the new person? " + (person == newPerson).toString) - _ <- ZIO.debug("old person: " + person) - _ <- ZIO.debug("new person: " + newPerson) - } yield ExitCode.success -} -``` diff --git a/docs/sidebars.js b/docs/sidebars.js index d5db775ae..180622df5 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -32,17 +32,16 @@ const sidebars = { collapsed: true, link: { type: "doc", id: "codecs/index" }, items: [ - "codecs/apache-avro", + "codecs/avro", + "codecs/thrift", "codecs/bson", "codecs/json", "codecs/message-pack", - "codecs/protobuf", - "codecs/thrift" + "codecs/protobuf" ], }, "dynamic-data-representation", "reified-optics", - "protobuf-example", "combining-different-encoders", { type: "category", From 84df435d970a7935817c63ff92eaeaf328a532a9 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Tue, 1 Aug 2023 18:43:11 +0330 Subject: [PATCH 57/63] refactor. --- .../combining-different-encoders.md | 0 .../mapping-dto-to-domain-object.md | 0 docs/{reified-optics.md => optics.md} | 41 +++++++++++-------- docs/sidebars.js | 8 ++-- 4 files changed, 27 insertions(+), 22 deletions(-) rename docs/{ => examples}/combining-different-encoders.md (100%) rename docs/{ => examples}/mapping-dto-to-domain-object.md (100%) rename docs/{reified-optics.md => optics.md} (81%) diff --git a/docs/combining-different-encoders.md b/docs/examples/combining-different-encoders.md similarity index 100% rename from docs/combining-different-encoders.md rename to docs/examples/combining-different-encoders.md diff --git a/docs/mapping-dto-to-domain-object.md b/docs/examples/mapping-dto-to-domain-object.md similarity index 100% rename from docs/mapping-dto-to-domain-object.md rename to docs/examples/mapping-dto-to-domain-object.md diff --git a/docs/reified-optics.md b/docs/optics.md similarity index 81% rename from docs/reified-optics.md rename to docs/optics.md index a41e84900..5fdd04a6f 100644 --- a/docs/reified-optics.md +++ b/docs/optics.md @@ -1,20 +1,18 @@ --- -id: reified-optics -title: "Reified Optics" +id: optics +title: "Optics" --- -Reified optics is a technique in functional programming that allows you to treat optics as first-class values. This means that we can pass them around, compose them, and store optics in data structures. Reified optics is one of the solutions to the problem of making computations as first-class values. - Optics are a way of accessing and manipulating data in a functional way. They can be used to get, set, and update values in data structures, as well as to traverse and explore data. -## Pure Optics (Manual Derivation) +## Manual Derivation of Optics -Before we dive into reified optics and how we can have an automatic derivation of optics, let's take a look at the pure optics and how should we create them manually. +Before we dive into auto-derivation of optics and how we can derive optics from a ZIO Schema, let's take a look at the pure optics and how we can create them manually using [ZIO Optics](https://zio.dev/zio-optics) library. -First, we should add `zio-schema-optics` to our `build.sbt` file: +First, we should add `zio-optics` to our `build.sbt` file: ```scala -libraryDependencies += "dev.zio" %% "zio-schema-optics" % @VERSION@ +libraryDependencies += "dev.zio" %% "zio-optics" % "" ``` Now let's define a simple data type called `User` and create two optics for its `name` and `age` fields: @@ -62,9 +60,12 @@ object Main extends ZIOAppDefault { } ``` -## Reified Optics (Automatic Derivation) +## Automatic Derivation of Optics + +ZIO Schema has a module called `zio-schema-optics` which provides functionalities to derive various optics from a ZIO Schema. -With reified optics, we can derive optics automatically from a schema. This means that we don't have to write the optics manually, but instead, we can use the `Schema#makeAccessors` method which will derive the optics for us: + +By having a `Schema[A]`, we can derive optics automatically from a schema. This means that we don't have to write the optics manually, but instead, we can use the `Schema#makeAccessors` method which will derive the optics for us: ```scala trait Schema[A] { @@ -99,13 +100,15 @@ trait AccessorBuilder { It has three methods for creating three types of optics: -- Lens is an optic used to get and update values in a product type. -- Prism is an optic used to get and update values in a sum type. -- Traversal is an optic used to get and update values in a collection type. +- **Lens** is an optic used to get and update values in a product type. +- **Prism** is an optic used to get and update values in a sum type. +- **Traversal** is an optic used to get and update values in a collection type. + +Let's take a look at how we can derive optics using ZIO Schema Optics. -Let's take a look at how we can derive optics using ZIO Schema. +### Installation -First we should add `zio-schema-optics` to our `build.sbt` file: +To be able to derive optics from a ZIO Schema, we need to add the following line to our `build.sbt` file: ```scala libraryDependencies += "dev.zio" %% "zio-schema-optics" % @VERSION@ @@ -115,7 +118,9 @@ This package contains a `ZioOpticsBuilder` which is an implementation of the `Ac Now we are ready to try any of the following examples: -### Lens +### Examples + +#### Lens Now we can derive the schema for our `User` data type in its companion object, and then derive optics using `Schema#makeAccessors` method: @@ -167,7 +172,7 @@ Age of user updated: Right(User(John,32)) Name and age of the user updated: Right(User(Jane,32)) ``` -### Prism +#### Prism ```scala mdoc:compile-only import zio._ @@ -215,7 +220,7 @@ Original shape: Circle(1.2) Updated shape: Rectangle(2.0,3.0) ``` -### Traversal +#### Traversal ```scala mdoc:compile-only import zio._ diff --git a/docs/sidebars.js b/docs/sidebars.js index 180622df5..bbb7edc58 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -25,6 +25,7 @@ const sidebars = { "schema-migration", "schema-serialization", "transforming-schemas", + "dynamic-data-representation", "validation", { type: "category", @@ -40,15 +41,14 @@ const sidebars = { "codecs/protobuf" ], }, - "dynamic-data-representation", - "reified-optics", - "combining-different-encoders", + "optics", { type: "category", label: "Examples", collapsed: true, items: [ - "mapping-dto-to-domain-object" + "examples/mapping-dto-to-domain-object", + "examples/combining-different-encoders", ], } ], From efef62970edd23b45b941c1ad357cdeb622f903d Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Wed, 2 Aug 2023 14:25:56 +0330 Subject: [PATCH 58/63] organize sidebar. --- docs/{ => derivations}/codecs/avro.md | 2 +- docs/{ => derivations}/codecs/bson.md | 0 docs/{ => derivations}/codecs/index.md | 0 docs/{ => derivations}/codecs/json.md | 0 .../{ => derivations}/codecs/messsage-pack.md | 0 docs/{ => derivations}/codecs/protobuf.md | 0 docs/{ => derivations}/codecs/thrift.md | 0 .../optics-derivation.md} | 4 +- .../ordering-derivation.md} | 4 +- docs/derivations/zio-test-gen-derivation.md | 54 +++++++++++++++ docs/{ => operations}/diffing-and-patching.md | 0 .../dynamic-data-representation.md | 0 docs/{operations.md => operations/index.md} | 3 +- docs/{ => operations}/schema-migration.md | 0 .../serialization-of-the-schema-itself.md | 0 docs/{ => operations}/the-default-value.md | 0 docs/{ => operations}/transforming-schemas.md | 0 docs/{ => operations}/validating-types.md | 0 docs/sidebars.js | 68 +++++++++++-------- 19 files changed, 101 insertions(+), 34 deletions(-) rename docs/{ => derivations}/codecs/avro.md (98%) rename docs/{ => derivations}/codecs/bson.md (100%) rename docs/{ => derivations}/codecs/index.md (100%) rename docs/{ => derivations}/codecs/json.md (100%) rename docs/{ => derivations}/codecs/messsage-pack.md (100%) rename docs/{ => derivations}/codecs/protobuf.md (100%) rename docs/{ => derivations}/codecs/thrift.md (100%) rename docs/{optics.md => derivations/optics-derivation.md} (99%) rename docs/{derive-ordering.md => derivations/ordering-derivation.md} (92%) create mode 100644 docs/derivations/zio-test-gen-derivation.md rename docs/{ => operations}/diffing-and-patching.md (100%) rename docs/{ => operations}/dynamic-data-representation.md (100%) rename docs/{operations.md => operations/index.md} (94%) rename docs/{ => operations}/schema-migration.md (100%) rename docs/{ => operations}/serialization-of-the-schema-itself.md (100%) rename docs/{ => operations}/the-default-value.md (100%) rename docs/{ => operations}/transforming-schemas.md (100%) rename docs/{ => operations}/validating-types.md (100%) diff --git a/docs/codecs/avro.md b/docs/derivations/codecs/avro.md similarity index 98% rename from docs/codecs/avro.md rename to docs/derivations/codecs/avro.md index c32e28cce..5953379ae 100644 --- a/docs/codecs/avro.md +++ b/docs/derivations/codecs/avro.md @@ -25,7 +25,7 @@ It has two codecs: ### AvroSchemaCodec -The `AvroSchemaCodec` provides methods to encode a `Schema[_]` to Avro JSON schema and decode an Avro JSON schema to a `Schema[_]` ([`Schema.GenericRecord`](../dynamic-data-representation.md)): +The `AvroSchemaCodec` provides methods to encode a `Schema[_]` to Avro JSON schema and decode an Avro JSON schema to a `Schema[_]` ([`Schema.GenericRecord`](../../operations/dynamic-data-representation.md)): ```scala trait AvroSchemaCodec { diff --git a/docs/codecs/bson.md b/docs/derivations/codecs/bson.md similarity index 100% rename from docs/codecs/bson.md rename to docs/derivations/codecs/bson.md diff --git a/docs/codecs/index.md b/docs/derivations/codecs/index.md similarity index 100% rename from docs/codecs/index.md rename to docs/derivations/codecs/index.md diff --git a/docs/codecs/json.md b/docs/derivations/codecs/json.md similarity index 100% rename from docs/codecs/json.md rename to docs/derivations/codecs/json.md diff --git a/docs/codecs/messsage-pack.md b/docs/derivations/codecs/messsage-pack.md similarity index 100% rename from docs/codecs/messsage-pack.md rename to docs/derivations/codecs/messsage-pack.md diff --git a/docs/codecs/protobuf.md b/docs/derivations/codecs/protobuf.md similarity index 100% rename from docs/codecs/protobuf.md rename to docs/derivations/codecs/protobuf.md diff --git a/docs/codecs/thrift.md b/docs/derivations/codecs/thrift.md similarity index 100% rename from docs/codecs/thrift.md rename to docs/derivations/codecs/thrift.md diff --git a/docs/optics.md b/docs/derivations/optics-derivation.md similarity index 99% rename from docs/optics.md rename to docs/derivations/optics-derivation.md index 5fdd04a6f..1ac56bc9b 100644 --- a/docs/optics.md +++ b/docs/derivations/optics-derivation.md @@ -1,6 +1,6 @@ --- -id: optics -title: "Optics" +id: optics-derivation +title: "Optics Derivation" --- Optics are a way of accessing and manipulating data in a functional way. They can be used to get, set, and update values in data structures, as well as to traverse and explore data. diff --git a/docs/derive-ordering.md b/docs/derivations/ordering-derivation.md similarity index 92% rename from docs/derive-ordering.md rename to docs/derivations/ordering-derivation.md index 734395551..dc8dff3a5 100644 --- a/docs/derive-ordering.md +++ b/docs/derivations/ordering-derivation.md @@ -1,6 +1,6 @@ --- -id: deriving-ordering -title: "Deriving Ordering" +id: ordering-derivation +title: "Ordering Derivation" --- Standard Scala library provides a type class called `Ordering[A]` that allows us to compare values of type `A`. ZIO Schema provides a method called `ordering` that generates an `Ordering[A]` instance for the underlying type described by the schema: diff --git a/docs/derivations/zio-test-gen-derivation.md b/docs/derivations/zio-test-gen-derivation.md new file mode 100644 index 000000000..66de50f3b --- /dev/null +++ b/docs/derivations/zio-test-gen-derivation.md @@ -0,0 +1,54 @@ +--- +id: zio-test-gen-derivation +title: "Derivation of ZIO Test Generators" +sidebar_label: "ZIO Test Gen Derivation" +--- + +## Introduction + +ZIO Test supports property-based testing via the `Gen` type. `Gen[R, A]` is a random generator of values of type `A`. Such a generator can be used to produce test cases for a property, which can then be checked for validity. The `zio-schema-zio-test` module provides a way to derive a `Gen[R, A]` from a `Schema[A]`. In this section, we will see how this functionality works. + +## Installation + +In order to derive a generator from a ZIO Schema, we need to add the following dependency to our `build.sbt` file: + +```scala +libraryDependencies += "dev.zio" %% "zio-schema-zio-test" % @VERSION@ +``` + +## DriveGen + +The `DriveGen` inside `zio.schema` package provides the `gen` operator which takes a `Schmea[A]` implicitly and returns a `Gen[Sized, A]`: + +```scala +object DeriveGen { + def gen[A](implicit schema: Schema[A]): Gen[Sized, A] = ??? +} +``` + +## Example + +In the following example, we will derive a generator for the `Person` class using the `DeriveGen.gen` operator: + +```scala +import zio.schema.{DeriveGen, DeriveSchema, Schema} +import zio.test.{Gen, Sized} + +case class Person(name: String, age: Int) + +object Person { + implicit val schema: Schema[Person] = DeriveSchema.gen + val gen: Gen[Sized, Person] = DeriveGen.gen +} + +import zio.test._ + +object ExampleSpec extends ZIOSpecDefault { + def spec = + test("example test") { + check(Person.gen) { p => + assertTrue(???) + } + } +} +``` diff --git a/docs/diffing-and-patching.md b/docs/operations/diffing-and-patching.md similarity index 100% rename from docs/diffing-and-patching.md rename to docs/operations/diffing-and-patching.md diff --git a/docs/dynamic-data-representation.md b/docs/operations/dynamic-data-representation.md similarity index 100% rename from docs/dynamic-data-representation.md rename to docs/operations/dynamic-data-representation.md diff --git a/docs/operations.md b/docs/operations/index.md similarity index 94% rename from docs/operations.md rename to docs/operations/index.md index ece308c1e..44571914d 100644 --- a/docs/operations.md +++ b/docs/operations/index.md @@ -1,7 +1,8 @@ --- -id: operations +id: index title: "ZIO Schema Operations" sidebar_label: "Operations" --- Once we have defined our schemas, we can use them to perform a variety of operations. In this section, we will explore some of the most common operations that we can perform on schemas. + \ No newline at end of file diff --git a/docs/schema-migration.md b/docs/operations/schema-migration.md similarity index 100% rename from docs/schema-migration.md rename to docs/operations/schema-migration.md diff --git a/docs/serialization-of-the-schema-itself.md b/docs/operations/serialization-of-the-schema-itself.md similarity index 100% rename from docs/serialization-of-the-schema-itself.md rename to docs/operations/serialization-of-the-schema-itself.md diff --git a/docs/the-default-value.md b/docs/operations/the-default-value.md similarity index 100% rename from docs/the-default-value.md rename to docs/operations/the-default-value.md diff --git a/docs/transforming-schemas.md b/docs/operations/transforming-schemas.md similarity index 100% rename from docs/transforming-schemas.md rename to docs/operations/transforming-schemas.md diff --git a/docs/validating-types.md b/docs/operations/validating-types.md similarity index 100% rename from docs/validating-types.md rename to docs/operations/validating-types.md diff --git a/docs/sidebars.js b/docs/sidebars.js index bbb7edc58..6c0129e4c 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -13,44 +13,56 @@ const sidebars = { type: "category", label: "Writing Schema", collapsed: true, + items: ["manual-schema-construction", "automatic-schema-derivation"], + }, + { + type: "category", + label: "Operations", + link: { type: "doc", id: "operations/index" }, + collapsed: true, items: [ - "manual-schema-construction", - "automatic-schema-derivation" + "operations/the-default-value", + "operations/transforming-schemas", + "operations/validation", + "operations/diffing-and-patching", + "operations/schema-migration", + "operations/schema-serialization", + "operations/dynamic-data-representation", ], }, - "operations", - "the-default-value", - "diffing-and-patching", - "deriving-ordering", - "schema-migration", - "schema-serialization", - "transforming-schemas", - "dynamic-data-representation", - "validation", { type: "category", - label: "Codecs", + label: "Derivations", collapsed: true, - link: { type: "doc", id: "codecs/index" }, items: [ - "codecs/avro", - "codecs/thrift", - "codecs/bson", - "codecs/json", - "codecs/message-pack", - "codecs/protobuf" + "derivations/ordering-derivation", + "derivations/optics-derivation", + "derivations/zio-test-gen-derivation", + { + type: "category", + label: "Codecs", + collapsed: true, + link: { type: "doc", id: "derivations/codecs/index" }, + items: [ + "derivations/codecs/avro", + "derivations/codecs/thrift", + "derivations/codecs/bson", + "derivations/codecs/json", + "derivations/codecs/message-pack", + "derivations/codecs/protobuf", + ], + }, ], }, - "optics", { - type: "category", - label: "Examples", - collapsed: true, - items: [ - "examples/mapping-dto-to-domain-object", - "examples/combining-different-encoders", - ], - } + type: "category", + label: "Examples", + collapsed: true, + items: [ + "examples/mapping-dto-to-domain-object", + "examples/combining-different-encoders", + ], + }, ], }, ], From 47af3260f1772dba68a632ebb19dc232fc60b511 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Wed, 2 Aug 2023 14:35:36 +0330 Subject: [PATCH 59/63] overview of all operations. --- docs/operations/index.md | 56 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/docs/operations/index.md b/docs/operations/index.md index 44571914d..0c99bdddf 100644 --- a/docs/operations/index.md +++ b/docs/operations/index.md @@ -5,4 +5,60 @@ sidebar_label: "Operations" --- Once we have defined our schemas, we can use them to perform a variety of operations. In this section, we will explore some of the most common operations that we can perform on schemas. + +Before diving into the details, let's see a quick overview of the operations that we can perform on schemas: + +```scala +sealed trait Schema[A] { + self => + + type Accessors[Lens[_, _, _], Prism[_, _, _], Traversal[_, _]] + + def ? : Schema[Option[A]] + + def <*>[B](that: Schema[B]): Schema[(A, B)] + + def <+>[B](that: Schema[B]): Schema[scala.util.Either[A, B]] + + def defaultValue: scala.util.Either[String, A] + + def annotations: Chunk[Any] + + def ast: MetaSchema + + def annotate(annotation: Any): Schema[A] + + def coerce[B](newSchema: Schema[B]): Either[String, Schema[B]] + + def diff(thisValue: A, thatValue: A): Patch[A] + + def patch(oldValue: A, diff: Patch[A]): scala.util.Either[String, A] + + def fromDynamic(value: DynamicValue): scala.util.Either[String, A] + + def makeAccessors(b: AccessorBuilder): Accessors[b.Lens, b.Prism, b.Traversal] + + def migrate[B](newSchema: Schema[B]): Either[String, A => scala.util.Either[String, B]] + + def optional: Schema[Option[A]] + + def ordering: Ordering[A] + + def orElseEither[B](that: Schema[B]): Schema[scala.util.Either[A, B]] + + def repeated: Schema[Chunk[A]] + + def serializable: Schema[Schema[A]] + + def toDynamic(value: A): DynamicValue + + def transform[B](f: A => B, g: B => A): Schema[B] + + def transformOrFail[B](f: A => scala.util.Either[String, B], g: B => scala.util.Either[String, A]): Schema[B] + + def validate(value: A)(implicit schema: Schema[A]): Chunk[ValidationError] + + def zip[B](that: Schema[B]): Schema[(A, B)] +} +``` \ No newline at end of file From 7bb60b17a32d5eb729d7d199c717fc0b071ce1ed Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Wed, 2 Aug 2023 15:41:08 +0330 Subject: [PATCH 60/63] update introduction section. --- docs/index.md | 80 +++++++++++++++++++++++++++++---------------------- 1 file changed, 46 insertions(+), 34 deletions(-) diff --git a/docs/index.md b/docs/index.md index a80a98615..9ef5f803c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -10,20 +10,29 @@ sidebar_label: "Introduction" ## Introduction -Schema is a structure of a data type. ZIO Schema reifies the concept of structure for data types. It makes a high-level description of any data type and makes them as first-class values. +ZIO Schema helps us to solve some of the most common problems in distributed computing, such as serialization, deserialization, and data migration. + +It turns a compiled-time construct (the type of a data structure) into a runtime construct (a value that can be read, manipulated, and composed at runtime). A schema is a structure of a data type. ZIO Schema reifies the concept of structure for data types. It makes a high-level description of any data type and makes them first-class values. Creating a schema for a data type helps us to write codecs for that data type. So this library can be a host of functionalities useful for writing codecs and protocols like JSON, Protobuf, CSV, and so forth. -With schema descriptions that can be automatically derived for case classes and sealed traits, _ZIO Schema_ will be going to provide powerful features for free (Note that the project is in the development stage and all these features are not supported yet): +## What Problems Does ZIO Schema Solve? + +With schema descriptions that can be automatically derived for case classes and sealed traits, _ZIO Schema_ will be going to provide powerful features for free: -- Codecs for any supported protocol (JSON, protobuf, etc.), so data structures can be serialized and deserialized in a principled way -- Diffing, patching, merging, and other generic-data-based operations -- Migration of data structures from one schema to another compatible schema -- Derivation of arbitrary type classes (`Eq`, `Show`, `Ord`, etc.) from the structure of the data +1. Metaprogramming without macros, reflection, or complicated implicit derivations. + 1. Creating serialization and deserialization codecs for any supported protocol (JSON, Protobuf, etc.) + 2. Deriving standard type classes (`Eq`, `Show`, `Ordering`, etc.) from the structure of the data + 3. Default values for data types +2. Automate ETL (Extract, Transform, Load) pipelines + 1. Diffing: diffing between two values of the same type + 2. Patching: applying a diff to a value to update it + 3. Migration: migrating values from one type to another +3. Computations as data: Not only we can turn types into values, but we can also turn computations into values. This opens up a whole new world of possibilities concerning distributed computing. When our data structures need to be serialized, deserialized, persisted, or transported across the wire, then _ZIO Schema_ lets us focus on data modeling and automatically tackle all the low-level, messy details for us. -_ZIO Schema_ is used by a growing number of ZIO libraries, including _ZIO Flow_, _ZIO Redis_, _ZIO Web_, _ZIO SQL_ and _ZIO DynamoDB_. +_ZIO Schema_ is used by a growing number of ZIO libraries, including [ZIO Flow](https://zio.dev/zio-flow), [ZIO Redis](https://zio-redis), [ZIO SQL](https://zio.dev/zio-sql) and [ZIO DynamoDB](https://zio.dev/zio-dynamodb). ## Installation @@ -37,51 +46,54 @@ libraryDependencies += "dev.zio" %% "zio-schema-json" % "@VERSION@" libraryDependencies += "dev.zio" %% "zio-schema-msg-pack" % "@VERSION@" libraryDependencies += "dev.zio" %% "zio-schema-protobuf" % "@VERSION@" libraryDependencies += "dev.zio" %% "zio-schema-thrift" % "@VERSION@" +libraryDependencies += "dev.zio" %% "zio-schema-zio-test" % "@VERSION@" -// Required for automatic generic derivation of schemas +// Required for the automatic generic derivation of schemas libraryDependencies += "dev.zio" %% "zio-schema-derivation" % "@VERSION@" libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value % "provided" ``` ## Example -In this simple example first, we create a schema for `Person` and then run the _diff_ operation on two instances of the `Person` data type, and finally we encode a Person instance using _Protobuf_ protocol: +In this simple example first, we create a schema for `Person` and then run the _diff_ operation on two instances of the `Person` data type, and finally, we encode a Person instance using _Protobuf_ protocol: -```scala -import zio.console.putStrLn -import zio.schema.codec.ProtobufCodec._ +```scala mdoc:compile-only +import zio._ +import zio.stream._ +import zio.schema.codec.{BinaryCodec, ProtobufCodec} import zio.schema.{DeriveSchema, Schema} -import zio.stream.ZStream -import zio.{Chunk, ExitCode, URIO} -final case class Person(name: String, age: Int, id: String) +final case class Person(name: String, age: Int) + object Person { - implicit val schema: Schema[Person] = DeriveSchema.gen[Person] + implicit val schema: Schema[Person] = DeriveSchema.gen + val protobufCodec: BinaryCodec[Person] = ProtobufCodec.protobufCodec } -Person.schema - -import zio.schema.syntax._ - -Person("Alex", 31, "0123").diff(Person("Alex", 31, "124")) +object Main extends ZIOAppDefault { + def run = + ZStream + .succeed(Person("John", 43)) + .via(Person.protobufCodec.streamEncoder) + .runCollect + .flatMap(x => + Console.printLine(s"Encoded data with protobuf codec: ${toHex(x)}") + ) + + def toHex(chunk: Chunk[Byte]): String = + chunk.map("%02X".format(_)).mkString +} +``` -def toHex(chunk: Chunk[Byte]): String = - chunk.toArray.map("%02X".format(_)).mkString +Here is the output of running the above program: -zio.Runtime.default.unsafe.run( - ZStream - .succeed(Person("Thomas", 23, "2354")) - .transduce( - encoder(Person.schema) - ) - .runCollect - .flatMap(x => putStrLn(s"Encoded data with protobuf codec: ${toHex(x)}")) -).getOrThrowFiberFailure() +```scala +Encoded data with protobuf codec: 0A044A6F686E102B ``` ## Resources -- [Zymposium - ZIO Schema](https://www.youtube.com/watch?v=GfNiDaL5aIM) by John A. De Goes, Adam Fraser and Kit Langton (May 2021) +- [Zymposium - ZIO Schema](https://www.youtube.com/watch?v=GfNiDaL5aIM) by John A. De Goes, Adam Fraser, and Kit Langton (May 2021) - [ZIO SCHEMA: A Toolkit For Functional Distributed Computing](https://www.youtube.com/watch?v=lJziseYKvHo&t=481s) by Dan Harris (Functional Scala 2021) - [Creating Declarative Query Plans With ZIO Schema](https://www.youtube.com/watch?v=ClePN4P9_pg) by Dan Harris (ZIO World 2022) -- [Describing Data...with free applicative functors (and more)](https://www.youtube.com/watch?v=oRLkb6mqvVM) by Kris Nuttycombe (Scala World) on the idea behind the [xenomorph](https://github.com/nuttycom/xenomorph) library \ No newline at end of file +- [Describing Data...with free applicative functors (and more)](https://www.youtube.com/watch?v=oRLkb6mqvVM) by Kris Nuttycombe (Scala World) on the idea behind the [xenomorph](https://github.com/nuttycom/xenomorph) library From 93251014ac664a17c6eab925e841f6135b10afbb Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Wed, 2 Aug 2023 15:53:11 +0330 Subject: [PATCH 61/63] update readme. --- README.md | 99 +++++++++++++++++++++++++------------------------------ 1 file changed, 44 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index 6439bc41b..54933bc94 100644 --- a/README.md +++ b/README.md @@ -12,43 +12,27 @@ ZIO Schema helps us to solve some of the most common problems in distributed computing, such as serialization, deserialization, and data migration. -```scala -trait Schema[A] { - def schema: Schema[A] -} -``` +It turns a compiled-time construct (the type of a data structure) into a runtime construct (a value that can be read, manipulated, and composed at runtime). A schema is a structure of a data type. ZIO Schema reifies the concept of structure for data types. It makes a high-level description of any data type and makes them first-class values. -The trait `Schema[A]` is the core data type of this library. The `Schema[A]` is a description of the structure of a data type `A`, it is a data type that describes other data types. It is a first-class value that can be passed around, composed, and manipulated. - -It turns a compiled-time construct (the type of a data structure) into a runtime construct (a value that can be read, manipulated, and composed at runtime). +Creating a schema for a data type helps us to write codecs for that data type. So this library can be a host of functionalities useful for writing codecs and protocols like JSON, Protobuf, CSV, and so forth. ## What Problems Does ZIO Schema Solve? -1. Metaprogramming without macros, reflection or complicated implicit derivations. - 1. Creating serialization and deserialization codecs for any supported protocol (JSON, protobuf, etc.) - 2. Deriving standard type classes (`Eq`, `Show`, `Ordering`, etc.) from the structure of the data - 4. Default values for data types -2. Automate ETL (Extract, Transform, Load) pipelines - 1. Diffing: diffing between two values of the same type - 2. Patching: applying a diff to a value to update it - 3. Migration: migrating values from one type to another -3. Computations as data: Not only we can turn types into values, but we can also turn computations into values. This opens up a whole new world of possibilities in respect to distributed computing. - 1. Optics - -Schema is a structure of a data type. ZIO Schema reifies the concept of structure for data types. It makes a high-level description of any data type and makes them as first-class values. - -Creating a schema for a data type helps us to write codecs for that data type. So this library can be a host of functionalities useful for writing codecs and protocols like JSON, Protobuf, CSV, and so forth. - -With schema descriptions that can be automatically derived for case classes and sealed traits, _ZIO Schema_ will be going to provide powerful features for free (Note that the project is in the development stage and all these features are not supported yet): +With schema descriptions that can be automatically derived for case classes and sealed traits, _ZIO Schema_ will be going to provide powerful features for free: -- Codecs for any supported protocol (JSON, protobuf, etc.), so data structures can be serialized and deserialized in a principled way -- Diffing, patching, merging, and other generic-data-based operations -- Migration of data structures from one schema to another compatible schema -- Derivation of arbitrary type classes (`Eq`, `Show`, `Ord`, etc.) from the structure of the data +1. Metaprogramming without macros, reflection, or complicated implicit derivations. + 1. Creating serialization and deserialization codecs for any supported protocol (JSON, Protobuf, etc.) + 2. Deriving standard type classes (`Eq`, `Show`, `Ordering`, etc.) from the structure of the data + 3. Default values for data types +2. Automate ETL (Extract, Transform, Load) pipelines + 1. Diffing: diffing between two values of the same type + 2. Patching: applying a diff to a value to update it + 3. Migration: migrating values from one type to another +3. Computations as data: Not only we can turn types into values, but we can also turn computations into values. This opens up a whole new world of possibilities concerning distributed computing. When our data structures need to be serialized, deserialized, persisted, or transported across the wire, then _ZIO Schema_ lets us focus on data modeling and automatically tackle all the low-level, messy details for us. -_ZIO Schema_ is used by a growing number of ZIO libraries, including _ZIO Flow_, _ZIO Redis_, _ZIO Web_, _ZIO SQL_ and _ZIO DynamoDB_. +_ZIO Schema_ is used by a growing number of ZIO libraries, including [ZIO Flow](https://zio.dev/zio-flow), [ZIO Redis](https://zio-redis), [ZIO SQL](https://zio.dev/zio-sql) and [ZIO DynamoDB](https://zio.dev/zio-dynamodb). ## Installation @@ -62,52 +46,57 @@ libraryDependencies += "dev.zio" %% "zio-schema-json" % "0.4.12" libraryDependencies += "dev.zio" %% "zio-schema-msg-pack" % "0.4.12" libraryDependencies += "dev.zio" %% "zio-schema-protobuf" % "0.4.12" libraryDependencies += "dev.zio" %% "zio-schema-thrift" % "0.4.12" +libraryDependencies += "dev.zio" %% "zio-schema-zio-test" % "0.4.12" -// Required for automatic generic derivation of schemas +// Required for the automatic generic derivation of schemas libraryDependencies += "dev.zio" %% "zio-schema-derivation" % "0.4.12" libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value % "provided" ``` ## Example -In this simple example first, we create a schema for `Person` and then run the _diff_ operation on two instances of the `Person` data type, and finally we encode a Person instance using _Protobuf_ protocol: +In this simple example first, we create a schema for `Person` and then run the _diff_ operation on two instances of the `Person` data type, and finally, we encode a Person instance using _Protobuf_ protocol: ```scala -import zio.console.putStrLn -import zio.schema.codec.ProtobufCodec._ +import zio._ +import zio.stream._ +import zio.schema.codec.{BinaryCodec, ProtobufCodec} import zio.schema.{DeriveSchema, Schema} -import zio.stream.ZStream -import zio.{Chunk, ExitCode, URIO} -final case class Person(name: String, age: Int, id: String) +final case class Person(name: String, age: Int) + object Person { - implicit val schema: Schema[Person] = DeriveSchema.gen[Person] + implicit val schema: Schema[Person] = DeriveSchema.gen + val protobufCodec: BinaryCodec[Person] = ProtobufCodec.protobufCodec } -Person.schema - -import zio.schema.syntax._ - -Person("Alex", 31, "0123").diff(Person("Alex", 31, "124")) +object Main extends ZIOAppDefault { + def run = + ZStream + .succeed(Person("John", 43)) + .via(Person.protobufCodec.streamEncoder) + .runCollect + .flatMap(x => + Console.printLine(s"Encoded data with protobuf codec: ${toHex(x)}") + ) + + def toHex(chunk: Chunk[Byte]): String = + chunk.map("%02X".format(_)).mkString +} +``` -def toHex(chunk: Chunk[Byte]): String = - chunk.toArray.map("%02X".format(_)).mkString +Here is the output of running the above program: -zio.Runtime.default.unsafe.run( - ZStream - .succeed(Person("Thomas", 23, "2354")) - .transduce( - encoder(Person.schema) - ) - .runCollect - .flatMap(x => putStrLn(s"Encoded data with protobuf codec: ${toHex(x)}")) -).getOrThrowFiberFailure() +```scala +Encoded data with protobuf codec: 0A044A6F686E102B ``` - ## Resources -- [Zymposium - ZIO Schema](https://www.youtube.com/watch?v=GfNiDaL5aIM) by John A. De Goes, Adam Fraser and Kit Langton (May 2021) +- [Zymposium - ZIO Schema](https://www.youtube.com/watch?v=GfNiDaL5aIM) by John A. De Goes, Adam Fraser, and Kit Langton (May 2021) +- [ZIO SCHEMA: A Toolkit For Functional Distributed Computing](https://www.youtube.com/watch?v=lJziseYKvHo&t=481s) by Dan Harris (Functional Scala 2021) +- [Creating Declarative Query Plans With ZIO Schema](https://www.youtube.com/watch?v=ClePN4P9_pg) by Dan Harris (ZIO World 2022) +- [Describing Data...with free applicative functors (and more)](https://www.youtube.com/watch?v=oRLkb6mqvVM) by Kris Nuttycombe (Scala World) on the idea behind the [xenomorph](https://github.com/nuttycom/xenomorph) library ## Documentation From d464145c7ac9ca8ad2f721f2f866f900c4fd868f Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Wed, 2 Aug 2023 16:35:21 +0330 Subject: [PATCH 62/63] update readme. --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 54933bc94..1c7a51bbb 100644 --- a/README.md +++ b/README.md @@ -39,17 +39,17 @@ _ZIO Schema_ is used by a growing number of ZIO libraries, including [ZIO Flow]( In order to use this library, we need to add the following lines in our `build.sbt` file: ```scala -libraryDependencies += "dev.zio" %% "zio-schema" % "0.4.12" -libraryDependencies += "dev.zio" %% "zio-schema-avro" % "0.4.12" -libraryDependencies += "dev.zio" %% "zio-schema-bson" % "0.4.12" -libraryDependencies += "dev.zio" %% "zio-schema-json" % "0.4.12" -libraryDependencies += "dev.zio" %% "zio-schema-msg-pack" % "0.4.12" -libraryDependencies += "dev.zio" %% "zio-schema-protobuf" % "0.4.12" -libraryDependencies += "dev.zio" %% "zio-schema-thrift" % "0.4.12" -libraryDependencies += "dev.zio" %% "zio-schema-zio-test" % "0.4.12" +libraryDependencies += "dev.zio" %% "zio-schema" % "0.4.13" +libraryDependencies += "dev.zio" %% "zio-schema-avro" % "0.4.13" +libraryDependencies += "dev.zio" %% "zio-schema-bson" % "0.4.13" +libraryDependencies += "dev.zio" %% "zio-schema-json" % "0.4.13" +libraryDependencies += "dev.zio" %% "zio-schema-msg-pack" % "0.4.13" +libraryDependencies += "dev.zio" %% "zio-schema-protobuf" % "0.4.13" +libraryDependencies += "dev.zio" %% "zio-schema-thrift" % "0.4.13" +libraryDependencies += "dev.zio" %% "zio-schema-zio-test" % "0.4.13" // Required for the automatic generic derivation of schemas -libraryDependencies += "dev.zio" %% "zio-schema-derivation" % "0.4.12" +libraryDependencies += "dev.zio" %% "zio-schema-derivation" % "0.4.13" libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value % "provided" ``` From 97064360f9039b7bca380e28c1cafbba9fcb2a9b Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Sat, 9 Sep 2023 16:31:57 +0330 Subject: [PATCH 63/63] update readme. --- README.md | 95 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 56 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 2e33a70fe..f259c2092 100644 --- a/README.md +++ b/README.md @@ -10,76 +10,93 @@ ## Introduction -Schema is a structure of a data type. ZIO Schema reifies the concept of structure for data types. It makes a high-level description of any data type and makes them as first-class values. +ZIO Schema helps us to solve some of the most common problems in distributed computing, such as serialization, deserialization, and data migration. + +It turns a compiled-time construct (the type of a data structure) into a runtime construct (a value that can be read, manipulated, and composed at runtime). A schema is a structure of a data type. ZIO Schema reifies the concept of structure for data types. It makes a high-level description of any data type and makes them first-class values. Creating a schema for a data type helps us to write codecs for that data type. So this library can be a host of functionalities useful for writing codecs and protocols like JSON, Protobuf, CSV, and so forth. -With schema descriptions that can be automatically derived for case classes and sealed traits, _ZIO Schema_ will be going to provide powerful features for free (Note that the project is in the development stage and all these features are not supported yet): +## What Problems Does ZIO Schema Solve? + +With schema descriptions that can be automatically derived for case classes and sealed traits, _ZIO Schema_ will be going to provide powerful features for free: -- Codecs for any supported protocol (JSON, protobuf, etc.), so data structures can be serialized and deserialized in a principled way -- Diffing, patching, merging, and other generic-data-based operations -- Migration of data structures from one schema to another compatible schema -- Derivation of arbitrary type classes (`Eq`, `Show`, `Ord`, etc.) from the structure of the data +1. Metaprogramming without macros, reflection, or complicated implicit derivations. + 1. Creating serialization and deserialization codecs for any supported protocol (JSON, Protobuf, etc.) + 2. Deriving standard type classes (`Eq`, `Show`, `Ordering`, etc.) from the structure of the data + 3. Default values for data types +2. Automate ETL (Extract, Transform, Load) pipelines + 1. Diffing: diffing between two values of the same type + 2. Patching: applying a diff to a value to update it + 3. Migration: migrating values from one type to another +3. Computations as data: Not only we can turn types into values, but we can also turn computations into values. This opens up a whole new world of possibilities concerning distributed computing. When our data structures need to be serialized, deserialized, persisted, or transported across the wire, then _ZIO Schema_ lets us focus on data modeling and automatically tackle all the low-level, messy details for us. -_ZIO Schema_ is used by a growing number of ZIO libraries, including _ZIO Flow_, _ZIO Redis_, _ZIO Web_, _ZIO SQL_ and _ZIO DynamoDB_. +_ZIO Schema_ is used by a growing number of ZIO libraries, including [ZIO Flow](https://zio.dev/zio-flow), [ZIO Redis](https://zio-redis), [ZIO SQL](https://zio.dev/zio-sql) and [ZIO DynamoDB](https://zio.dev/zio-dynamodb). ## Installation In order to use this library, we need to add the following lines in our `build.sbt` file: ```scala -libraryDependencies += "dev.zio" %% "zio-schema" % "0.4.13" -libraryDependencies += "dev.zio" %% "zio-schema-bson" % "0.4.13" -libraryDependencies += "dev.zio" %% "zio-schema-json" % "0.4.13" -libraryDependencies += "dev.zio" %% "zio-schema-protobuf" % "0.4.13" - -// Required for automatic generic derivation of schemas -libraryDependencies += "dev.zio" %% "zio-schema-derivation" % "0.4.13", +libraryDependencies += "dev.zio" %% "zio-schema" % "0.4.14" +libraryDependencies += "dev.zio" %% "zio-schema-avro" % "0.4.14" +libraryDependencies += "dev.zio" %% "zio-schema-bson" % "0.4.14" +libraryDependencies += "dev.zio" %% "zio-schema-json" % "0.4.14" +libraryDependencies += "dev.zio" %% "zio-schema-msg-pack" % "0.4.14" +libraryDependencies += "dev.zio" %% "zio-schema-protobuf" % "0.4.14" +libraryDependencies += "dev.zio" %% "zio-schema-thrift" % "0.4.14" +libraryDependencies += "dev.zio" %% "zio-schema-zio-test" % "0.4.14" + +// Required for the automatic generic derivation of schemas +libraryDependencies += "dev.zio" %% "zio-schema-derivation" % "0.4.14" libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value % "provided" ``` ## Example -In this simple example first, we create a schema for `Person` and then run the _diff_ operation on two instances of the `Person` data type, and finally we encode a Person instance using _Protobuf_ protocol: +In this simple example first, we create a schema for `Person` and then run the _diff_ operation on two instances of the `Person` data type, and finally, we encode a Person instance using _Protobuf_ protocol: ```scala -import zio.console.putStrLn -import zio.schema.codec.ProtobufCodec._ +import zio._ +import zio.stream._ +import zio.schema.codec.{BinaryCodec, ProtobufCodec} import zio.schema.{DeriveSchema, Schema} -import zio.stream.ZStream -import zio.{Chunk, ExitCode, URIO} -final case class Person(name: String, age: Int, id: String) +final case class Person(name: String, age: Int) + object Person { - implicit val schema: Schema[Person] = DeriveSchema.gen[Person] + implicit val schema: Schema[Person] = DeriveSchema.gen + val protobufCodec: BinaryCodec[Person] = ProtobufCodec.protobufCodec } -Person.schema - -import zio.schema.syntax._ - -Person("Alex", 31, "0123").diff(Person("Alex", 31, "124")) +object Main extends ZIOAppDefault { + def run = + ZStream + .succeed(Person("John", 43)) + .via(Person.protobufCodec.streamEncoder) + .runCollect + .flatMap(x => + Console.printLine(s"Encoded data with protobuf codec: ${toHex(x)}") + ) + + def toHex(chunk: Chunk[Byte]): String = + chunk.map("%02X".format(_)).mkString +} +``` -def toHex(chunk: Chunk[Byte]): String = - chunk.toArray.map("%02X".format(_)).mkString +Here is the output of running the above program: -zio.Runtime.default.unsafe.run( - ZStream - .succeed(Person("Thomas", 23, "2354")) - .transduce( - encoder(Person.schema) - ) - .runCollect - .flatMap(x => putStrLn(s"Encoded data with protobuf codec: ${toHex(x)}")) -).getOrThrowFiberFailure() +```scala +Encoded data with protobuf codec: 0A044A6F686E102B ``` - ## Resources -- [Zymposium - ZIO Schema](https://www.youtube.com/watch?v=GfNiDaL5aIM) by John A. De Goes, Adam Fraser and Kit Langton (May 2021) +- [Zymposium - ZIO Schema](https://www.youtube.com/watch?v=GfNiDaL5aIM) by John A. De Goes, Adam Fraser, and Kit Langton (May 2021) +- [ZIO SCHEMA: A Toolkit For Functional Distributed Computing](https://www.youtube.com/watch?v=lJziseYKvHo&t=481s) by Dan Harris (Functional Scala 2021) +- [Creating Declarative Query Plans With ZIO Schema](https://www.youtube.com/watch?v=ClePN4P9_pg) by Dan Harris (ZIO World 2022) +- [Describing Data...with free applicative functors (and more)](https://www.youtube.com/watch?v=oRLkb6mqvVM) by Kris Nuttycombe (Scala World) on the idea behind the [xenomorph](https://github.com/nuttycom/xenomorph) library ## Documentation