diff --git a/README.md b/README.md index 8780217..3e410ad 100644 --- a/README.md +++ b/README.md @@ -3,3 +3,9 @@ A source code for the article "Scala Serialization" at [medium](https://medium.com/@dkomanov/scala-serialization-419d175c888a). Recent charts for the article is at https://dkomanov.github.io/scala-serialization/. + +To build and run benchmarks use the following command: + +```sbt +sbt clean 'scala-serialization-test/jmh:run -prof gc -rf json -rff jmh-result.json .*' +``` diff --git a/build.sbt b/build.sbt new file mode 100644 index 0000000..c3b1ccf --- /dev/null +++ b/build.sbt @@ -0,0 +1,59 @@ +lazy val commonSettings = Seq( + organization := "com.komanov", + organizationHomepage := Some(url("https://komanov.com")), + homepage := Some(url("https://dkomanov.github.io/scala-serialization/")), + licenses := Seq(("MIT License", url("https://opensource.org/licenses/mit-license.html"))), + startYear := Some(2016), + scalaVersion := "2.11.12", + resolvers += Resolver.bintrayRepo("evolutiongaming", "maven"), + scalacOptions ++= Seq( + "-deprecation", + "-feature", + "-Xmax-classfile-name", "240", + //"-Xmacro-settings:print-serializers" //to log sources of serializaer which are generated by kryo-macros + //"-Xmacro-settings:print-codecs" //to log sources of codecs which are generated by jsoniter-scala + ) +) + +lazy val `scala-serialization-all` = project.in(file(".")) + .aggregate(`scala-serialization`, `scala-serialization-test`) + +lazy val `scala-serialization` = project + .settings(commonSettings: _*) + .settings( + fork in Test := true, + libraryDependencies ++= Seq( + "com.twitter" %% "util-core" % "18.3.0", + "commons-io" % "commons-io" % "2.6", + "com.evolutiongaming" %% "kryo-macros" % "1.1.8", + "com.github.plokhotnyuk.jsoniter-scala" %% "macros" % "0.22.2", + "com.fasterxml.jackson.core" % "jackson-databind" % "2.9.5", + "com.fasterxml.jackson.core" % "jackson-core" % "2.9.5", + "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % "2.9.5", + "com.fasterxml.jackson.dataformat" % "jackson-dataformat-smile" % "2.9.5", + "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.9.5", + "com.google.protobuf" % "protobuf-java" % "3.0.0", + "com.trueaccord.scalapb" %% "scalapb-runtime" % "0.5.45", + "org.scala-lang.modules" %% "scala-pickling" % "0.11.0-M2", + "io.suzaku" %% "boopickle" % "1.2.6", + "com.twitter" %% "chill" % "0.9.2", + "com.esotericsoftware" % "kryo-shaded" % "4.0.1", //for chill (should match with version for kryo-macros dependencies) + "org.apache.thrift" % "libthrift" % "0.11.0", + "org.slf4j" % "slf4j-simple" % "1.7.21", //for thrift + "com.twitter" %% "scrooge-core" % "18.3.0", + "org.specs2" %% "specs2-core" % "3.8.3" % Test, + "org.specs2" %% "specs2-matcher-extra" % "3.8.3" % Test, + "org.specs2" %% "specs2-mock" % "3.8.3" % Test, + "org.specs2" %% "specs2-junit" % "3.8.3" % Test + ) + ) + +lazy val `scala-serialization-test` = project + .dependsOn(`scala-serialization`) + .enablePlugins(JmhPlugin) + .settings(commonSettings: _*) + .settings( + libraryDependencies ++= Seq( + "pl.project13.scala" % "sbt-jmh-extras" % "0.3.3" + ) + ) diff --git a/pom.xml b/pom.xml index 49e49e9..5d3265a 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ UTF-8 - 1.12 + 1.20 default 1.8 benchmarks @@ -26,6 +26,11 @@ Scala-tools Maven2 Repository http://scala-tools.org/repo-releases + + bintray-evolutiongaming-maven + EvolutionGaming Bintray Repository + https://dl.bintray.com/evolutiongaming/maven + diff --git a/project/build.properties b/project/build.properties new file mode 100644 index 0000000..31334bb --- /dev/null +++ b/project/build.properties @@ -0,0 +1 @@ +sbt.version=1.1.1 diff --git a/project/plugins.sbt b/project/plugins.sbt new file mode 100644 index 0000000..495aeb0 --- /dev/null +++ b/project/plugins.sbt @@ -0,0 +1,2 @@ +addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.3.3") +addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.0") \ No newline at end of file diff --git a/scala-serialization-test/src/main/scala/com/komanov/serialization/jmh/Benchmarks.scala b/scala-serialization-test/src/main/scala/com/komanov/serialization/jmh/Benchmarks.scala index 1647121..a56233a 100644 --- a/scala-serialization-test/src/main/scala/com/komanov/serialization/jmh/Benchmarks.scala +++ b/scala-serialization-test/src/main/scala/com/komanov/serialization/jmh/Benchmarks.scala @@ -6,7 +6,7 @@ import com.komanov.serialization.converters._ import com.komanov.serialization.domain.{EventProcessor, Site, SiteEvent} import org.openjdk.jmh.annotations._ -@State(Scope.Benchmark) +@State(Scope.Thread) // Kryo modifies bytes during parsing, see: https://github.com/EsotericSoftware/kryo#threading @BenchmarkMode(Array(Mode.AverageTime)) @OutputTimeUnit(TimeUnit.NANOSECONDS) @Fork(value = 1, jvmArgs = Array("-Xmx2G")) @@ -229,7 +229,15 @@ abstract class BenchmarkBase(converter: MyConverter) { } -class JsonBenchmark extends BenchmarkBase(JsonConverter) +class KryoMacrosBenchmark extends BenchmarkBase(KryoMacrosConverter) + +class JsoniterScalaBenchmark extends BenchmarkBase(JsoniterScalaConverter) + +class JacksonCborBenchmark extends BenchmarkBase(JacksonCborConverter) + +class JacksonSmileBenchmark extends BenchmarkBase(JacksonSmileConverter) + +class JacksonJsonBenchmark extends BenchmarkBase(JacksonJsonConverter) class ScalaPbBenchmark extends BenchmarkBase(ScalaPbConverter) diff --git a/scala-serialization/pom.xml b/scala-serialization/pom.xml index 6e8a736..c20d312 100644 --- a/scala-serialization/pom.xml +++ b/scala-serialization/pom.xml @@ -18,43 +18,62 @@ org.scala-lang scala-library - 2.11.7 + 2.11.12 com.twitter util-core_2.11 - 6.34.0 + 18.3.0 commons-io commons-io - 2.4 + 2.6 + + + com.evolutiongaming + kryo-macros_2.11 + 1.1.8 + + + com.github.plokhotnyuk.jsoniter-scala + macros_2.11 + 0.22.2 - com.fasterxml.jackson.core jackson-databind - 2.7.3 + 2.9.5 com.fasterxml.jackson.core jackson-core - 2.7.3 + 2.9.5 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-cbor + 2.9.5 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + 2.9.5 com.fasterxml.jackson.module jackson-module-scala_2.11 - 2.7.3 + 2.9.5 com.google.protobuf protobuf-java - 3.0.0-beta-2 + 3.0.0 com.trueaccord.scalapb scalapb-runtime_2.11 - 0.5.31 + 0.5.45 org.scala-lang.modules @@ -62,19 +81,19 @@ 0.11.0-M2 - me.chrons + io.suzaku boopickle_2.11 - 1.2.4 + 1.2.6 com.twitter chill_2.11 - 0.8.0 + 0.9.2 org.apache.thrift libthrift - 0.9.1 + 0.11.0 @@ -85,7 +104,7 @@ com.twitter scrooge-core_2.11 - 4.7.0 + 18.3.0 diff --git a/scala-serialization/src/main/scala/com/komanov/serialization/converters/JsonConverter.scala b/scala-serialization/src/main/scala/com/komanov/serialization/converters/JacksonConverters.scala similarity index 75% rename from scala-serialization/src/main/scala/com/komanov/serialization/converters/JsonConverter.scala rename to scala-serialization/src/main/scala/com/komanov/serialization/converters/JacksonConverters.scala index dfe20b6..c9e078e 100644 --- a/scala-serialization/src/main/scala/com/komanov/serialization/converters/JsonConverter.scala +++ b/scala-serialization/src/main/scala/com/komanov/serialization/converters/JacksonConverters.scala @@ -2,16 +2,23 @@ package com.komanov.serialization.converters import java.time.Instant -import com.fasterxml.jackson.core.{JsonGenerator, JsonParser, Version} +import com.fasterxml.jackson.core.{JsonFactory, JsonGenerator, JsonParser, Version} import com.fasterxml.jackson.databind.Module.SetupContext import com.fasterxml.jackson.databind._ import com.fasterxml.jackson.databind.module.{SimpleDeserializers, SimpleSerializers} +import com.fasterxml.jackson.dataformat.cbor.{CBORFactory, CBORGenerator} +import com.fasterxml.jackson.dataformat.smile.SmileFactory import com.fasterxml.jackson.module.scala.DefaultScalaModule -import com.komanov.serialization.domain.{Site, SiteEvent, SiteEventData} +import com.komanov.serialization.domain.{Site, SiteEvent} /** https://github.com/FasterXML/jackson */ -object JsonConverter extends MyConverter { +object JacksonJsonConverter extends JacksonConverter(new JsonFactory()) +object JacksonCborConverter extends JacksonConverter(new CBORFactory().disable(CBORGenerator.Feature.WRITE_MINIMAL_INTS)) + +object JacksonSmileConverter extends JacksonConverter(new SmileFactory()) + +abstract class JacksonConverter(jsonFactory: JsonFactory) extends MyConverter { private object InstantModule extends Module { override def getModuleName: String = "Instant" @@ -38,7 +45,7 @@ object JsonConverter extends MyConverter { } private val objectMapper = { - val om = new ObjectMapper() + val om = new ObjectMapper(jsonFactory) om.registerModule(new DefaultScalaModule) om.registerModule(InstantModule) om diff --git a/scala-serialization/src/main/scala/com/komanov/serialization/converters/JsoniterScalaConverter.scala b/scala-serialization/src/main/scala/com/komanov/serialization/converters/JsoniterScalaConverter.scala new file mode 100644 index 0000000..d3a464d --- /dev/null +++ b/scala-serialization/src/main/scala/com/komanov/serialization/converters/JsoniterScalaConverter.scala @@ -0,0 +1,30 @@ +package com.komanov.serialization.converters + +import java.time.Instant + +import com.github.plokhotnyuk.jsoniter_scala.core._ +import com.github.plokhotnyuk.jsoniter_scala.macros._ +import com.komanov.serialization.domain.{Site, SiteEvent} + +/** https://github.com/plokhotnyuk/jsoniter-scala */ +object JsoniterScalaConverter extends MyConverter { + private[this] val writerConfig = WriterConfig(preferredBufSize = 131072) + private[this] val readerConfig = ReaderConfig(preferredBufSize = 131072, preferredCharBufSize = 131072) + private[this] implicit val instantCodec: JsonValueCodec[Instant] = new JsonValueCodec[Instant] { + override def nullValue: Instant = null + + override def decodeValue(in: JsonReader, default: Instant): Instant = Instant.ofEpochMilli(in.readLong()) + + override def encodeValue(x: Instant, out: JsonWriter): Unit = out.writeVal(x.toEpochMilli) + } + private[this] implicit val siteCodec: JsonValueCodec[Site] = JsonCodecMaker.make[Site](CodecMakerConfig()) + private[this] implicit val siteEventCodec: JsonValueCodec[SiteEvent] = JsonCodecMaker.make[SiteEvent](CodecMakerConfig()) + + def toByteArray(site: Site): Array[Byte] = writeToArray(site, writerConfig) + + def fromByteArray(bytes: Array[Byte]): Site = readFromArray[Site](bytes, readerConfig) + + def toByteArray(event: SiteEvent): Array[Byte] = writeToArray(event, writerConfig) + + def siteEventFromByteArray(clazz: Class[_], bytes: Array[Byte]): SiteEvent = readFromArray[SiteEvent](bytes, readerConfig) +} diff --git a/scala-serialization/src/main/scala/com/komanov/serialization/converters/KryoMacrosConverter.scala b/scala-serialization/src/main/scala/com/komanov/serialization/converters/KryoMacrosConverter.scala new file mode 100644 index 0000000..0ddc239 --- /dev/null +++ b/scala-serialization/src/main/scala/com/komanov/serialization/converters/KryoMacrosConverter.scala @@ -0,0 +1,81 @@ +package com.komanov.serialization.converters + +import com.esotericsoftware.kryo.Kryo +import com.esotericsoftware.kryo.io.{FastInput, FastOutput} +import com.evolutiongaming.kryo.Serializer +import com.komanov.serialization.domain._ + +/** https://github.com/evolution-gaming/kryo-macros */ +case class KryoMacrosConverter(kryo: Kryo, in: FastInput, out: FastOutput) { + def read[A <: AnyRef](bs: Array[Byte])(implicit s: com.esotericsoftware.kryo.Serializer[A], m: Manifest[A]): A = { + in.setBuffer(bs) + kryo.readObject(in, m.runtimeClass.asInstanceOf[Class[A]], s) + } + + def write[A <: AnyRef](a: A)(implicit s: com.esotericsoftware.kryo.Serializer[A]): Array[Byte] = { + out.clear() + kryo.writeObject(out, a, s) + out.toBytes + } +} + +object KryoMacrosConverter extends MyConverter { + private[this] val pool = new ThreadLocal[KryoMacrosConverter] { + override def initialValue(): KryoMacrosConverter = + KryoMacrosConverter(new Kryo, new FastInput(), new FastOutput(131072, 131072)) + } + private[this] implicit val domainSerializer = Serializer.make[Domain] + private[this] implicit val metaTagSerializer = Serializer.make[MetaTag] + private[this] implicit val entryPointSerializer = Serializer.makeCommon[EntryPoint] { + case 0 => Serializer.inner[DomainEntryPoint] + case 1 => Serializer.inner[FreeEntryPoint] + } + private[this] implicit val pageComponentDataSerializer = Serializer.makeCommon[PageComponentData] { + case 0 => Serializer.inner[TextComponentData] + case 1 => Serializer.inner[ButtonComponentData] + case 2 => Serializer.inner[BlogComponentData] + } + private[this] implicit val pageComponentPositionSerializer = Serializer.make[PageComponentPosition] + private[this] implicit val pageComponentSerializer = Serializer.make[PageComponent] + private[this] implicit val pageSerializer = Serializer.make[Page] + private[this] implicit val siteSerializer = Serializer.make[Site] + private[this] implicit val siteEventSerializer = Serializer.makeCommon[SiteEvent] { + case 0 => Serializer.inner[SiteCreated] + case 1 => Serializer.inner[SiteNameSet] + case 2 => Serializer.inner[SiteDescriptionSet] + case 3 => Serializer.inner[SiteRevisionSet] + case 4 => Serializer.inner[SitePublished] + case 5 => Serializer.inner[SiteUnpublished] + case 6 => Serializer.inner[SiteFlagAdded] + case 7 => Serializer.inner[SiteFlagRemoved] + case 8 => Serializer.inner[DomainAdded] + case 9 => Serializer.inner[DomainRemoved] + case 10 => Serializer.inner[PrimaryDomainSet] + case 11 => Serializer.inner[DefaultMetaTagAdded] + case 12 => Serializer.inner[DefaultMetaTagRemoved] + case 13 => Serializer.inner[PageAdded] + case 14 => Serializer.inner[PageRemoved] + case 15 => Serializer.inner[PageNameSet] + case 16 => Serializer.inner[PageMetaTagAdded] + case 17 => Serializer.inner[PageMetaTagRemoved] + case 18 => Serializer.inner[PageComponentAdded] + case 19 => Serializer.inner[PageComponentRemoved] + case 20 => Serializer.inner[PageComponentPositionSet] + case 21 => Serializer.inner[PageComponentPositionReset] + case 22 => Serializer.inner[TextComponentDataSet] + case 23 => Serializer.inner[ButtonComponentDataSet] + case 24 => Serializer.inner[BlogComponentDataSet] + case 25 => Serializer.inner[DomainEntryPointAdded] + case 26 => Serializer.inner[FreeEntryPointAdded] + case 27 => Serializer.inner[EntryPointRemoved] + case 28 => Serializer.inner[PrimaryEntryPointSet] + } + + def toByteArray(site: Site): Array[Byte] = pool.get().write(site) + + def fromByteArray(bytes: Array[Byte]): Site = pool.get().read[Site](bytes) + + def toByteArray(event: SiteEvent): Array[Byte] = pool.get().write(event) + + def siteEventFromByteArray(clazz: Class[_], bytes: Array[Byte]): SiteEvent = pool.get().read[SiteEvent](bytes) +} diff --git a/scala-serialization/src/test/scala/com/komanov/serialization/converters/Converters.scala b/scala-serialization/src/test/scala/com/komanov/serialization/converters/Converters.scala index a497185..1e19736 100644 --- a/scala-serialization/src/test/scala/com/komanov/serialization/converters/Converters.scala +++ b/scala-serialization/src/test/scala/com/komanov/serialization/converters/Converters.scala @@ -3,7 +3,11 @@ package com.komanov.serialization.converters object Converters { val all: Seq[(String, MyConverter)] = Seq( - "JSON" -> JsonConverter, + "KryoMacros" -> KryoMacrosConverter, + "JsoniterScala" -> JsoniterScalaConverter, + "Jackson CBOR" -> JacksonCborConverter, + "Jackson JSON" -> JacksonJsonConverter, + "Jackson Smile" -> JacksonSmileConverter, "ScalaPB" -> ScalaPbConverter, "Java PB" -> JavaPbConverter, "Java Thrift" -> JavaThriftConverter, diff --git a/scala-serialization/src/test/scala/com/komanov/serialization/converters/SerializationTest.scala b/scala-serialization/src/test/scala/com/komanov/serialization/converters/SerializationTest.scala index bb44bf6..2a74a52 100644 --- a/scala-serialization/src/test/scala/com/komanov/serialization/converters/SerializationTest.scala +++ b/scala-serialization/src/test/scala/com/komanov/serialization/converters/SerializationTest.scala @@ -12,7 +12,11 @@ class SerializationTest extends SpecificationWithJUnit { sequential - doTest("JSON", JsonConverter) + doTest("KryoMacros", KryoMacrosConverter) + doTest("JsoniterScala", JsoniterScalaConverter) + doTest("Jackson CBOR", JacksonCborConverter) + doTest("Jackson JSON", JacksonJsonConverter) + doTest("Jackson Smile", JacksonSmileConverter) doTest("ScalaPB", ScalaPbConverter) doTest("Java Protobuf", JavaPbConverter) doTest("Java Thrift", JavaThriftConverter) diff --git a/version.sbt b/version.sbt new file mode 100644 index 0000000..6243c9f --- /dev/null +++ b/version.sbt @@ -0,0 +1 @@ +version in ThisBuild := "1.0-SNAPSHOT"