diff --git a/CHANGELOG.md b/CHANGELOG.md index c0d1924..1863f95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,9 @@ The `Unreleased` section name is replaced by the expected version of next releas ## [Unreleased] ### Added +- Brotli Compression as default ### Changed +- `TryDeflate` -> `TryCompress` ### Removed ### Fixed diff --git a/src/FsCodec.Box/FsCodec.Box.fsproj b/src/FsCodec.Box/FsCodec.Box.fsproj index 23c8461..aa9b213 100644 --- a/src/FsCodec.Box/FsCodec.Box.fsproj +++ b/src/FsCodec.Box/FsCodec.Box.fsproj @@ -8,7 +8,7 @@ - + diff --git a/src/FsCodec.Box/TryDeflate.fs b/src/FsCodec.Box/TryCompress.fs similarity index 58% rename from src/FsCodec.Box/TryDeflate.fs rename to src/FsCodec.Box/TryCompress.fs index e4fc0ff..fef97e4 100644 --- a/src/FsCodec.Box/TryDeflate.fs +++ b/src/FsCodec.Box/TryCompress.fs @@ -4,14 +4,22 @@ open System open System.Runtime.CompilerServices open System.Runtime.InteropServices -module private EncodedMaybeDeflated = +module private EncodedMaybeCompressed = - type Encoding = - | Direct = 0 - | Deflate = 1 + module Encoding = + let [] Direct = 0 + let [] Deflate = 1 + let [] Brotli = 2 type Encoded = (struct (int * ReadOnlyMemory)) let empty : Encoded = int Encoding.Direct, ReadOnlyMemory.Empty + let private brotliDecompress (data: ReadOnlyMemory): byte[] = + let s = new System.IO.MemoryStream(data.ToArray(), writable = false) + use decompressor = new System.IO.Compression.BrotliStream(s, System.IO.Compression.CompressionMode.Decompress) + use output = new System.IO.MemoryStream() + decompressor.CopyTo(output) + output.ToArray() + (* EncodedBody can potentially hold compressed content, that we'll inflate on demand *) let private inflate (data : ReadOnlyMemory) : byte[] = @@ -21,22 +29,33 @@ module private EncodedMaybeDeflated = decompressor.CopyTo(output) output.ToArray() let decode struct (encoding, data) : ReadOnlyMemory = - if encoding = int Encoding.Deflate then inflate data |> ReadOnlyMemory - else data + match encoding with + | Encoding.Deflate -> inflate data |> ReadOnlyMemory + | Encoding.Brotli -> brotliDecompress data |> ReadOnlyMemory + | Encoding.Direct | _ -> data (* Compression is conditional on the input meeting a minimum size, and the result meeting a required gain *) - let private deflate (eventBody : ReadOnlyMemory) : System.IO.MemoryStream = + let private brotliCompress (eventBody: ReadOnlyMemory): System.IO.MemoryStream = let output = new System.IO.MemoryStream() - let compressor = new System.IO.Compression.DeflateStream(output, System.IO.Compression.CompressionLevel.Optimal, leaveOpen = true) + use compressor = new System.IO.Compression.BrotliStream(output, System.IO.Compression.CompressionLevel.Optimal, leaveOpen = true) compressor.Write(eventBody.Span) - compressor.Flush() + compressor.Close() output + + // NOTE: this is kept in the codebase to serve as a record of what the encoding implementation looked like for Encoding=1 + // let private deflate (eventBody : ReadOnlyMemory) : System.IO.MemoryStream = + // let output = new System.IO.MemoryStream() + // let compressor = new System.IO.Compression.DeflateStream(output, System.IO.Compression.CompressionLevel.Optimal, leaveOpen = true) + // compressor.Write(eventBody.Span) + // compressor.Flush() + // output + let encodeUncompressed (raw : ReadOnlyMemory) : Encoded = 0, raw let encode minSize minGain (raw : ReadOnlyMemory) : Encoded = if raw.Length < minSize then encodeUncompressed raw - else match deflate raw with - | tmp when raw.Length > int tmp.Length + minGain -> int Encoding.Deflate, tmp.ToArray() |> ReadOnlyMemory + else match brotliCompress raw with + | tmp when raw.Length > int tmp.Length + minGain -> int Encoding.Brotli, tmp.ToArray() |> ReadOnlyMemory | _ -> encodeUncompressed raw type [] CompressionOptions = { minSize : int; minGain : int } with @@ -50,40 +69,40 @@ type [] CompressionOptions = { minSize : int; minGain : int } with static member Uncompressed = { minSize = Int32.MaxValue; minGain = 0 } [] -type Deflate = +type Compression = static member Utf8ToEncodedDirect (x : ReadOnlyMemory) : struct (int * ReadOnlyMemory) = - EncodedMaybeDeflated.encodeUncompressed x - static member Utf8ToEncodedTryDeflate options (x : ReadOnlyMemory) : struct (int * ReadOnlyMemory) = - EncodedMaybeDeflated.encode options.minSize options.minGain x + EncodedMaybeCompressed.encodeUncompressed x + static member Utf8ToEncodedTryCompress options (x : ReadOnlyMemory) : struct (int * ReadOnlyMemory) = + EncodedMaybeCompressed.encode options.minSize options.minGain x static member EncodedToUtf8(x) : ReadOnlyMemory = - EncodedMaybeDeflated.decode x + EncodedMaybeCompressed.decode x static member EncodedToByteArray(x) : byte[] = - let u8 = EncodedMaybeDeflated.decode x in u8.ToArray() + let u8 = EncodedMaybeCompressed.decode x in u8.ToArray() /// Adapts an IEventCodec rendering to ReadOnlyMemory Event Bodies to attempt to compress the UTF-8 data.
/// If sufficient compression, as defined by options is not achieved, the body is saved as-is.
/// The int conveys a flag indicating whether compression was applied.
[] - static member EncodeTryDeflate<'Event, 'Context>(native : IEventCodec<'Event, ReadOnlyMemory, 'Context>, [] ?options) + static member EncodeTryCompress<'Event, 'Context>(native : IEventCodec<'Event, ReadOnlyMemory, 'Context>, [] ?options) : IEventCodec<'Event, struct (int * ReadOnlyMemory), 'Context> = let opts = defaultArg options CompressionOptions.Default - FsCodec.Core.EventCodec.Map(native, Deflate.Utf8ToEncodedTryDeflate opts, Func<_, _> Deflate.EncodedToUtf8) + FsCodec.Core.EventCodec.Map(native, Compression.Utf8ToEncodedTryCompress opts, Func<_, _> Compression.EncodedToUtf8) - /// Adapts an IEventCodec rendering to ReadOnlyMemory Event Bodies to encode as per EncodeTryDeflate, but without attempting compression.
+ /// Adapts an IEventCodec rendering to ReadOnlyMemory Event Bodies to encode as per EncodeTryCompress, but without attempting compression.
[] static member EncodeUncompressed<'Event, 'Context>(native : IEventCodec<'Event, ReadOnlyMemory, 'Context>) : IEventCodec<'Event, struct (int * ReadOnlyMemory), 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> Deflate.Utf8ToEncodedDirect, Func<_, _> Deflate.EncodedToUtf8) + FsCodec.Core.EventCodec.Map(native, Func<_, _> Compression.Utf8ToEncodedDirect, Func<_, _> Compression.EncodedToUtf8) /// Adapts an IEventCodec rendering to int * ReadOnlyMemory Event Bodies to render and/or consume from Uncompressed ReadOnlyMemory. [] static member ToUtf8Codec<'Event, 'Context>(native : IEventCodec<'Event, struct (int * ReadOnlyMemory), 'Context>) : IEventCodec<'Event, ReadOnlyMemory, 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> Deflate.EncodedToUtf8, Func<_, _> Deflate.Utf8ToEncodedDirect) + FsCodec.Core.EventCodec.Map(native, Func<_, _> Compression.EncodedToUtf8, Func<_, _> Compression.Utf8ToEncodedDirect) /// Adapts an IEventCodec rendering to int * ReadOnlyMemory Event Bodies to render and/or consume from Uncompressed byte[]. [] static member ToByteArrayCodec<'Event, 'Context>(native : IEventCodec<'Event, struct (int * ReadOnlyMemory), 'Context>) : IEventCodec<'Event, byte[], 'Context> = - FsCodec.Core.EventCodec.Map(native, Func<_, _> Deflate.EncodedToByteArray, Func<_, _> Deflate.Utf8ToEncodedDirect) + FsCodec.Core.EventCodec.Map(native, Func<_, _> Compression.EncodedToByteArray, Func<_, _> Compression.Utf8ToEncodedDirect) diff --git a/tests/FsCodec.Tests/FsCodec.Tests.fsproj b/tests/FsCodec.Tests/FsCodec.Tests.fsproj index db5a8f9..3f17ca8 100644 --- a/tests/FsCodec.Tests/FsCodec.Tests.fsproj +++ b/tests/FsCodec.Tests/FsCodec.Tests.fsproj @@ -7,7 +7,7 @@ - + diff --git a/tests/FsCodec.Tests/TryDeflateTests.fs b/tests/FsCodec.Tests/TryCompressTests.fs similarity index 66% rename from tests/FsCodec.Tests/TryDeflateTests.fs rename to tests/FsCodec.Tests/TryCompressTests.fs index 91f92c1..c48bbba 100644 --- a/tests/FsCodec.Tests/TryDeflateTests.fs +++ b/tests/FsCodec.Tests/TryCompressTests.fs @@ -1,4 +1,4 @@ -module FsCodec.Tests.DeflateTests +module FsCodec.Tests.CompressionTests open System open Swensen.Unquote @@ -28,9 +28,9 @@ module StringUtf8 = let res' = roundtrip sut value res' =! ValueSome value -module TryDeflate = +module TryCompress = - let sut = FsCodec.Deflate.EncodeTryDeflate(StringUtf8.sut) + let sut = FsCodec.Compression.EncodeTryCompress(StringUtf8.sut) let compressibleValue = String('x', 5000) @@ -52,10 +52,10 @@ module TryDeflate = module Uncompressed = - let sut = FsCodec.Deflate.EncodeUncompressed(StringUtf8.sut) + let sut = FsCodec.Compression.EncodeUncompressed(StringUtf8.sut) // Borrow a demonstrably compressible value - let value = TryDeflate.compressibleValue + let value = TryCompress.compressibleValue let [] roundtrips () = let res' = roundtrip sut value @@ -66,3 +66,21 @@ module Uncompressed = let encoded = sut.Encode((), value) let struct (_encoding, result) = encoded.Data true =! directResult.Span.SequenceEqual(result.Span) + + +module Decoding = + let raw = struct(0, Text.Encoding.UTF8.GetBytes("Hello World") |> ReadOnlyMemory) + let deflated = struct(1, Convert.FromBase64String("8kjNyclXCM8vykkBAAAA//8=") |> ReadOnlyMemory) + let brotli = struct(2, Convert.FromBase64String("CwWASGVsbG8gV29ybGQ=") |> ReadOnlyMemory) + + let [] ``Can decode all known bodies`` () = + let decode = FsCodec.Compression.EncodedToByteArray >> Text.Encoding.UTF8.GetString + test <@ decode raw = "Hello World" @> + test <@ decode deflated = "Hello World" @> + test <@ decode brotli = "Hello World" @> + + let [] ``Defaults to leaving the memory alone if unknown`` () = + let struct(_, mem) = raw + let body = struct(99, mem) + let decoded = body |> FsCodec.Compression.EncodedToByteArray |> Text.Encoding.UTF8.GetString + test <@ decoded = "Hello World" @>