Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add smart animated format conversion. #2588

Merged
merged 28 commits into from
Nov 29, 2023
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
328e046
Wire up connectors and gif encoder
JimBobSquarePants Nov 15, 2023
a486558
Complete Webp and add tests
JimBobSquarePants Nov 17, 2023
7f4b457
Default loop count should be 1
JimBobSquarePants Nov 17, 2023
aea44fa
Merge branch 'main' into js/animation-synergy
JimBobSquarePants Nov 20, 2023
958c9c9
Deduper works
JimBobSquarePants Nov 21, 2023
a42f6b6
Enable dedup for png
JimBobSquarePants Nov 23, 2023
cc0727b
Add dedup to webp
JimBobSquarePants Nov 23, 2023
4b852e6
Update tests, fix issues
JimBobSquarePants Nov 24, 2023
4f8ea7f
Tweak bounds clamping
JimBobSquarePants Nov 24, 2023
78b1cad
Merge branch 'main' into js/animation-synergy
JimBobSquarePants Nov 25, 2023
c306c56
Use correct buffer dimensions
JimBobSquarePants Nov 25, 2023
6cda7b0
Fix scanline lengths
JimBobSquarePants Nov 26, 2023
90fa817
Update PngDecoderCore.cs
JimBobSquarePants Nov 26, 2023
4029b15
Remove duplicate condition check
JimBobSquarePants Nov 26, 2023
44b0311
Remove "new"
JimBobSquarePants Nov 26, 2023
55e69c7
Try disabling new high memory tests on failing platforms.
JimBobSquarePants Nov 26, 2023
a6f96f7
Optimize and fix deduper
JimBobSquarePants Nov 28, 2023
6bfabe9
Reduce memory pressure on gif decoder tests
JimBobSquarePants Nov 28, 2023
9677be0
More memory pressure reductions
JimBobSquarePants Nov 28, 2023
35d55b5
More memory pressure reduction
JimBobSquarePants Nov 28, 2023
e35e9a8
Reduce memory usage in pixel map
JimBobSquarePants Nov 29, 2023
5da17f3
Enable Sse2, simplify
JimBobSquarePants Nov 29, 2023
10cd3d5
Update AnimationUtilities.cs
JimBobSquarePants Nov 29, 2023
f4dc0fc
Disable Sse2 for now.
JimBobSquarePants Nov 29, 2023
43c8614
Fix Sse2 offset
JimBobSquarePants Nov 29, 2023
cf4106b
Add note
JimBobSquarePants Nov 29, 2023
3424efa
Merge branch 'main' into js/animation-synergy
JimBobSquarePants Nov 29, 2023
3b759a5
Merge branch 'main' into js/animation-synergy
JimBobSquarePants Nov 29, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 93 additions & 0 deletions src/ImageSharp/Formats/AnimatedImageFrameMetadata.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.

namespace SixLabors.ImageSharp.Formats;
internal class AnimatedImageFrameMetadata
{
/// <summary>
/// Gets or sets the frame color table.
/// </summary>
public ReadOnlyMemory<Color>? ColorTable { get; set; }

/// <summary>
/// Gets or sets the frame color table mode.
/// </summary>
public FrameColorTableMode ColorTableMode { get; set; }

/// <summary>
/// Gets or sets the duration of the frame.
/// </summary>
public TimeSpan Duration { get; set; }

/// <summary>
/// Gets or sets the frame alpha blending mode.
/// </summary>
public FrameBlendMode BlendMode { get; set; }

/// <summary>
/// Gets or sets the frame disposal mode.
/// </summary>
public FrameDisposalMode DisposalMode { get; set; }
}

#pragma warning disable SA1201 // Elements should appear in the correct order
internal enum FrameBlendMode
#pragma warning restore SA1201 // Elements should appear in the correct order
{
/// <summary>
/// Do not blend. Render the current frame on the canvas by overwriting the rectangle covered by the current frame.
/// </summary>
Source = 0,

/// <summary>
/// Blend the current frame with the previous frame in the animation sequence within the rectangle covered
/// by the current frame.
/// If the current has any transparent areas, the corresponding areas of the previous frame will be visible
/// through these transparent regions.
/// </summary>
Over = 1
}

internal enum FrameDisposalMode
{
/// <summary>
/// No disposal specified.
/// The decoder is not required to take any action.
/// </summary>
Unspecified = 0,

/// <summary>
/// Do not dispose. The current frame is not disposed of, or in other words, not cleared or altered when moving to
/// the next frame. This means that the next frame is drawn over the current frame, and if the next frame contains
/// transparency, the previous frame will be visible through these transparent areas.
/// </summary>
DoNotDispose = 1,

/// <summary>
/// Restore to background color. When transitioning to the next frame, the area occupied by the current frame is
/// filled with the background color specified in the image metadata.
/// This effectively erases the current frame by replacing it with the background color before the next frame is displayed.
/// </summary>
RestoreToBackground = 2,

/// <summary>
/// Restore to previous. This method restores the area affected by the current frame to what it was before the
/// current frame was displayed. It essentially "undoes" the current frame, reverting to the state of the image
/// before the frame was displayed, then the next frame is drawn. This is useful for animations where only a small
/// part of the image changes from frame to frame.
/// </summary>
RestoreToPrevious = 3
}

internal enum FrameColorTableMode
{
/// <summary>
/// The frame uses the shared color table specified by the image metadata.
/// </summary>
Global,

/// <summary>
/// The frame uses a color table specified by the frame metadata.
/// </summary>
Local
}
32 changes: 32 additions & 0 deletions src/ImageSharp/Formats/AnimatedImageMetadata.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.

namespace SixLabors.ImageSharp.Formats;
internal class AnimatedImageMetadata
{
/// <summary>
/// Gets or sets the shared color table.
/// </summary>
public ReadOnlyMemory<Color>? ColorTable { get; set; }

/// <summary>
/// Gets or sets the shared color table mode.
/// </summary>
public FrameColorTableMode ColorTableMode { get; set; }

/// <summary>
/// Gets or sets the default background color of the canvas when animating.
/// This color may be used to fill the unused space on the canvas around the frames,
/// as well as the transparent pixels of the first frame.
/// The background color is also used when the disposal mode is <see cref="FrameDisposalMode.RestoreToBackground"/>.
/// </summary>
public Color BackgroundColor { get; set; }

/// <summary>
/// Gets or sets the number of times any animation is repeated.
/// <remarks>
/// 0 means to repeat indefinitely, count is set as repeat n-1 times. Defaults to 1.
/// </remarks>
/// </summary>
public ushort RepeatCount { get; set; }
}
190 changes: 190 additions & 0 deletions src/ImageSharp/Formats/AnimationUtilities.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.

using System.Buffers;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
using SixLabors.ImageSharp.Advanced;
using SixLabors.ImageSharp.Memory;
using SixLabors.ImageSharp.PixelFormats;

namespace SixLabors.ImageSharp.Formats;

/// <summary>
/// Utility methods for animated formats.
/// </summary>
internal static class AnimationUtilities
{
/// <summary>
/// Deduplicates pixels between the previous and current frame returning only the changed pixels and bounds.
/// </summary>
/// <typeparam name="TPixel">The type of pixel format.</typeparam>
/// <param name="configuration">The configuration.</param>
/// <param name="previousFrame">The previous frame if present.</param>
/// <param name="currentFrame">The current frame.</param>
/// <param name="resultFrame">The resultant output.</param>
/// <param name="replacement">The value to use when replacing duplicate pixels.</param>
/// <param name="clampingMode">The clamping bound to apply when calculating difference bounds.</param>
/// <returns>The <see cref="ValueTuple{Boolean, Rectangle}"/> representing the operation result.</returns>
public static (bool Difference, Rectangle Bounds) DeDuplicatePixels<TPixel>(
Configuration configuration,
ImageFrame<TPixel>? previousFrame,
ImageFrame<TPixel> currentFrame,
ImageFrame<TPixel> resultFrame,
Vector4 replacement,
ClampingMode clampingMode = ClampingMode.None)
where TPixel : unmanaged, IPixel<TPixel>
{
// TODO: This would be faster (but more complicated to find diff bounds) if we operated on Rgba32.
// If someone wants to do that, they have my unlimited thanks.
MemoryAllocator memoryAllocator = configuration.MemoryAllocator;
IMemoryOwner<Vector4> buffers = memoryAllocator.Allocate<Vector4>(currentFrame.Width * 3, AllocationOptions.Clean);
Span<Vector4> previous = buffers.GetSpan()[..currentFrame.Width];
Span<Vector4> current = buffers.GetSpan().Slice(currentFrame.Width, currentFrame.Width);
Span<Vector4> result = buffers.GetSpan()[(currentFrame.Width * 2)..];

int top = int.MinValue;
int bottom = int.MaxValue;
int left = int.MaxValue;
int right = int.MinValue;

bool hasDiff = false;
for (int y = 0; y < currentFrame.Height; y++)
{
if (previousFrame != null)
{
PixelOperations<TPixel>.Instance.ToVector4(configuration, previousFrame.DangerousGetPixelRowMemory(y).Span, previous, PixelConversionModifiers.Scale);
}

PixelOperations<TPixel>.Instance.ToVector4(configuration, currentFrame.DangerousGetPixelRowMemory(y).Span, current, PixelConversionModifiers.Scale);

ref Vector256<float> previousBase = ref Unsafe.As<Vector4, Vector256<float>>(ref MemoryMarshal.GetReference(previous));
ref Vector256<float> currentBase = ref Unsafe.As<Vector4, Vector256<float>>(ref MemoryMarshal.GetReference(current));
ref Vector256<float> resultBase = ref Unsafe.As<Vector4, Vector256<float>>(ref MemoryMarshal.GetReference(result));

Vector256<float> replacement256 = Vector256.Create(replacement.X, replacement.Y, replacement.Z, replacement.W, replacement.X, replacement.Y, replacement.Z, replacement.W);

int size = Unsafe.SizeOf<Vector4>();

bool hasRowDiff = false;
int i = 0;
uint x = 0;
int length = current.Length;
int remaining = current.Length;
if (Avx2.IsSupported && remaining >= 2)
{
while (remaining >= 2)
{
JimBobSquarePants marked this conversation as resolved.
Show resolved Hide resolved
Vector256<float> p = Unsafe.Add(ref previousBase, x);
Vector256<float> c = Unsafe.Add(ref currentBase, x);

// Compare the previous and current pixels
Vector256<int> mask = Avx2.CompareEqual(p.AsInt32(), c.AsInt32());
mask = Avx2.CompareEqual(mask.AsInt64(), Vector256<long>.AllBitsSet).AsInt32();
mask = Avx2.And(mask, Avx2.Shuffle(mask, 0b_01_00_11_10)).AsInt32();

Vector256<int> neq = Avx2.Xor(mask.AsInt64(), Vector256<long>.AllBitsSet).AsInt32();
int m = Avx2.MoveMask(neq.AsByte());
if (m != 0)
{
// If is diff is found, the left side is marked by the min of previously found left side and the start position.
// The right is the max of the previously found right side and the end position.
int start = i + (BitOperations.TrailingZeroCount(m) / size);
int end = i + (2 - (BitOperations.LeadingZeroCount((uint)m) / size));
left = Math.Min(left, start);
right = Math.Max(right, end);
hasRowDiff = true;
hasDiff = true;
}

// Replace the pixel value with the replacement if the full pixel is matched.
Vector256<float> r = Avx.BlendVariable(c, replacement256, mask.AsSingle());
Unsafe.Add(ref resultBase, x) = r;

x++;
i += 2;
remaining -= 2;
}
}

for (i = remaining; i > 0; i--)
{
x = (uint)(length - i);

Vector4 p = Unsafe.Add(ref Unsafe.As<Vector256<float>, Vector4>(ref previousBase), x);
Vector4 c = Unsafe.Add(ref Unsafe.As<Vector256<float>, Vector4>(ref currentBase), x);
ref Vector4 r = ref Unsafe.Add(ref Unsafe.As<Vector256<float>, Vector4>(ref resultBase), x);

if (p != c)
{
r = c;

// If is diff is found, the left side is marked by the min of previously found left side and the diff position.
// The right is the max of the previously found right side and the diff position + 1.
left = Math.Min(left, (int)x);
right = Math.Max(right, (int)x + 1);
hasRowDiff = true;
hasDiff = true;
}
else
{
r = replacement;
}
}

if (hasRowDiff)
{
if (top == int.MinValue)
{
top = y;
}

bottom = y + 1;
}

PixelOperations<TPixel>.Instance.FromVector4Destructive(configuration, result, resultFrame.DangerousGetPixelRowMemory(y).Span, PixelConversionModifiers.Scale);
}

Rectangle bounds = Rectangle.FromLTRB(
left = Numerics.Clamp(left, 0, resultFrame.Width - 1),
top = Numerics.Clamp(top, 0, resultFrame.Height - 1),
Numerics.Clamp(right, left + 1, resultFrame.Width),
Numerics.Clamp(bottom, top + 1, resultFrame.Height));

// Webp requires even bounds
if (clampingMode == ClampingMode.Even)
{
bounds.Width = Math.Min(resultFrame.Width, bounds.Width + (bounds.X & 1));
bounds.Height = Math.Min(resultFrame.Height, bounds.Height + (bounds.Y & 1));
bounds.X = Math.Max(0, bounds.X - (bounds.X & 1));
bounds.Y = Math.Max(0, bounds.Y - (bounds.Y & 1));
}

return new(hasDiff, bounds);
JimBobSquarePants marked this conversation as resolved.
Show resolved Hide resolved
}

public static void CopySource<TPixel>(ImageFrame<TPixel> source, ImageFrame<TPixel> destination, Rectangle bounds)
where TPixel : unmanaged, IPixel<TPixel>
{
Buffer2DRegion<TPixel> sourceBuffer = source.PixelBuffer.GetRegion(bounds);
Buffer2DRegion<TPixel> destBuffer = destination.PixelBuffer.GetRegion(bounds);
for (int y = 0; y < destBuffer.Height; y++)
{
Span<TPixel> sourceRow = sourceBuffer.DangerousGetRowSpan(y);
Span<TPixel> destRow = destBuffer.DangerousGetRowSpan(y);
sourceRow.CopyTo(destRow);
}
}
}

#pragma warning disable SA1201 // Elements should appear in the correct order
JimBobSquarePants marked this conversation as resolved.
Show resolved Hide resolved
internal enum ClampingMode
#pragma warning restore SA1201 // Elements should appear in the correct order
{
None,

Even,
}
2 changes: 2 additions & 0 deletions src/ImageSharp/Formats/Gif/GifDecoderCore.cs
Original file line number Diff line number Diff line change
Expand Up @@ -797,6 +797,8 @@ private void ReadLogicalScreenDescriptorAndGlobalColorTable(BufferedReadStream s
this.gifMetadata.GlobalColorTable = colorTable;
}
}

this.gifMetadata.BackgroundColorIndex = this.logicalScreenDescriptor.BackgroundColorIndex;
}

private unsafe struct ScratchBuffer
Expand Down
2 changes: 0 additions & 2 deletions src/ImageSharp/Formats/Gif/GifEncoder.cs
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.

using SixLabors.ImageSharp.Advanced;

namespace SixLabors.ImageSharp.Formats.Gif;

/// <summary>
Expand Down
Loading
Loading