Add BaseFileHashAlgorithm and BaseFileHash fields to DeltaMetadata st…

…ructure

build/FastRsync.nuspec

@@ -2,7 +2,7 @@
 <package xmlns="http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd">
     <metadata>
         <id>FastRsyncNet</id>
-        <version>2.2.0</version>
+        <version>2.3.0</version>
         <authors>Grzegorz Blok</authors>
         <owners>Grzegorz Blok</owners>
         <licenseUrl>https://github.com/GrzegorzBlok/FastRsyncNet/blob/master/LICENSE</licenseUrl>

source/FastRsync.Tests/DeltaReaderTests.cs

@@ -0,0 +1,75 @@
+using System.IO;
+using FastRsync.Core;
+using FastRsync.Delta;
+using FastRsync.Hash;
+using FastRsync.Signature;
+using FastRsync.Tests.FastRsyncLegacy;
+using NUnit.Framework;
+
+namespace FastRsync.Tests
+{
+    [TestFixture]
+    public class DeltaReaderTests
+    {
+        /// <summary>
+        /// Metadata without BaseFileHashAlgorithm and BaseFileHash fields
+        /// </summary>
+        private static readonly byte[] FastRsyncLegacyMetadataDelta =
+        {
+            0x46, 0x52, 0x53, 0x4e, 0x43, 0x44, 0x4c, 0x54, 0x41, 0x01, 0x69, 0x7b, 0x22, 0x68, 0x61, 0x73,
+            0x68, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x22, 0x3a, 0x22, 0x58, 0x58, 0x48,
+            0x36, 0x34, 0x22, 0x2c, 0x22, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x46, 0x69, 0x6c,
+            0x65, 0x48, 0x61, 0x73, 0x68, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x22, 0x3a,
+            0x22, 0x4d, 0x44, 0x35, 0x22, 0x2c, 0x22, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x46,
+            0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x22, 0x3a, 0x22, 0x4e, 0x33, 0x48, 0x65, 0x65, 0x51,
+            0x62, 0x48, 0x52, 0x5a, 0x62, 0x65, 0x53, 0x47, 0x35, 0x4c, 0x4c, 0x50, 0x39, 0x46, 0x2f, 0x41,
+            0x3d, 0x3d, 0x22, 0x7d, 0x80, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x65, 0x63,
+            0x64
+        };
+
+        private static readonly string FastRsyncLegacyDeltaExpectedFileHash = "N3HeeQbHRZbeSG5LLP9F/A==";
+
+        [Test]
+        public void BinaryDeltaReader_ReadsLegacyDelta()
+        {
+            // Arrange
+            var deltaStream = new MemoryStream(FastRsyncLegacyMetadataDelta);
+
+            // Act
+            IDeltaReader target = new BinaryDeltaReader(deltaStream, null);
+
+            // Assert
+            Assert.AreEqual(RsyncFormatType.FastRsync, target.Type);
+            Assert.AreEqual(new XxHashAlgorithm().Name, target.HashAlgorithm.Name);
+            Assert.AreEqual(new XxHashAlgorithm().HashLength, target.HashAlgorithm.HashLength);
+            Assert.AreEqual(FastRsyncLegacyDeltaExpectedFileHash, target.Metadata.ExpectedFileHash);
+            Assert.AreEqual("MD5", target.Metadata.ExpectedFileHashAlgorithm);
+            Assert.AreEqual(new XxHashAlgorithm().Name, target.Metadata.HashAlgorithm);
+            Assert.Null(target.Metadata.BaseFileHash);
+            Assert.Null(target.Metadata.BaseFileHashAlgorithm);
+        }
+
+        [Test]
+        public void LegacyBinaryDeltaReader_ReadsDelta()
+        {
+            // Arrange
+            var (_, baseSignatureStream, _, newDataStream) = Utils.PrepareTestData(16974, 8452, SignatureBuilder.DefaultChunkSize);
+
+            var deltaStream = new MemoryStream();
+            var deltaBuilder = new DeltaBuilder();
+            deltaBuilder.BuildDelta(newDataStream, new SignatureReader(baseSignatureStream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream)));
+            deltaStream.Seek(0, SeekOrigin.Begin);
+
+            // Act
+            var target = new BinaryDeltaReaderLegacy(deltaStream, null);
+
+            // Assert
+            Assert.AreEqual(new XxHashAlgorithm().Name, target.HashAlgorithm.Name);
+            Assert.AreEqual(new XxHashAlgorithm().HashLength, target.HashAlgorithm.HashLength);
+            Assert.AreEqual(RsyncFormatType.FastRsync, target.Type); 
+            Assert.IsNotEmpty(target.Metadata.ExpectedFileHash);
+            Assert.AreEqual("MD5", target.Metadata.ExpectedFileHashAlgorithm);
+            Assert.AreEqual(new XxHashAlgorithm().Name, target.Metadata.HashAlgorithm);
+        }
+    }
+}

source/FastRsync.Tests/FastRsyncLegacy/BinaryDeltaReaderLegacy.cs

@@ -0,0 +1,248 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using FastRsync.Core;
+using FastRsync.Delta;
+using FastRsync.Diagnostics;
+using FastRsync.Hash;
+using FastRsync.Signature;
+using Newtonsoft.Json;
+
+namespace FastRsync.Tests.FastRsyncLegacy
+{
+    internal class BinaryFormat
+    {
+        public const int SignatureFormatHeaderLength = 7; // OCTOSIG or FRSNCSG
+        public const int DeltaFormatHeaderLength = 9; // OCTODELTA or FRSNCDLTA
+
+        public const byte CopyCommand = 0x60;
+        public const byte DataCommand = 0x80;
+    }
+
+    internal class OctoBinaryFormat
+    {
+        public static readonly byte[] SignatureHeader = Encoding.ASCII.GetBytes("OCTOSIG");
+        public static readonly byte[] DeltaHeader = Encoding.ASCII.GetBytes("OCTODELTA");
+        public static readonly byte[] EndOfMetadata = Encoding.ASCII.GetBytes(">>>");
+
+        public const byte Version = 0x01;
+    }
+
+    internal class FastRsyncBinaryFormat
+    {
+        public static readonly byte[] SignatureHeader = Encoding.ASCII.GetBytes("FRSNCSG");
+        public static readonly byte[] DeltaHeader = Encoding.ASCII.GetBytes("FRSNCDLTA");
+
+        public const byte Version = 0x01;
+    }
+
+    public sealed class ProgressReport
+    {
+        public ProgressOperationType Operation { get; internal set; }
+
+        public long CurrentPosition { get; internal set; }
+
+        public long Total { get; internal set; }
+    }
+
+    internal class BinaryDeltaReaderLegacy : IDeltaReaderLegacy
+    {
+        private readonly BinaryReader reader;
+        private readonly IProgress<ProgressReport> progressReport;
+        private byte[] expectedHash;
+        private IHashAlgorithm hashAlgorithm;
+        private readonly int readBufferSize;
+
+        public BinaryDeltaReaderLegacy(Stream stream, IProgress<ProgressReport> progressHandler, int readBufferSize = 4 * 1024 * 1024)
+        {
+            this.reader = new BinaryReader(stream);
+            this.progressReport = progressHandler;
+            this.readBufferSize = readBufferSize;
+        }
+
+        private DeltaMetadataLegacy _metadata;
+        public DeltaMetadataLegacy Metadata
+        {
+            get
+            {
+                ReadMetadata();
+                return _metadata;
+            }
+        }
+
+        public RsyncFormatType Type { get; private set; }
+
+        public byte[] ExpectedHash
+        {
+            get
+            {
+                ReadMetadata();
+                return expectedHash;
+            }
+        }
+
+        public IHashAlgorithm HashAlgorithm
+        {
+            get
+            {
+                ReadMetadata();
+                return hashAlgorithm;
+            }
+        }
+
+        private void ReadMetadata()
+        {
+            if (_metadata != null)
+                return;
+
+            reader.BaseStream.Seek(0, SeekOrigin.Begin);
+
+            var header = reader.ReadBytes(BinaryFormat.DeltaFormatHeaderLength);
+
+            if (StructuralComparisons.StructuralEqualityComparer.Equals(FastRsyncBinaryFormat.DeltaHeader, header))
+            {
+                ReadFastRsyncDeltaHeader();
+                return;
+            }
+
+            if (StructuralComparisons.StructuralEqualityComparer.Equals(OctoBinaryFormat.DeltaHeader, header))
+            {
+                ReadOctoDeltaHeader();
+                return;
+            }
+
+            throw new InvalidDataException("The delta file uses a different file format than this program can handle.");
+        }
+
+        private void ReadFastRsyncDeltaHeader()
+        {
+            var version = reader.ReadByte();
+            if (version != FastRsyncBinaryFormat.Version)
+                throw new InvalidDataException("The delta file uses a newer file format than this program can handle.");
+
+            var metadataStr = reader.ReadString();
+            _metadata = JsonConvert.DeserializeObject<DeltaMetadataLegacy>(metadataStr, JsonSerializationSettings.JsonSettings);
+
+            hashAlgorithm = SupportedAlgorithms.Hashing.Create(_metadata.HashAlgorithm);
+            expectedHash = Convert.FromBase64String(_metadata.ExpectedFileHash);
+
+            Type = RsyncFormatType.FastRsync;
+        }
+
+        private void ReadOctoDeltaHeader()
+        {
+            var version = reader.ReadByte();
+            if (version != OctoBinaryFormat.Version)
+                throw new InvalidDataException("The delta file uses a newer file format than this program can handle.");
+
+            var hashAlgorithmName = reader.ReadString();
+            hashAlgorithm = SupportedAlgorithms.Hashing.Create(hashAlgorithmName);
+
+            var hashLength = reader.ReadInt32();
+            expectedHash = reader.ReadBytes(hashLength);
+            var endOfMeta = reader.ReadBytes(OctoBinaryFormat.EndOfMetadata.Length);
+            if (!StructuralComparisons.StructuralEqualityComparer.Equals(OctoBinaryFormat.EndOfMetadata, endOfMeta))
+                throw new InvalidDataException("The delta file appears to be corrupt.");
+
+            _metadata = new DeltaMetadataLegacy
+            {
+                HashAlgorithm = hashAlgorithmName,
+                ExpectedFileHashAlgorithm = hashAlgorithmName,
+                ExpectedFileHash = Convert.ToBase64String(expectedHash)
+            };
+
+            Type = RsyncFormatType.Octodiff;
+        }
+
+        public void Apply(
+            Action<byte[]> writeData,
+            Action<long, long> copy)
+        {
+            var fileLength = reader.BaseStream.Length;
+
+            ReadMetadata();
+
+            while (reader.BaseStream.Position != fileLength)
+            {
+                var b = reader.ReadByte();
+
+                progressReport?.Report(new ProgressReport
+                {
+                    Operation = ProgressOperationType.ApplyingDelta,
+                    CurrentPosition = reader.BaseStream.Position,
+                    Total = fileLength
+                });
+
+                if (b == BinaryFormat.CopyCommand)
+                {
+                    var start = reader.ReadInt64();
+                    var length = reader.ReadInt64();
+                    copy(start, length);
+                }
+                else if (b == BinaryFormat.DataCommand)
+                {
+                    var length = reader.ReadInt64();
+                    long soFar = 0;
+                    while (soFar < length)
+                    {
+                        var bytes = reader.ReadBytes((int)Math.Min(length - soFar, readBufferSize));
+                        soFar += bytes.Length;
+                        writeData(bytes);
+                    }
+                }
+            }
+        }
+
+        public async Task ApplyAsync(
+            Func<byte[], Task> writeData,
+            Func<long, long, Task> copy)
+        {
+            var fileLength = reader.BaseStream.Length;
+
+            ReadMetadata();
+
+            var buffer = new byte[readBufferSize];
+
+            while (reader.BaseStream.Position != fileLength)
+            {
+                var b = reader.ReadByte();
+
+                progressReport?.Report(new ProgressReport
+                {
+                    Operation = ProgressOperationType.ApplyingDelta,
+                    CurrentPosition = reader.BaseStream.Position,
+                    Total = fileLength
+                });
+
+                if (b == BinaryFormat.CopyCommand)
+                {
+                    var start = reader.ReadInt64();
+                    var length = reader.ReadInt64();
+                    await copy(start, length).ConfigureAwait(false);
+                }
+                else if (b == BinaryFormat.DataCommand)
+                {
+                    var length = reader.ReadInt64();
+                    long soFar = 0;
+                    while (soFar < length)
+                    {
+                        var bytesRead = await reader.BaseStream.ReadAsync(buffer, 0, (int)Math.Min(length - soFar, buffer.Length)).ConfigureAwait(false);
+                        var bytes = buffer;
+                        if (bytesRead != buffer.Length)
+                        {
+                            bytes = new byte[bytesRead];
+                            Array.Copy(buffer, bytes, bytesRead);
+                        }
+
+                        soFar += bytes.Length;
+                        await writeData(bytes).ConfigureAwait(false);
+                    }
+                }
+            }
+        }
+    }
+}

source/FastRsync.Tests/FastRsyncLegacy/DeltaMetadataLegacy.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace FastRsync.Tests.FastRsyncLegacy
+{
+    internal class DeltaMetadataLegacy
+    {
+        public string HashAlgorithm { get; set; }
+        public string ExpectedFileHashAlgorithm { get; set; }
+        public string ExpectedFileHash { get; set; }
+    }
+}

source/FastRsync.Tests/FastRsyncLegacy/IDeltaReaderLegacy.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using FastRsync.Delta;
+using FastRsync.Hash;
+using FastRsync.Signature;
+
+namespace FastRsync.Tests.FastRsyncLegacy
+{
+    internal interface IDeltaReaderLegacy
+    {
+        byte[] ExpectedHash { get; }
+        IHashAlgorithm HashAlgorithm { get; }
+        DeltaMetadataLegacy Metadata { get; }
+        RsyncFormatType Type { get; }
+        void Apply(
+            Action<byte[]> writeData,
+            Action<long, long> copy
+        );
+
+        Task ApplyAsync(
+            Func<byte[], Task> writeData,
+            Func<long, long, Task> copy
+        );
+    }
+}