diff --git a/src/libraries/System.Formats.Tar/src/Resources/Strings.resx b/src/libraries/System.Formats.Tar/src/Resources/Strings.resx index 90001cf8bcc60..811972432933e 100644 --- a/src/libraries/System.Formats.Tar/src/Resources/Strings.resx +++ b/src/libraries/System.Formats.Tar/src/Resources/Strings.resx @@ -238,7 +238,7 @@ The size field is negative in a tar entry. - The value of the size field for the current entry of type '{0}' is beyond the expected length. + The value of the size field for the current entry of type '{0}' is greater than the expected length. Cannot create the symbolic link '{0}' because the specified target '{1}' does not exist. @@ -264,4 +264,7 @@ The field '{0}' exceeds the maximum allowed length for this format. + + The value of the size field for the current entry of format '{0}' is greater than the format allows. + \ No newline at end of file diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Read.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Read.cs index c1eacf8da7c72..0d5ec998497f9 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Read.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Read.cs @@ -376,7 +376,8 @@ private async Task ProcessDataBlockAsync(Stream archiveStream, bool copyData, Ca return null; } - long size = (int)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.Size, FieldLengths.Size)); + long size = (long)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.Size, FieldLengths.Size)); + Debug.Assert(size <= TarHelpers.MaxSizeLength, "size exceeded the max value possible with 11 octal digits. Actual size " + size); if (size < 0) { throw new InvalidDataException(string.Format(SR.TarSizeFieldNegative)); diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs index c63e19bd5cd0b..884c2bf1bcaf9 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs @@ -28,13 +28,13 @@ internal sealed partial class TarHeader // Writes the current header as a V7 entry into the archive stream. internal void WriteAsV7(Stream archiveStream, Span buffer) { - long actualLength = WriteV7FieldsToBuffer(buffer); + WriteV7FieldsToBuffer(buffer); archiveStream.Write(buffer); if (_dataStream != null) { - WriteData(archiveStream, _dataStream, actualLength); + WriteData(archiveStream, _dataStream, _size); } } @@ -43,39 +43,37 @@ internal async Task WriteAsV7Async(Stream archiveStream, Memory buffer, Ca { cancellationToken.ThrowIfCancellationRequested(); - long actualLength = WriteV7FieldsToBuffer(buffer.Span); + WriteV7FieldsToBuffer(buffer.Span); await archiveStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); if (_dataStream != null) { - await WriteDataAsync(archiveStream, _dataStream, actualLength, cancellationToken).ConfigureAwait(false); + await WriteDataAsync(archiveStream, _dataStream, _size, cancellationToken).ConfigureAwait(false); } } // Writes the V7 header fields to the specified buffer, calculates and writes the checksum, then returns the final data length. - private long WriteV7FieldsToBuffer(Span buffer) + private void WriteV7FieldsToBuffer(Span buffer) { - long actualLength = GetTotalDataBytesToWrite(); + _size = GetTotalDataBytesToWrite(); TarEntryType actualEntryType = TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.V7, _typeFlag); int tmpChecksum = WriteName(buffer); - tmpChecksum += WriteCommonFields(buffer, actualLength, actualEntryType); + tmpChecksum += WriteCommonFields(buffer, actualEntryType); _checksum = WriteChecksum(tmpChecksum, buffer); - - return actualLength; } // Writes the current header as a Ustar entry into the archive stream. internal void WriteAsUstar(Stream archiveStream, Span buffer) { - long actualLength = WriteUstarFieldsToBuffer(buffer); + WriteUstarFieldsToBuffer(buffer); archiveStream.Write(buffer); if (_dataStream != null) { - WriteData(archiveStream, _dataStream, actualLength); + WriteData(archiveStream, _dataStream, _size); } } @@ -84,29 +82,27 @@ internal async Task WriteAsUstarAsync(Stream archiveStream, Memory buffer, { cancellationToken.ThrowIfCancellationRequested(); - long actualLength = WriteUstarFieldsToBuffer(buffer.Span); + WriteUstarFieldsToBuffer(buffer.Span); await archiveStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); if (_dataStream != null) { - await WriteDataAsync(archiveStream, _dataStream, actualLength, cancellationToken).ConfigureAwait(false); + await WriteDataAsync(archiveStream, _dataStream, _size, cancellationToken).ConfigureAwait(false); } } // Writes the Ustar header fields to the specified buffer, calculates and writes the checksum, then returns the final data length. - private long WriteUstarFieldsToBuffer(Span buffer) + private void WriteUstarFieldsToBuffer(Span buffer) { - long actualLength = GetTotalDataBytesToWrite(); + _size = GetTotalDataBytesToWrite(); TarEntryType actualEntryType = TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.Ustar, _typeFlag); int tmpChecksum = WriteUstarName(buffer); - tmpChecksum += WriteCommonFields(buffer, actualLength, actualEntryType); + tmpChecksum += WriteCommonFields(buffer, actualEntryType); tmpChecksum += WritePosixMagicAndVersion(buffer); tmpChecksum += WritePosixAndGnuSharedFields(buffer); _checksum = WriteChecksum(tmpChecksum, buffer); - - return actualLength; } // Writes the current header as a PAX Global Extended Attributes entry into the archive stream. @@ -144,6 +140,7 @@ internal void WriteAsPax(Stream archiveStream, Span buffer) // First, we write the preceding extended attributes header TarHeader extendedAttributesHeader = new(TarEntryFormat.Pax); // Fill the current header's dict + _size = GetTotalDataBytesToWrite(); CollectExtendedAttributesFromStandardFieldsIfNeeded(); // And pass the attributes to the preceding extended attributes header for writing extendedAttributesHeader.WriteAsPaxExtendedAttributes(archiveStream, buffer, ExtendedAttributes, isGea: false, globalExtendedAttributesEntryNumber: -1); @@ -157,12 +154,12 @@ internal void WriteAsPax(Stream archiveStream, Span buffer) internal async Task WriteAsPaxAsync(Stream archiveStream, Memory buffer, CancellationToken cancellationToken) { Debug.Assert(_typeFlag is not TarEntryType.GlobalExtendedAttributes); - cancellationToken.ThrowIfCancellationRequested(); // First, we write the preceding extended attributes header TarHeader extendedAttributesHeader = new(TarEntryFormat.Pax); // Fill the current header's dict + _size = GetTotalDataBytesToWrite(); CollectExtendedAttributesFromStandardFieldsIfNeeded(); // And pass the attributes to the preceding extended attributes header for writing await extendedAttributesHeader.WriteAsPaxExtendedAttributesAsync(archiveStream, buffer, ExtendedAttributes, isGea: false, globalExtendedAttributesEntryNumber: -1, cancellationToken).ConfigureAwait(false); @@ -243,13 +240,13 @@ private static TarHeader GetGnuLongMetadataHeader(TarEntryType entryType, string // Writes the current header as a GNU entry into the archive stream. internal void WriteAsGnuInternal(Stream archiveStream, Span buffer) { - WriteAsGnuSharedInternal(buffer, out long actualLength); + WriteAsGnuSharedInternal(buffer); archiveStream.Write(buffer); if (_dataStream != null) { - WriteData(archiveStream, _dataStream, actualLength); + WriteData(archiveStream, _dataStream, _size); } } @@ -258,23 +255,23 @@ internal async Task WriteAsGnuInternalAsync(Stream archiveStream, Memory b { cancellationToken.ThrowIfCancellationRequested(); - WriteAsGnuSharedInternal(buffer.Span, out long actualLength); + WriteAsGnuSharedInternal(buffer.Span); await archiveStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); if (_dataStream != null) { - await WriteDataAsync(archiveStream, _dataStream, actualLength, cancellationToken).ConfigureAwait(false); + await WriteDataAsync(archiveStream, _dataStream, _size, cancellationToken).ConfigureAwait(false); } } // Shared checksum and data length calculations for GNU entry writing. - private void WriteAsGnuSharedInternal(Span buffer, out long actualLength) + private void WriteAsGnuSharedInternal(Span buffer) { - actualLength = GetTotalDataBytesToWrite(); + _size = GetTotalDataBytesToWrite(); int tmpChecksum = WriteName(buffer); - tmpChecksum += WriteCommonFields(buffer, actualLength, TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.Gnu, _typeFlag)); + tmpChecksum += WriteCommonFields(buffer, TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.Gnu, _typeFlag)); tmpChecksum += WriteGnuMagicAndVersion(buffer); tmpChecksum += WritePosixAndGnuSharedFields(buffer); tmpChecksum += WriteGnuFields(buffer); @@ -285,8 +282,7 @@ private void WriteAsGnuSharedInternal(Span buffer, out long actualLength) // Writes the current header as a PAX Extended Attributes entry into the archive stream. private void WriteAsPaxExtendedAttributes(Stream archiveStream, Span buffer, Dictionary extendedAttributes, bool isGea, int globalExtendedAttributesEntryNumber) { - WriteAsPaxExtendedAttributesShared(isGea, globalExtendedAttributesEntryNumber); - _dataStream = GenerateExtendedAttributesDataStream(extendedAttributes); + WriteAsPaxExtendedAttributesShared(isGea, globalExtendedAttributesEntryNumber, extendedAttributes); WriteAsPaxInternal(archiveStream, buffer); } @@ -294,22 +290,22 @@ private void WriteAsPaxExtendedAttributes(Stream archiveStream, Span buffe private Task WriteAsPaxExtendedAttributesAsync(Stream archiveStream, Memory buffer, Dictionary extendedAttributes, bool isGea, int globalExtendedAttributesEntryNumber, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - - WriteAsPaxExtendedAttributesShared(isGea, globalExtendedAttributesEntryNumber); - _dataStream = GenerateExtendedAttributesDataStream(extendedAttributes); + WriteAsPaxExtendedAttributesShared(isGea, globalExtendedAttributesEntryNumber, extendedAttributes); return WriteAsPaxInternalAsync(archiveStream, buffer, cancellationToken); } // Initializes the name, mode and type flag of a PAX extended attributes entry. - private void WriteAsPaxExtendedAttributesShared(bool isGea, int globalExtendedAttributesEntryNumber) + private void WriteAsPaxExtendedAttributesShared(bool isGea, int globalExtendedAttributesEntryNumber, Dictionary extendedAttributes) { Debug.Assert(isGea && globalExtendedAttributesEntryNumber >= 0 || !isGea && globalExtendedAttributesEntryNumber < 0); + _dataStream = GenerateExtendedAttributesDataStream(extendedAttributes); _name = isGea ? GenerateGlobalExtendedAttributeName(globalExtendedAttributesEntryNumber) : GenerateExtendedAttributeName(); _mode = TarHelpers.GetDefaultMode(_typeFlag); + _size = GetTotalDataBytesToWrite(); _typeFlag = isGea ? TarEntryType.GlobalExtendedAttributes : TarEntryType.ExtendedAttributes; } @@ -317,13 +313,13 @@ private void WriteAsPaxExtendedAttributesShared(bool isGea, int globalExtendedAt // This method writes an entry as both entries require, using the data from the current header instance. private void WriteAsPaxInternal(Stream archiveStream, Span buffer) { - WriteAsPaxSharedInternal(buffer, out long actualLength); + WriteAsPaxSharedInternal(buffer); archiveStream.Write(buffer); if (_dataStream != null) { - WriteData(archiveStream, _dataStream, actualLength); + WriteData(archiveStream, _dataStream, _size); } } @@ -333,23 +329,21 @@ private async Task WriteAsPaxInternalAsync(Stream archiveStream, Memory bu { cancellationToken.ThrowIfCancellationRequested(); - WriteAsPaxSharedInternal(buffer.Span, out long actualLength); + WriteAsPaxSharedInternal(buffer.Span); await archiveStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); if (_dataStream != null) { - await WriteDataAsync(archiveStream, _dataStream, actualLength, cancellationToken).ConfigureAwait(false); + await WriteDataAsync(archiveStream, _dataStream, _size, cancellationToken).ConfigureAwait(false); } } // Shared checksum and data length calculations for PAX entry writing. - private void WriteAsPaxSharedInternal(Span buffer, out long actualLength) + private void WriteAsPaxSharedInternal(Span buffer) { - actualLength = GetTotalDataBytesToWrite(); - int tmpChecksum = WriteName(buffer); - tmpChecksum += WriteCommonFields(buffer, actualLength, TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.Pax, _typeFlag)); + tmpChecksum += WriteCommonFields(buffer, TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.Pax, _typeFlag)); tmpChecksum += WritePosixMagicAndVersion(buffer); tmpChecksum += WritePosixAndGnuSharedFields(buffer); @@ -446,7 +440,7 @@ private int WriteUstarName(Span buffer) } // Writes all the common fields shared by all formats into the specified spans. - private int WriteCommonFields(Span buffer, long actualLength, TarEntryType actualEntryType) + private int WriteCommonFields(Span buffer, TarEntryType actualEntryType) { // Don't write an empty LinkName if the entry is a hardlink or symlink Debug.Assert(!string.IsNullOrEmpty(_linkName) ^ (_typeFlag is not TarEntryType.SymbolicLink and not TarEntryType.HardLink)); @@ -468,11 +462,21 @@ private int WriteCommonFields(Span buffer, long actualLength, TarEntryType checksum += FormatOctal(_gid, buffer.Slice(FieldLocations.Gid, FieldLengths.Gid)); } - _size = actualLength; - if (_size > 0) { - checksum += FormatOctal(_size, buffer.Slice(FieldLocations.Size, FieldLengths.Size)); + if (_size <= TarHelpers.MaxSizeLength) + { + checksum += FormatOctal(_size, buffer.Slice(FieldLocations.Size, FieldLengths.Size)); + } + else if (_format is not TarEntryFormat.Pax) + { + throw new ArgumentException(SR.Format(SR.TarSizeFieldTooLargeForEntryFormat, _format)); + } + else + { + Debug.Assert(_typeFlag is not TarEntryType.ExtendedAttributes and not TarEntryType.GlobalExtendedAttributes); + Debug.Assert(Convert.ToInt64(ExtendedAttributes[PaxEaSize]) > TarHelpers.MaxSizeLength); + } } checksum += WriteAsTimestamp(_mTime, buffer.Slice(FieldLocations.MTime, FieldLengths.MTime)); @@ -732,10 +736,14 @@ private void CollectExtendedAttributesFromStandardFieldsIfNeeded() ExtendedAttributes[PaxEaLinkName] = _linkName; } - if (_size > 99_999_999) + if (_size > TarHelpers.MaxSizeLength) { ExtendedAttributes[PaxEaSize] = _size.ToString(); } + else + { + ExtendedAttributes.Remove(PaxEaSize); + } // Sets the specified string to the dictionary if it's longer than the specified max byte length; otherwise, remove it. static void TryAddStringField(Dictionary extendedAttributes, string key, string? value, int maxLength) diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs index fb12a2d3faa74..d5393b45ffc83 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs @@ -19,6 +19,7 @@ internal static partial class TarHelpers { internal const short RecordSize = 512; internal const int MaxBufferLength = 4096; + internal const long MaxSizeLength = (1L << 33) - 1; // Max value of 11 octal digits = 2^33 - 1 or 8 Gb. // Default mode for TarEntry created for a file-type. private const UnixFileMode DefaultFileMode = diff --git a/src/libraries/System.Formats.Tar/tests/SimulatedDataStream.cs b/src/libraries/System.Formats.Tar/tests/SimulatedDataStream.cs new file mode 100644 index 0000000000000..60ba2e86be88b --- /dev/null +++ b/src/libraries/System.Formats.Tar/tests/SimulatedDataStream.cs @@ -0,0 +1,110 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO; + +namespace System.Formats.Tar.Tests +{ + // Stream that returns `length` amount of bytes with leading and trailing dummy data to verify it was correctly preserved + // e.g: + // 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x00, 0x00, 0x00, ...0x00, 0x01, 0x02, 0x03, 0x04, 0x05. + // or in decimal: + // 1, 2, 3, 4, 5, 0, 0, 0 ,0, ...0, 1, 2, 3, 4, 5. + internal class SimulatedDataStream : Stream + { + private readonly long _length; + private long _position; + internal static ReadOnlyMemory DummyData { get; } = GetDummyData(); + + private static ReadOnlyMemory GetDummyData() + { + byte[] data = new byte[5]; + new Random(42).NextBytes(data); + return data; + } + + public override bool CanRead => true; + + public override bool CanSeek => true; + + public override bool CanWrite => false; + + public override long Length => _length; + + public override long Position + { + get => _position; + set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(); + } + + _position = value; + } + } + + public SimulatedDataStream(long length) + { + if (length < 10) + { + throw new ArgumentException("Length must be at least 10 to append 5 bytes of dummy data at the beginning and end."); + } + + _length = length; + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) + => Read(buffer.AsSpan(offset, count)); + + public override int Read(Span buffer) + { + if (_position == _length || buffer.Length == 0) + { + return 0; + } + + ReadOnlySpan dummyData = DummyData.Span; + + // Write leading data and return. + if (_position < dummyData.Length - 1) + { + int bytesToWrite = Math.Min(dummyData.Length, buffer.Length); + dummyData.Slice((int)_position, bytesToWrite).CopyTo(buffer); + + _position += bytesToWrite; + return bytesToWrite; + } + + // write middle data by just zero'ing the read buffer. + int bytesToConsume = (int)Math.Min(_length - _position, buffer.Length); + Span usefulBuffer = buffer.Slice(0, bytesToConsume); + usefulBuffer.Clear(); + + _position += bytesToConsume; + long tempPos = _position; + long dummyDataTrailingLimit = _length - dummyData.Length; + + // and write trailing data at the end. + while (tempPos > dummyDataTrailingLimit) + { + int dummyDataIdx = (int)(tempPos - dummyDataTrailingLimit) - 1; + int bufferIdx = usefulBuffer.Length - 1 - (int)(_length - tempPos); + + usefulBuffer[bufferIdx] = dummyData[dummyDataIdx]; + tempPos--; + } + + return bytesToConsume; + } + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + } +} diff --git a/src/libraries/System.Formats.Tar/tests/System.Formats.Tar.Tests.csproj b/src/libraries/System.Formats.Tar/tests/System.Formats.Tar.Tests.csproj index d474185002926..ca1b4d99e508f 100644 --- a/src/libraries/System.Formats.Tar/tests/System.Formats.Tar.Tests.csproj +++ b/src/libraries/System.Formats.Tar/tests/System.Formats.Tar.Tests.csproj @@ -10,6 +10,7 @@ + @@ -48,8 +49,10 @@ + + @@ -67,6 +70,7 @@ + diff --git a/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs b/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs index 81e5b3de3c4bd..7a4aca955bf32 100644 --- a/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs +++ b/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs @@ -94,6 +94,7 @@ public abstract partial class TarTestsBase : FileCleanupTestBase internal const string FourBytesCharacter = "\uD83D\uDE12"; internal const char Separator = '/'; internal const int MaxPathComponent = 255; + internal const long LegacyMaxFileSize = (1L << 33) - 1; // Max value of 11 octal digits = 2^33 - 1 or 8 Gb. private static readonly string[] V7TestCaseNames = new[] { diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Roundtrip.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Roundtrip.Tests.cs index b13863d0b0858..650c31b1f8673 100644 --- a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Roundtrip.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Roundtrip.Tests.cs @@ -160,6 +160,7 @@ public void PaxExtendedAttributes_DoNotOverwritePublicProperties_WhenTheyFitOnLe extendedAttributes[PaxEaGName] = "ea_gname"; extendedAttributes[PaxEaUName] = "ea_uname"; extendedAttributes[PaxEaMTime] = GetTimestampStringFromDateTimeOffset(TestModificationTime); + extendedAttributes[PaxEaSize] = 42.ToString(); if (entryType is TarEntryType.HardLink or TarEntryType.SymbolicLink) { @@ -195,6 +196,9 @@ public void PaxExtendedAttributes_DoNotOverwritePublicProperties_WhenTheyFitOnLe Assert.Equal(writeEntry.UserName, readEntry.UserName); Assert.Equal(writeEntry.ModificationTime, readEntry.ModificationTime); Assert.Equal(writeEntry.LinkName, readEntry.LinkName); + + Assert.Equal(0, writeEntry.Length); + Assert.Equal(0, readEntry.Length); } [Theory] diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.LongFile.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.LongFile.Tests.cs new file mode 100644 index 0000000000000..6fd2166e81c35 --- /dev/null +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.LongFile.Tests.cs @@ -0,0 +1,91 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.IO; +using Xunit; + +namespace System.Formats.Tar.Tests +{ + [OuterLoop] + [Collection(nameof(DisableParallelization))] // don't create multiple large files at the same time + public class TarWriter_WriteEntry_LongFile_Tests : TarTestsBase + { + public static IEnumerable WriteEntry_LongFileSize_TheoryData() + { + foreach (bool unseekableStream in new[] { false, true }) + { + foreach (TarEntryFormat entryFormat in new[] { TarEntryFormat.V7, TarEntryFormat.Ustar, TarEntryFormat.Gnu, TarEntryFormat.Pax }) + { + yield return new object[] { entryFormat, LegacyMaxFileSize, unseekableStream }; + } + + // Pax supports unlimited size files. + yield return new object[] { TarEntryFormat.Pax, LegacyMaxFileSize + 1, unseekableStream }; + } + } + + [Theory] + [MemberData(nameof(WriteEntry_LongFileSize_TheoryData))] + public void WriteEntry_LongFileSize(TarEntryFormat entryFormat, long size, bool unseekableStream) + { + // Write archive with a 8 Gb long entry. + FileStream tarFile = File.Open(GetTestFilePath(), new FileStreamOptions { Access = FileAccess.ReadWrite, Mode = FileMode.Create, Options = FileOptions.DeleteOnClose }); + Stream s = unseekableStream ? new WrappedStream(tarFile, tarFile.CanRead, tarFile.CanWrite, canSeek: false) : tarFile; + + using (TarWriter writer = new(s, leaveOpen: true)) + { + TarEntry writeEntry = InvokeTarEntryCreationConstructor(entryFormat, entryFormat is TarEntryFormat.V7 ? TarEntryType.V7RegularFile : TarEntryType.RegularFile, "foo"); + writeEntry.DataStream = new SimulatedDataStream(size); + writer.WriteEntry(writeEntry); + } + + tarFile.Position = 0; + + // Read archive back. + using TarReader reader = new TarReader(s); + TarEntry entry = reader.GetNextEntry(); + Assert.Equal(size, entry.Length); + + Stream dataStream = entry.DataStream; + Assert.Equal(size, dataStream.Length); + Assert.Equal(0, dataStream.Position); + + ReadOnlySpan dummyData = SimulatedDataStream.DummyData.Span; + + // Read the first bytes. + Span buffer = new byte[dummyData.Length]; + Assert.Equal(buffer.Length, dataStream.Read(buffer)); + AssertExtensions.SequenceEqual(dummyData, buffer); + Assert.Equal(0, dataStream.ReadByte()); // check next byte is correct. + buffer.Clear(); + + // Read the last bytes. + long dummyDataOffset = size - dummyData.Length - 1; + if (dataStream.CanSeek) + { + Assert.False(unseekableStream); + dataStream.Seek(dummyDataOffset, SeekOrigin.Begin); + } + else + { + Assert.True(unseekableStream); + Span seekBuffer = new byte[4_096]; + + while (dataStream.Position < dummyDataOffset) + { + int bufSize = (int)Math.Min(seekBuffer.Length, dummyDataOffset - dataStream.Position); + int res = dataStream.Read(seekBuffer.Slice(0, bufSize)); + Assert.True(res > 0, "Unseekable stream finished before expected - Something went very wrong"); + } + } + + Assert.Equal(0, dataStream.ReadByte()); // check previous byte is correct. + Assert.Equal(buffer.Length, dataStream.Read(buffer)); + AssertExtensions.SequenceEqual(dummyData, buffer); + Assert.Equal(size, dataStream.Position); + + Assert.Null(reader.GetNextEntry()); + } + } +} diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Tests.cs index 11d3784c08b58..94522d54a6173 100644 --- a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Tests.cs @@ -490,5 +490,28 @@ public void WriteEntry_UsingTarEntry_FromTarReader_IntoTarWriter(TarEntryFormat AssertExtensions.SequenceEqual(msSource.ToArray(), msDestination.ToArray()); } + + [Theory] + [InlineData(TarEntryFormat.V7, false)] + [InlineData(TarEntryFormat.Ustar, false)] + [InlineData(TarEntryFormat.Gnu, false)] + [InlineData(TarEntryFormat.V7, true)] + [InlineData(TarEntryFormat.Ustar, true)] + [InlineData(TarEntryFormat.Gnu, true)] + public void WriteEntry_FileSizeOverLegacyLimit_Throws(TarEntryFormat entryFormat, bool unseekableStream) + { + const long FileSizeOverLimit = LegacyMaxFileSize + 1; + + MemoryStream ms = new(); + Stream s = unseekableStream ? new WrappedStream(ms, ms.CanRead, ms.CanWrite, canSeek: false) : ms; + + using TarWriter writer = new(s); + TarEntry writeEntry = InvokeTarEntryCreationConstructor(entryFormat, entryFormat is TarEntryFormat.V7 ? TarEntryType.V7RegularFile : TarEntryType.RegularFile, "foo"); + writeEntry.DataStream = new SimulatedDataStream(FileSizeOverLimit); + + Assert.Equal(FileSizeOverLimit, writeEntry.Length); + + Assert.Throws(() => writer.WriteEntry(writeEntry)); + } } } diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Roundtrip.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Roundtrip.Tests.cs index 727474e50b125..e1ba5ef7819ca 100644 --- a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Roundtrip.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Roundtrip.Tests.cs @@ -109,6 +109,7 @@ public async Task PaxExtendedAttributes_DoNotOverwritePublicProperties_WhenTheyF extendedAttributes[PaxEaGName] = "ea_gname"; extendedAttributes[PaxEaUName] = "ea_uname"; extendedAttributes[PaxEaMTime] = GetTimestampStringFromDateTimeOffset(TestModificationTime); + extendedAttributes[PaxEaSize] = 42.ToString(); if (entryType is TarEntryType.HardLink or TarEntryType.SymbolicLink) { @@ -144,6 +145,9 @@ public async Task PaxExtendedAttributes_DoNotOverwritePublicProperties_WhenTheyF Assert.Equal(writeEntry.UserName, readEntry.UserName); Assert.Equal(writeEntry.ModificationTime, readEntry.ModificationTime); Assert.Equal(writeEntry.LinkName, readEntry.LinkName); + + Assert.Equal(0, writeEntry.Length); + Assert.Equal(0, readEntry.Length); } [Theory] diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.LongFile.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.LongFile.Tests.cs new file mode 100644 index 0000000000000..0f6202662d609 --- /dev/null +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.LongFile.Tests.cs @@ -0,0 +1,81 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Xunit; + +namespace System.Formats.Tar.Tests +{ + [OuterLoop] + [Collection(nameof(DisableParallelization))] // don't create multiple large files at the same time + public class TarWriter_WriteEntryAsync_LongFile_Tests : TarTestsBase + { + public static IEnumerable WriteEntry_LongFileSize_TheoryDataAsync() + => TarWriter_WriteEntry_LongFile_Tests.WriteEntry_LongFileSize_TheoryData(); + + [Theory] + [MemberData(nameof(WriteEntry_LongFileSize_TheoryDataAsync))] + public async Task WriteEntry_LongFileSizeAsync(TarEntryFormat entryFormat, long size, bool unseekableStream) + { + // Write archive with a 8 Gb long entry. + FileStream tarFile = File.Open(GetTestFilePath(), new FileStreamOptions { Access = FileAccess.ReadWrite, Mode = FileMode.Create, Options = FileOptions.DeleteOnClose }); + Stream s = unseekableStream ? new WrappedStream(tarFile, tarFile.CanRead, tarFile.CanWrite, canSeek: false) : tarFile; + + await using (TarWriter writer = new(s, leaveOpen: true)) + { + TarEntry writeEntry = InvokeTarEntryCreationConstructor(entryFormat, entryFormat is TarEntryFormat.V7 ? TarEntryType.V7RegularFile : TarEntryType.RegularFile, "foo"); + writeEntry.DataStream = new SimulatedDataStream(size); + await writer.WriteEntryAsync(writeEntry); + } + + tarFile.Position = 0; + + // Read the archive back. + await using TarReader reader = new TarReader(s); + TarEntry entry = await reader.GetNextEntryAsync(); + Assert.Equal(size, entry.Length); + + Stream dataStream = entry.DataStream; + Assert.Equal(size, dataStream.Length); + Assert.Equal(0, dataStream.Position); + + ReadOnlyMemory dummyData = SimulatedDataStream.DummyData; + + // Read the first bytes. + byte[] buffer = new byte[dummyData.Length]; + Assert.Equal(buffer.Length, dataStream.Read(buffer)); + AssertExtensions.SequenceEqual(dummyData.Span, buffer); + Assert.Equal(0, dataStream.ReadByte()); // check next byte is correct. + buffer.AsSpan().Clear(); + + // Read the last bytes. + long dummyDataOffset = size - dummyData.Length - 1; + if (dataStream.CanSeek) + { + Assert.False(unseekableStream); + dataStream.Seek(dummyDataOffset, SeekOrigin.Begin); + } + else + { + Assert.True(unseekableStream); + Memory seekBuffer = new byte[4_096]; + + while (dataStream.Position < dummyDataOffset) + { + int bufSize = (int)Math.Min(seekBuffer.Length, dummyDataOffset - dataStream.Position); + int res = await dataStream.ReadAsync(seekBuffer.Slice(0, bufSize)); + Assert.True(res > 0, "Unseekable stream finished before expected - Something went very wrong"); + } + } + + Assert.Equal(0, dataStream.ReadByte()); // check previous byte is correct. + Assert.Equal(buffer.Length, dataStream.Read(buffer)); + AssertExtensions.SequenceEqual(dummyData.Span, buffer); + Assert.Equal(size, dataStream.Position); + + Assert.Null(await reader.GetNextEntryAsync()); + } + } +} diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Tests.cs index 84ba2d8d83c2a..f0d4d238b2f8e 100644 --- a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Tests.cs @@ -408,5 +408,29 @@ public async Task WriteEntry_UsingTarEntry_FromTarReader_IntoTarWriter_Async(Tar AssertExtensions.SequenceEqual(msSource.ToArray(), msDestination.ToArray()); } + + [Theory] + [InlineData(TarEntryFormat.V7, false)] + [InlineData(TarEntryFormat.Ustar, false)] + [InlineData(TarEntryFormat.Gnu, false)] + [InlineData(TarEntryFormat.V7, true)] + [InlineData(TarEntryFormat.Ustar, true)] + [InlineData(TarEntryFormat.Gnu, true)] + public async Task WriteEntry_FileSizeOverLegacyLimit_Throws_Async(TarEntryFormat entryFormat, bool unseekableStream) + { + const long FileSizeOverLimit = LegacyMaxFileSize + 1; + + MemoryStream ms = new(); + Stream s = unseekableStream ? new WrappedStream(ms, ms.CanRead, ms.CanWrite, canSeek: false) : ms; + + string tarFilePath = GetTestFilePath(); + await using TarWriter writer = new(File.Create(tarFilePath)); + TarEntry writeEntry = InvokeTarEntryCreationConstructor(entryFormat, entryFormat is TarEntryFormat.V7 ? TarEntryType.V7RegularFile : TarEntryType.RegularFile, "foo"); + writeEntry.DataStream = new SimulatedDataStream(FileSizeOverLimit); + + Assert.Equal(FileSizeOverLimit, writeEntry.Length); + + await Assert.ThrowsAsync(() => writer.WriteEntryAsync(writeEntry)); + } } }