Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[release/6.0] Don't create multiple large files at the same time #63032

Merged
merged 6 commits into from
Dec 31, 2021
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 0 additions & 15 deletions src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllBytes.cs
Original file line number Diff line number Diff line change
Expand Up @@ -57,21 +57,6 @@ public void ValidWrite(int size)
File.Delete(path);
}

[Fact]
[OuterLoop]
[ActiveIssue("https://github.com/dotnet/runtime/issues/45954", TestPlatforms.Browser)]
public void ReadFileOver2GB()
{
string path = GetTestFilePath();
using (FileStream fs = File.Create(path))
{
fs.SetLength(int.MaxValue + 1L);
}

// File is too large for ReadAllBytes at once
Assert.Throws<IOException>(() => File.ReadAllBytes(path));
}

[Fact]
public void Overwrite()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
using System.Threading.Tasks;
using Xunit;
using System.IO.Pipes;
using Microsoft.DotNet.XUnitExtensions;

namespace System.IO.Tests
{
Expand Down Expand Up @@ -70,21 +69,6 @@ public Task AlreadyCanceledAsync()
async () => await File.WriteAllBytesAsync(path, new byte[0], token));
}

[Fact]
[OuterLoop]
[ActiveIssue("https://github.com/dotnet/runtime/issues/45954", TestPlatforms.Browser)]
public Task ReadFileOver2GBAsync()
{
string path = GetTestFilePath();
using (FileStream fs = File.Create(path))
{
fs.SetLength(int.MaxValue + 1L);
}

// File is too large for ReadAllBytes at once
return Assert.ThrowsAsync<IOException>(async () => await File.ReadAllBytesAsync(path));
}

[Fact]
public async Task OverwriteAsync()
{
Expand Down
44 changes: 0 additions & 44 deletions src/libraries/System.IO.FileSystem/tests/FileStream/Read.cs
Original file line number Diff line number Diff line change
Expand Up @@ -14,49 +14,5 @@ public void NegativeReadRootThrows()
Assert.Throws<UnauthorizedAccessException>(() =>
new FileStream(Path.GetPathRoot(Directory.GetCurrentDirectory()), FileMode.Open, FileAccess.Read));
}

[ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.Is64BitProcess))]
[ActiveIssue("https://github.com/dotnet/runtime/issues/45954", TestPlatforms.Browser)]
public void NoInt32OverflowInTheBufferingLogic()
{
const long position1 = 10;
const long position2 = (1L << 32) + position1;

string filePath = GetTestFilePath();
byte[] data1 = new byte[] { 1, 2, 3, 4, 5 };
byte[] data2 = new byte[] { 6, 7, 8, 9, 10 };
byte[] buffer = new byte[5];

using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write))
{
stream.Seek(position1, SeekOrigin.Begin);
stream.Write(data1, 0, data1.Length);

stream.Seek(position2, SeekOrigin.Begin);
stream.Write(data2, 0, data2.Length);
}

using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read))
{
stream.Seek(position1, SeekOrigin.Begin);
Assert.Equal(buffer.Length, stream.Read(buffer));
Assert.Equal(data1, buffer);

stream.Seek(position2, SeekOrigin.Begin);
Assert.Equal(buffer.Length, stream.Read(buffer));
Assert.Equal(data2, buffer);
}

using (var stream = new BufferedStream(new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.None, bufferSize: 0)))
{
stream.Seek(position1, SeekOrigin.Begin);
Assert.Equal(buffer.Length, stream.Read(buffer));
Assert.Equal(data1, buffer);

stream.Seek(position2, SeekOrigin.Begin);
Assert.Equal(buffer.Length, stream.Read(buffer));
Assert.Equal(data2, buffer);
}
}
}
}
64 changes: 64 additions & 0 deletions src/libraries/System.IO.FileSystem/tests/LargeFileTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.IO.Tests;
using System.Threading.Tasks;
using Xunit;

namespace System.IO.FileSystem.Tests
{
[OuterLoop]
[ActiveIssue("https://github.com/dotnet/runtime/issues/45954", TestPlatforms.Browser)]
[Collection("NoParallelTests")] // don't create multiple large files at the same time
jozkee marked this conversation as resolved.
Show resolved Hide resolved
public class LargeFileTests : FileSystemTest
{
[Fact]
public async Task ReadAllBytesOverLimit()
{
using FileStream fs = new (GetTestFilePath(), FileMode.Create, FileAccess.Write, FileShare.Read, 4096, FileOptions.DeleteOnClose);

foreach (long lengthOverLimit in new long[] { Array.MaxLength + 1L, int.MaxValue + 1L })
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will need to get rid of this case because it's an edge case that isn't fixed in 6.0; see: #61519.

Suggested change
foreach (long lengthOverLimit in new long[] { Array.MaxLength + 1L, int.MaxValue + 1L })
foreach (long lengthOverLimit in new long[] { int.MaxValue + 1L })

{
fs.SetLength(lengthOverLimit);

Assert.Throws<IOException>(() => File.ReadAllBytes(fs.Name));
await Assert.ThrowsAsync<IOException>(async () => await File.ReadAllBytesAsync(fs.Name));
}
}

[Fact]
public void NoInt32OverflowInTheBufferingLogic()
{
const long position1 = 10;
const long position2 = (1L << 32) + position1;

string filePath = GetTestFilePath();
byte[] data1 = new byte[] { 1, 2, 3, 4, 5 };
byte[] data2 = new byte[] { 6, 7, 8, 9, 10 };
byte[] buffer = new byte[5];

using (FileStream stream = File.Create(filePath))
{
stream.Seek(position1, SeekOrigin.Begin);
stream.Write(data1);

stream.Seek(position2, SeekOrigin.Begin);
stream.Write(data2);
}

using (FileStream stream = new (filePath, FileMode.Open, FileAccess.Read, FileShare.None, 4096, FileOptions.DeleteOnClose))
{
stream.Seek(position1, SeekOrigin.Begin);
Assert.Equal(buffer.Length, stream.Read(buffer));
Assert.Equal(data1, buffer);

stream.Seek(position2, SeekOrigin.Begin);
Assert.Equal(buffer.Length, stream.Read(buffer));
Assert.Equal(data2, buffer);
}
}
}

[CollectionDefinition("NoParallelTests", DisableParallelization = true)]
jozkee marked this conversation as resolved.
Show resolved Hide resolved
public partial class NoParallelTests { }
Copy link
Member

@akoeplinger akoeplinger Dec 21, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we don't actually need this since we already have the same one here which you can include instead: https://github.com/dotnet/runtime/blob/release/6.0/src/libraries/Common/tests/System/Xml/DisableParallelization.cs

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah looks like it is already that way in main, you'll need to include the file in the .csproj so it works:

<Compile Include="$(CommonTestPath)TestUtilities\System\DisableParallelization.cs" Link="Common\TestUtilities\System\DisableParallelization.cs" />

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

File src\libraries\Common\tests\TestUtilities\System\DisableParallelization.cs doesn't exists in release/6.0.
I added the class here because that's how we were doing it prior to #62132.

Copy link
Member

@akoeplinger akoeplinger Dec 22, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah it was in a different path in release/6.0 (see the linked file in my first comment), but this is just a cosmetic issue so fine if we keep what you have :)

}
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
<Compile Include="Enumeration\ExampleTests.cs" />
<Compile Include="Enumeration\RemovedDirectoryTests.cs" />
<Compile Include="Enumeration\SymbolicLinksTests.cs" />
<Compile Include="LargeFileTests.cs" />
<Compile Include="PathInternalTests.cs" />
<Compile Include="RandomAccess\Base.cs" />
<Compile Include="RandomAccess\GetLength.cs" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,40 @@ public async Task CopyToTest_ReadBeforeCopy_CopiesAllData(bool copyAsynchronousl
Array.Copy(data, 1, expected, 0, expected.Length);
Assert.Equal(expected, dst.ToArray());
}

[Fact]
[OuterLoop]
[ActiveIssue("https://github.com/dotnet/runtime/issues/45954", TestPlatforms.Browser)]
public void NoInt32OverflowInTheBufferingLogic()
{
const long position1 = 10;
const long position2 = (1L << 32) + position1;

string filePath = Path.GetTempFileName();
byte[] data1 = new byte[] { 1, 2, 3, 4, 5 };
byte[] data2 = new byte[] { 6, 7, 8, 9, 10 };
byte[] buffer = new byte[5];

using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write))
{
stream.Seek(position1, SeekOrigin.Begin);
stream.Write(data1);

stream.Seek(position2, SeekOrigin.Begin);
stream.Write(data2);
}

using (var stream = new BufferedStream(new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.None, bufferSize: 0, FileOptions.DeleteOnClose)))
{
stream.Seek(position1, SeekOrigin.Begin);
Assert.Equal(buffer.Length, stream.Read(buffer));
Assert.Equal(data1, buffer);

stream.Seek(position2, SeekOrigin.Begin);
Assert.Equal(buffer.Length, stream.Read(buffer));
Assert.Equal(data2, buffer);
}
}
}

public class BufferedStream_TestLeaveOpen : TestLeaveOpen
Expand Down