Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

restore: trying out mmap-based file extraction #3524

Merged
merged 8 commits into from
Aug 5, 2020
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 68 additions & 1 deletion src/NuGet.Core/NuGet.Packaging/PackageArchiveReader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,72 @@ protected override void Dispose(bool disposing)
}
}

/// <summary>
/// This class literally just exists so CopyToFile gets a file size
/// </summary>
private sealed class SizedArchiveEntryStream : Stream
{
private readonly Stream _inner;

private readonly long _size;

public SizedArchiveEntryStream(Stream inner, long size)
{
_inner = inner;
_size = size;
}

public override long Length { get => _size; }

public override bool CanRead => _inner.CanRead;

public override bool CanSeek => _inner.CanSeek;

public override bool CanWrite => _inner.CanWrite;

public override long Position { get => _inner.Position; set => _inner.Position = value; }

public override void Flush()
{
_inner.Flush();
}

public override int Read(byte[] buffer, int offset, int count)
{
return _inner.Read(buffer, offset, count);
}

public override long Seek(long offset, SeekOrigin origin)
{
return _inner.Seek(offset, origin);
}

public override void SetLength(long value)
{
_inner.SetLength(value);
}

public override void Write(byte[] buffer, int offset, int count)
{
_inner.Write(buffer, offset, count);
}

zkat marked this conversation as resolved.
Show resolved Hide resolved
private bool _isDisposed;
zkat marked this conversation as resolved.
Show resolved Hide resolved

protected override void Dispose(bool disposing)
{
if (!_isDisposed)
{
if (disposing)
{
_inner.Dispose();
}

_isDisposed = true;
}
}
}

public override IEnumerable<string> CopyFiles(
string destination,
IEnumerable<string> packageFiles,
Expand Down Expand Up @@ -191,8 +257,9 @@ public override IEnumerable<string> CopyFiles(
var targetFilePath = Path.Combine(destination, normalizedPath);

using (var stream = entry.Open())
using (var sizedStream = new SizedArchiveEntryStream(stream, entry.Length))
{
var copiedFile = extractFile(packageFileName, targetFilePath, stream);
var copiedFile = extractFile(packageFileName, targetFilePath, sizedStream);
zkat marked this conversation as resolved.
Show resolved Hide resolved
if (copiedFile != null)
{
entry.UpdateFileTimeFromEntry(copiedFile, logger);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,19 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using System.IO;
using System.IO.MemoryMappedFiles;

namespace NuGet.Packaging
{
public static class StreamExtensions
{
/**
Only files smaller than this value will be mmap'ed
*/
private const long MAX_MMAP_SIZE = 10 * 1024 * 1024;

public static string CopyToFile(this Stream inputStream, string fileFullPath)
{
if (Path.GetFileName(fileFullPath).Length == 0)
Expand All @@ -27,11 +34,38 @@ public static string CopyToFile(this Stream inputStream, string fileFullPath)
return fileFullPath;
}

// For files of a certain size, we can do some Cleverness and mmap
// them instead of writing directly to disk. This can improve
// performance by a lot on some operating systems and hardware,
// particularly Windows
long? size = null;
try
{
size = inputStream.Length;
}
catch (NotSupportedException)
{
// If we can't get Length, just move on.
}
using (var outputStream = NuGetExtractionFileIO.CreateFile(fileFullPath))
{
inputStream.CopyTo(outputStream);
if (size > 0 && size <= MAX_MMAP_SIZE)
{
// NOTE: Linux can't create a mmf from outputStream, so we
// need to close the file (which now has the desired
// perms), and then re-open it as a memory-mapped file.
nkolev92 marked this conversation as resolved.
Show resolved Hide resolved
outputStream.Dispose();
using (MemoryMappedFile mmf = MemoryMappedFile.CreateFromFile(fileFullPath, FileMode.Open, mapName: null, (long)size))
using (MemoryMappedViewStream mmstream = mmf.CreateViewStream())
{
inputStream.CopyTo(mmstream);
}
}
else
{
inputStream.CopyTo(outputStream);
}
}

return fileFullPath;
}
}
Expand Down