diff --git a/src/absil/bytes.fs b/src/absil/bytes.fs index fd5f14247cf..56f3a3c1aca 100644 --- a/src/absil/bytes.fs +++ b/src/absil/bytes.fs @@ -3,7 +3,14 @@ /// Byte arrays namespace FSharp.Compiler.AbstractIL.Internal +open System +open System.IO +open System.IO.MemoryMappedFiles +open System.Runtime.InteropServices +open System.Runtime.CompilerServices +open FSharp.NativeInterop +#nowarn "9" module internal Bytes = let b0 n = (n &&& 0xFF) @@ -26,10 +33,286 @@ module internal Bytes = Array.append (System.Text.Encoding.UTF8.GetBytes s) (ofInt32Array [| 0x0 |]) let stringAsUnicodeNullTerminated (s:string) = - Array.append (System.Text.Encoding.Unicode.GetBytes s) (ofInt32Array [| 0x0;0x0 |]) + Array.append (System.Text.Encoding.Unicode.GetBytes s) (ofInt32Array [| 0x0;0x0 |]) + +[] +type ByteMemory () = + + abstract Item: int -> byte with get, set + + abstract Length: int + + abstract ReadBytes: pos: int * count: int -> byte[] + + abstract ReadInt32: pos: int -> int + + abstract ReadUInt16: pos: int -> uint16 + + abstract ReadUtf8String: pos: int * count: int -> string + + abstract Slice: pos: int * count: int -> ByteMemory + + abstract CopyTo: Stream -> unit + + abstract Copy: srcOffset: int * dest: byte[] * destOffset: int * count: int -> unit + + abstract ToArray: unit -> byte[] + + abstract AsStream: unit -> Stream + + abstract AsReadOnlyStream: unit -> Stream + +[] +type ByteArrayMemory(bytes: byte[], offset, length) = + inherit ByteMemory() + + do + if length <= 0 || length > bytes.Length then + raise (ArgumentOutOfRangeException("length")) + + if offset < 0 || (offset + length) > bytes.Length then + raise (ArgumentOutOfRangeException("offset")) + + override _.Item + with get i = bytes.[offset + i] + and set i v = bytes.[offset + i] <- v + + override _.Length = length + + override _.ReadBytes(pos, count) = + Array.sub bytes (offset + pos) count + + override _.ReadInt32 pos = + let finalOffset = offset + pos + (uint32 bytes.[finalOffset]) ||| + ((uint32 bytes.[finalOffset + 1]) <<< 8) ||| + ((uint32 bytes.[finalOffset + 2]) <<< 16) ||| + ((uint32 bytes.[finalOffset + 3]) <<< 24) + |> int + + override _.ReadUInt16 pos = + let finalOffset = offset + pos + (uint16 bytes.[finalOffset]) ||| + ((uint16 bytes.[finalOffset + 1]) <<< 8) + + override _.ReadUtf8String(pos, count) = + System.Text.Encoding.UTF8.GetString(bytes, offset + pos, count) + + override _.Slice(pos, count) = + ByteArrayMemory(bytes, offset + pos, count) :> ByteMemory + + override _.CopyTo stream = + stream.Write(bytes, offset, length) + + override _.Copy(srcOffset, dest, destOffset, count) = + Array.blit bytes (offset + srcOffset) dest destOffset count + + override _.ToArray() = + Array.sub bytes offset length + + override _.AsStream() = + new MemoryStream(bytes, offset, length) :> Stream + + override _.AsReadOnlyStream() = + new MemoryStream(bytes, offset, length, false) :> Stream + +[] +type RawByteMemory(addr: nativeptr, length: int, hold: obj) = + inherit ByteMemory () + + let check i = + if i < 0 || i >= length then + raise (ArgumentOutOfRangeException("i")) + + do + if length <= 0 then + raise (ArgumentOutOfRangeException("length")) + + override _.Item + with get i = + check i + NativePtr.add addr i + |> NativePtr.read + and set i v = + check i + NativePtr.set addr i v + + override _.Length = length + + override _.ReadUtf8String(pos, count) = + check pos + check (pos + count - 1) + System.Text.Encoding.UTF8.GetString(NativePtr.add addr pos, count) + + override _.ReadBytes(pos, count) = + check pos + check (pos + count - 1) + let res = Bytes.zeroCreate count + Marshal.Copy(NativePtr.toNativeInt addr + nativeint pos, res, 0, count) + res + + override _.ReadInt32 pos = + check pos + check (pos + 3) + Marshal.ReadInt32(NativePtr.toNativeInt addr + nativeint pos) + + override _.ReadUInt16 pos = + check pos + check (pos + 1) + uint16(Marshal.ReadInt16(NativePtr.toNativeInt addr + nativeint pos)) + + override _.Slice(pos, count) = + check pos + check (pos + count - 1) + RawByteMemory(NativePtr.add addr pos, count, hold) :> ByteMemory + + override x.CopyTo stream = + use stream2 = x.AsStream() + stream2.CopyTo stream + + override x.Copy(srcOffset, dest, destOffset, count) = + check srcOffset + Marshal.Copy(NativePtr.toNativeInt addr + nativeint srcOffset, dest, destOffset, count) + + override _.ToArray() = + let res = Array.zeroCreate length + Marshal.Copy(NativePtr.toNativeInt addr, res, 0, res.Length) + res + + override _.AsStream() = + new UnmanagedMemoryStream(addr, int64 length) :> Stream + + override _.AsReadOnlyStream() = + new UnmanagedMemoryStream(addr, int64 length, int64 length, FileAccess.Read) :> Stream + +[] +type ReadOnlyByteMemory(bytes: ByteMemory) = + + member _.Item with [] get i = bytes.[i] + + member _.Length with [] get () = bytes.Length + + [] + member _.ReadBytes(pos, count) = bytes.ReadBytes(pos, count) + + [] + member _.ReadInt32 pos = bytes.ReadInt32 pos + + [] + member _.ReadUInt16 pos = bytes.ReadUInt16 pos + + [] + member _.ReadUtf8String(pos, count) = bytes.ReadUtf8String(pos, count) + + [] + member _.Slice(pos, count) = bytes.Slice(pos, count) |> ReadOnlyByteMemory + + [] + member _.CopyTo stream = bytes.CopyTo stream + + [] + member _.Copy(srcOffset, dest, destOffset, count) = bytes.Copy(srcOffset, dest, destOffset, count) + + [] + member _.ToArray() = bytes.ToArray() + + [] + member _.AsStream() = bytes.AsReadOnlyStream() + +type ByteMemory with + + member x.AsReadOnly() = ReadOnlyByteMemory x + + static member CreateMemoryMappedFile(bytes: ReadOnlyByteMemory) = + let length = int64 bytes.Length + let mmf = + let mmf = + MemoryMappedFile.CreateNew( + null, + length, + MemoryMappedFileAccess.ReadWrite, + MemoryMappedFileOptions.None, + HandleInheritability.None) + use stream = mmf.CreateViewStream(0L, length, MemoryMappedFileAccess.ReadWrite) + bytes.CopyTo stream + mmf + + let accessor = mmf.CreateViewAccessor(0L, length, MemoryMappedFileAccess.ReadWrite) + + let safeHolder = + { new obj() with + override x.Finalize() = + (x :?> IDisposable).Dispose() + interface IDisposable with + member x.Dispose() = + GC.SuppressFinalize x + accessor.Dispose() + mmf.Dispose() } + RawByteMemory.FromUnsafePointer(accessor.SafeMemoryMappedViewHandle.DangerousGetHandle(), int length, safeHolder) + + static member FromFile(path, access, ?canShadowCopy: bool) = + let canShadowCopy = defaultArg canShadowCopy false + + let memoryMappedFileAccess = + match access with + | FileAccess.Read -> MemoryMappedFileAccess.Read + | FileAccess.Write -> MemoryMappedFileAccess.Write + | _ -> MemoryMappedFileAccess.ReadWrite + + let mmf, accessor, length = + let fileStream = File.Open(path, FileMode.Open, access, FileShare.Read) + let length = fileStream.Length + let mmf = + if canShadowCopy then + let mmf = + MemoryMappedFile.CreateNew( + null, + length, + MemoryMappedFileAccess.ReadWrite, + MemoryMappedFileOptions.None, + HandleInheritability.None) + use stream = mmf.CreateViewStream(0L, length, MemoryMappedFileAccess.ReadWrite) + fileStream.CopyTo(stream) + fileStream.Dispose() + mmf + else + MemoryMappedFile.CreateFromFile( + fileStream, + null, + length, + memoryMappedFileAccess, + HandleInheritability.None, + leaveOpen=false) + mmf, mmf.CreateViewAccessor(0L, length, memoryMappedFileAccess), length + + match access with + | FileAccess.Read when not accessor.CanRead -> failwith "Cannot read file" + | FileAccess.Write when not accessor.CanWrite -> failwith "Cannot write file" + | _ when not accessor.CanRead || not accessor.CanWrite -> failwith "Cannot read or write file" + | _ -> () + + let safeHolder = + { new obj() with + override x.Finalize() = + (x :?> IDisposable).Dispose() + interface IDisposable with + member x.Dispose() = + GC.SuppressFinalize x + accessor.Dispose() + mmf.Dispose() } + RawByteMemory.FromUnsafePointer(accessor.SafeMemoryMappedViewHandle.DangerousGetHandle(), int length, safeHolder) + + static member FromUnsafePointer(addr, length, hold: obj) = + RawByteMemory(NativePtr.ofNativeInt addr, length, hold) :> ByteMemory + + static member FromArray(bytes, offset, length) = + ByteArrayMemory(bytes, offset, length) :> ByteMemory + + static member FromArray bytes = + ByteArrayMemory.FromArray(bytes, 0, bytes.Length) type internal ByteStream = - { bytes: byte[] + { bytes: ReadOnlyByteMemory mutable pos: int max: int } member b.ReadByte() = @@ -38,18 +321,18 @@ type internal ByteStream = b.pos <- b.pos + 1 res member b.ReadUtf8String n = - let res = System.Text.Encoding.UTF8.GetString(b.bytes,b.pos,n) + let res = b.bytes.ReadUtf8String(b.pos,n) b.pos <- b.pos + n; res - static member FromBytes (b:byte[],n,len) = + static member FromBytes (b: ReadOnlyByteMemory,n,len) = if n < 0 || (n+len) > b.Length then failwith "FromBytes" { bytes = b; pos = n; max = n+len } member b.ReadBytes n = if b.pos + n > b.max then failwith "ReadBytes: end of stream" - let res = Bytes.sub b.bytes b.pos n + let res = b.bytes.Slice(b.pos, n) b.pos <- b.pos + n - res + res member b.Position = b.pos #if LAZY_UNPICKLE @@ -108,6 +391,13 @@ type internal ByteBuffer = Bytes.blit i 0 buf.bbArray buf.bbCurrent n buf.bbCurrent <- newSize + member buf.EmitByteMemory (i:ReadOnlyByteMemory) = + let n = i.Length + let newSize = buf.bbCurrent + n + buf.Ensure newSize + i.Copy(0, buf.bbArray, buf.bbCurrent, n) + buf.bbCurrent <- newSize + member buf.EmitInt32AsUInt16 n = let newSize = buf.bbCurrent + 2 buf.Ensure newSize diff --git a/src/absil/bytes.fsi b/src/absil/bytes.fsi index c611e80c776..11e7c9f58cc 100644 --- a/src/absil/bytes.fsi +++ b/src/absil/bytes.fsi @@ -3,6 +3,7 @@ /// Blobs of bytes, cross-compiling namespace FSharp.Compiler.AbstractIL.Internal +open System.IO open Internal.Utilities open FSharp.Compiler.AbstractIL @@ -22,6 +23,85 @@ module internal Bytes = val stringAsUnicodeNullTerminated: string -> byte[] val stringAsUtf8NullTerminated: string -> byte[] +/// May be backed by managed or unmanaged memory, or memory mapped file. +[] +type internal ByteMemory = + + abstract Item: int -> byte with get + + abstract Length: int + + abstract ReadBytes: pos: int * count: int -> byte[] + + abstract ReadInt32: pos: int -> int + + abstract ReadUInt16: pos: int -> uint16 + + abstract ReadUtf8String: pos: int * count: int -> string + + abstract Slice: pos: int * count: int -> ByteMemory + + abstract CopyTo: Stream -> unit + + abstract Copy: srcOffset: int * dest: byte[] * destOffset: int * count: int -> unit + + abstract ToArray: unit -> byte[] + + /// Get a stream representation of the backing memory. + /// Disposing this will not free up any of the backing memory. + abstract AsStream: unit -> Stream + + /// Get a stream representation of the backing memory. + /// Disposing this will not free up any of the backing memory. + /// Stream cannot be written to. + abstract AsReadOnlyStream: unit -> Stream + +[] +type internal ReadOnlyByteMemory = + + new: ByteMemory -> ReadOnlyByteMemory + + member Item: int -> byte with get + + member Length: int + + member ReadBytes: pos: int * count: int -> byte[] + + member ReadInt32: pos: int -> int + + member ReadUInt16: pos: int -> uint16 + + member ReadUtf8String: pos: int * count: int -> string + + member Slice: pos: int * count: int -> ReadOnlyByteMemory + + member CopyTo: Stream -> unit + + member Copy: srcOffset: int * dest: byte[] * destOffset: int * count: int -> unit + + member ToArray: unit -> byte[] + + member AsStream: unit -> Stream + +type ByteMemory with + + member AsReadOnly: unit -> ReadOnlyByteMemory + + /// Create another ByteMemory object that has a backing memory mapped file based on another ByteMemory's contents. + static member CreateMemoryMappedFile: ReadOnlyByteMemory -> ByteMemory + + /// Creates a ByteMemory object that has a backing memory mapped file from a file on-disk. + static member FromFile: path: string * FileAccess * ?canShadowCopy: bool -> ByteMemory + + /// Creates a ByteMemory object that is backed by a raw pointer. + /// Use with care. + static member FromUnsafePointer: addr: nativeint * length: int * hold: obj -> ByteMemory + + /// Creates a ByteMemory object that is backed by a byte array with the specified offset and length. + static member FromArray: bytes: byte[] * offset: int * length: int -> ByteMemory + + /// Creates a ByteMemory object that is backed by a byte array. + static member FromArray: bytes: byte[] -> ByteMemory /// Imperative buffers and streams of byte[] [] @@ -31,6 +111,7 @@ type internal ByteBuffer = member EmitIntsAsBytes : int[] -> unit member EmitByte : byte -> unit member EmitBytes : byte[] -> unit + member EmitByteMemory : ReadOnlyByteMemory -> unit member EmitInt32 : int32 -> unit member EmitInt64 : int64 -> unit member FixupInt32 : pos: int -> value: int32 -> unit @@ -44,10 +125,10 @@ type internal ByteBuffer = [] type internal ByteStream = member ReadByte : unit -> byte - member ReadBytes : int -> byte[] + member ReadBytes : int -> ReadOnlyByteMemory member ReadUtf8String : int -> string member Position : int - static member FromBytes : byte[] * start:int * length:int -> ByteStream + static member FromBytes : ReadOnlyByteMemory * start:int * length:int -> ByteStream #if LAZY_UNPICKLE member CloneAndSeek : int -> ByteStream diff --git a/src/absil/il.fs b/src/absil/il.fs index fb6e5bc320a..4736a29b39e 100644 --- a/src/absil/il.fs +++ b/src/absil/il.fs @@ -2208,8 +2208,7 @@ type ILResourceAccess = [] type ILResourceLocation = - | LocalIn of string * int * int - | LocalOut of byte[] + | Local of ReadOnlyByteMemory | File of ILModuleRef * int32 | Assembly of ILAssemblyRef @@ -2223,9 +2222,7 @@ type ILResource = /// Read the bytes from a resource local to an assembly member r.GetBytes() = match r.Location with - | ILResourceLocation.LocalIn (file, start, len) -> - File.ReadBinaryChunk(file, start, len) - | ILResourceLocation.LocalOut bytes -> bytes + | ILResourceLocation.Local bytes -> bytes | _ -> failwith "GetBytes" member x.CustomAttrs = x.CustomAttrsStored.GetCustomAttrs x.MetadataIndex @@ -4208,8 +4205,7 @@ and refs_of_exported_types s (tab: ILExportedTypesAndForwarders) = List.iter (re and refs_of_resource_where s x = match x with - | ILResourceLocation.LocalIn _ -> () - | ILResourceLocation.LocalOut _ -> () + | ILResourceLocation.Local _ -> () | ILResourceLocation.File (mref, _) -> refs_of_modref s mref | ILResourceLocation.Assembly aref -> refs_of_assemblyRef s aref diff --git a/src/absil/il.fsi b/src/absil/il.fsi index e981e8e8b54..ccafd6334f4 100644 --- a/src/absil/il.fsi +++ b/src/absil/il.fsi @@ -3,6 +3,7 @@ /// The "unlinked" view of .NET metadata and code. Central to the Abstract IL library module public FSharp.Compiler.AbstractIL.IL +open FSharp.Compiler.AbstractIL.Internal open System.Collections.Generic open System.Reflection @@ -1405,11 +1406,9 @@ type ILResourceAccess = [] type ILResourceLocation = - /// Represents a manifest resource that can be read from within the PE file - | LocalIn of string * int * int - - /// Represents a manifest resource that is due to be written to the output PE file - | LocalOut of byte[] + internal + /// Represents a manifest resource that can be read or written to a PE file + | Local of ReadOnlyByteMemory /// Represents a manifest resource in an associated file | File of ILModuleRef * int32 @@ -1429,7 +1428,7 @@ type ILResource = MetadataIndex: int32 } /// Read the bytes from a resource local to an assembly. Will fail for non-local resources. - member GetBytes : unit -> byte[] + member internal GetBytes : unit -> ReadOnlyByteMemory member CustomAttrs: ILAttributes diff --git a/src/absil/ilprint.fs b/src/absil/ilprint.fs index 7f408d48cbc..2515634395f 100644 --- a/src/absil/ilprint.fs +++ b/src/absil/ilprint.fs @@ -1024,8 +1024,7 @@ let goutput_resource env os r = output_string os " { " goutput_custom_attrs env os r.CustomAttrs match r.Location with - | ILResourceLocation.LocalIn _ - | ILResourceLocation.LocalOut _ -> + | ILResourceLocation.Local _ -> output_string os " /* loc nyi */ " | ILResourceLocation.File (mref, off) -> output_string os " .file " diff --git a/src/absil/ilread.fs b/src/absil/ilread.fs index de201333a0d..fed858c4d50 100644 --- a/src/absil/ilread.fs +++ b/src/absil/ilread.fs @@ -19,6 +19,7 @@ open System.Runtime.InteropServices open System.Text open Internal.Utilities open Internal.Utilities.Collections +open FSharp.NativeInterop open FSharp.Compiler.AbstractIL.Internal open FSharp.Compiler.AbstractIL.Internal.Support open FSharp.Compiler.AbstractIL.Diagnostics @@ -29,6 +30,8 @@ open FSharp.Compiler.ErrorLogger open FSharp.Compiler.Range open System.Reflection +#nowarn "9" + let checking = false let logging = false let _ = if checking then dprintn "warning: ILBinaryReader.checking is on" @@ -104,125 +107,31 @@ let stats = let GetStatistics() = stats -[] -/// An abstraction over how we access the contents of .NET binaries. May be backed by managed or unmanaged memory, -/// memory mapped file or by on-disk resources. These objects should never need explicit disposal - they must either -/// not hold resources of clean up after themselves when collected. -type BinaryView() = - - /// Read a byte from the file - abstract ReadByte: addr: int -> byte - - /// Read a chunk of bytes from the file - abstract ReadBytes: addr: int -> int -> byte[] - - /// Read an Int32 from the file - abstract ReadInt32: addr: int -> int +type private BinaryView = ReadOnlyByteMemory - /// Read a UInt16 from the file - abstract ReadUInt16: addr: int -> uint16 - - /// Read a length of a UTF8 string from the file - abstract CountUtf8String: addr: int -> int - - /// Read a UTF8 string from the file - abstract ReadUTF8String: addr: int -> string - -/// An abstraction over how we access the contents of .NET binaries. May be backed by managed or unmanaged memory, -/// memory mapped file or by on-disk resources. +/// An abstraction over how we access the contents of .NET binaries. type BinaryFile = - /// Return a BinaryView for temporary use which eagerly holds any necessary memory resources for the duration of its lifetime, - /// and is faster to access byte-by-byte. The returned BinaryView should _not_ be captured in a closure that outlives the - /// desired lifetime. abstract GetView: unit -> BinaryView -/// A view over a raw pointer to memory -type RawMemoryView(obj: obj, start: nativeint, len: int) = - inherit BinaryView() - - override m.ReadByte i = - if nativeint i + 1n > nativeint len then failwithf "RawMemoryView overrun, i = %d, obj = %A" i obj - Marshal.ReadByte(start + nativeint i) - - override m.ReadBytes i n = - if nativeint i + nativeint n > nativeint len then failwithf "RawMemoryView overrun, i = %d, n = %d, obj = %A" i n obj - let res = Bytes.zeroCreate n - Marshal.Copy(start + nativeint i, res, 0, n) - res - - override m.ReadInt32 i = - if nativeint i + 4n > nativeint len then failwithf "RawMemoryView overrun, i = %d, obj = %A" i obj - Marshal.ReadInt32(start + nativeint i) - - override m.ReadUInt16 i = - if nativeint i + 2n > nativeint len then failwithf "RawMemoryView overrun, i = %d, obj = %A" i obj - uint16(Marshal.ReadInt16(start + nativeint i)) - - override m.CountUtf8String i = - if nativeint i > nativeint len then failwithf "RawMemoryView overrun, i = %d, obj = %A" i obj - let pStart = start + nativeint i - let mutable p = start - while Marshal.ReadByte p <> 0uy do - p <- p + 1n - int (p - pStart) - - override m.ReadUTF8String i = - let n = m.CountUtf8String i - if nativeint i + nativeint n > nativeint len then failwithf "RawMemoryView overrun, i = %d, n = %d, obj = %A" i n obj - System.Runtime.InteropServices.Marshal.PtrToStringAnsi(start + nativeint i, n) - - member __.HoldObj() = obj - - /// Gives views over a raw chunk of memory, for example those returned to us by the memory manager in Roslyn's /// Visual Studio integration. 'obj' must keep the memory alive. The object will capture it and thus also keep the memory alive for /// the lifetime of this object. type RawMemoryFile(fileName: string, obj: obj, addr: nativeint, length: int) = do stats.rawMemoryFileCount <- stats.rawMemoryFileCount + 1 - let view = RawMemoryView(obj, addr, length) + let view = ByteMemory.FromUnsafePointer(addr, length, obj).AsReadOnly() member __.HoldObj() = obj // make sure we capture 'obj' member __.FileName = fileName interface BinaryFile with - override __.GetView() = view :>_ - -/// Read file from memory blocks -type ByteView(bytes: byte[]) = - inherit BinaryView() - - override __.ReadByte addr = bytes.[addr] - - override __.ReadBytes addr len = Array.sub bytes addr len - - override __.CountUtf8String addr = - let mutable p = addr - while bytes.[p] <> 0uy do - p <- p + 1 - p - addr - - override bfv.ReadUTF8String addr = - let n = bfv.CountUtf8String addr - System.Text.Encoding.UTF8.GetString (bytes, addr, n) - - override bfv.ReadInt32 addr = - let b0 = bfv.ReadByte addr - let b1 = bfv.ReadByte (addr+1) - let b2 = bfv.ReadByte (addr+2) - let b3 = bfv.ReadByte (addr+3) - int b0 ||| (int b1 <<< 8) ||| (int b2 <<< 16) ||| (int b3 <<< 24) - - override bfv.ReadUInt16 addr = - let b0 = bfv.ReadByte addr - let b1 = bfv.ReadByte (addr+1) - uint16 b0 ||| (uint16 b1 <<< 8) + override __.GetView() = view /// A BinaryFile backed by an array of bytes held strongly as managed memory [] type ByteFile(fileName: string, bytes: byte[]) = - let view = ByteView bytes + let view = ByteMemory.FromArray(bytes).AsReadOnly() do stats.byteFileCount <- stats.byteFileCount + 1 member __.FileName = fileName interface BinaryFile with - override bf.GetView() = view :> BinaryView + override bf.GetView() = view /// Same as ByteFile but holds the bytes weakly. The bytes will be re-read from the backing file when a view is requested. /// This is the default implementation used by F# Compiler Services when accessing "stable" binaries. It is not used @@ -261,11 +170,11 @@ type WeakByteFile(fileName: string, chunk: (int * int) option) = tg - (ByteView strongBytes :> BinaryView) + ByteMemory.FromArray(strongBytes).AsReadOnly() -let seekReadByte (mdv: BinaryView) addr = mdv.ReadByte addr -let seekReadBytes (mdv: BinaryView) addr len = mdv.ReadBytes addr len +let seekReadByte (mdv: BinaryView) addr = mdv.[addr] +let seekReadBytes (mdv: BinaryView) addr len = mdv.ReadBytes(addr, len) let seekReadInt32 (mdv: BinaryView) addr = mdv.ReadInt32 addr let seekReadUInt16 (mdv: BinaryView) addr = mdv.ReadUInt16 addr @@ -1502,7 +1411,7 @@ let rvaToData (ctxt: ILMetadataReader) (pectxt: PEReader) nm rva = let isSorted (ctxt: ILMetadataReader) (tab: TableName) = ((ctxt.sorted &&& (int64 1 <<< tab.Index)) <> int64 0x0) // Note, pectxtEager and pevEager must not be captured by the results of this function -let rec seekReadModule (ctxt: ILMetadataReader) (pectxtEager: PEReader) pevEager peinfo ilMetadataVersion idx = +let rec seekReadModule (ctxt: ILMetadataReader) canReduceMemory (pectxtEager: PEReader) pevEager peinfo ilMetadataVersion idx = let (subsys, subsysversion, useHighEntropyVA, ilOnly, only32, is32bitpreferred, only64, platform, isDll, alignVirt, alignPhys, imageBaseReal) = peinfo let mdv = ctxt.mdfile.GetView() let (_generation, nameIdx, _mvidIdx, _encidIdx, _encbaseidIdx) = seekReadModuleRow ctxt mdv idx @@ -1531,7 +1440,7 @@ let rec seekReadModule (ctxt: ILMetadataReader) (pectxtEager: PEReader) pevEager PhysicalAlignment = alignPhys ImageBase = imageBaseReal MetadataVersion = ilMetadataVersion - Resources = seekReadManifestResources ctxt mdv pectxtEager pevEager } + Resources = seekReadManifestResources ctxt canReduceMemory mdv pectxtEager pevEager } and seekReadAssemblyManifest (ctxt: ILMetadataReader) pectxt idx = let mdview = ctxt.mdfile.GetView() @@ -3101,10 +3010,10 @@ and sigptrGetILNativeType ctxt bytes sigptr = else sigptrGetZInt32 bytes sigptr ILNativeType.Array (Some nt, Some(pnum, Some additive)), sigptr else (ILNativeType.Empty, sigptr) - + // Note, pectxtEager and pevEager must not be captured by the results of this function // As a result, reading the resource offsets in the physical file is done eagerly to avoid holding on to any resources -and seekReadManifestResources (ctxt: ILMetadataReader) (mdv: BinaryView) (pectxtEager: PEReader) (pevEager: BinaryView) = +and seekReadManifestResources (ctxt: ILMetadataReader) canReduceMemory (mdv: BinaryView) (pectxtEager: PEReader) (pevEager: BinaryView) = mkILResources [ for i = 1 to ctxt.getNumRows TableNames.ManifestResource do let (offset, flags, nameIdx, implIdx) = seekReadManifestResourceRow ctxt mdv i @@ -3117,10 +3026,14 @@ and seekReadManifestResources (ctxt: ILMetadataReader) (mdv: BinaryView) (pectxt let start = pectxtEager.anyV2P ("resource", offset + pectxtEager.resourcesAddr) let resourceLength = seekReadInt32 pevEager start let offsetOfBytesFromStartOfPhysicalPEFile = start + 4 - if pectxtEager.noFileOnDisk then - ILResourceLocation.LocalOut (seekReadBytes pevEager offsetOfBytesFromStartOfPhysicalPEFile resourceLength) - else - ILResourceLocation.LocalIn (ctxt.fileName, offsetOfBytesFromStartOfPhysicalPEFile, resourceLength) + let bytes = + let bytes = pevEager.Slice(offsetOfBytesFromStartOfPhysicalPEFile, resourceLength) + // If we are trying to reduce memory, create a memory mapped file based on the contents. + if canReduceMemory then + ByteMemory.CreateMemoryMappedFile bytes + else + ByteMemory.FromArray(bytes.ToArray()) + ILResourceLocation.Local(bytes.AsReadOnly()) | ILScopeRef.Module mref -> ILResourceLocation.File (mref, offset) | ILScopeRef.Assembly aref -> ILResourceLocation.Assembly aref @@ -3207,7 +3120,7 @@ let getPdbReader pdbDirPath fileName = #endif // Note, pectxtEager and pevEager must not be captured by the results of this function -let openMetadataReader (fileName, mdfile: BinaryFile, metadataPhysLoc, peinfo, pectxtEager: PEReader, pevEager: BinaryView, pectxtCaptured, reduceMemoryUsage, ilGlobals) = +let openMetadataReader (fileName, mdfile: BinaryFile, metadataPhysLoc, peinfo, pectxtEager: PEReader, pevEager, pectxtCaptured, reduceMemoryUsage, ilGlobals) = let mdv = mdfile.GetView() let magic = seekReadUInt16AsInt32 mdv metadataPhysLoc if magic <> 0x5342 then failwith (fileName + ": bad metadata magic number: " + string magic) @@ -3584,7 +3497,7 @@ let openMetadataReader (fileName, mdfile: BinaryFile, metadataPhysLoc, peinfo, p tableBigness=tableBigness } ctxtH := Some ctxt - let ilModule = seekReadModule ctxt pectxtEager pevEager peinfo (System.Text.Encoding.UTF8.GetString (ilMetadataVersion, 0, ilMetadataVersion.Length)) 1 + let ilModule = seekReadModule ctxt reduceMemoryUsage pectxtEager pevEager peinfo (System.Text.Encoding.UTF8.GetString (ilMetadataVersion, 0, ilMetadataVersion.Length)) 1 let ilAssemblyRefs = lazy [ for i in 1 .. getNumRows TableNames.AssemblyRef do yield seekReadAssemblyRef ctxt i ] ilModule, ilAssemblyRefs @@ -3797,8 +3710,8 @@ let openPE (fileName, pefile, pdbDirPath, reduceMemoryUsage, ilGlobals, noFileOn let ilModule, ilAssemblyRefs = openMetadataReader (fileName, pefile, metadataPhysLoc, peinfo, pectxt, pev, Some pectxt, reduceMemoryUsage, ilGlobals) ilModule, ilAssemblyRefs, pdb -let openPEMetadataOnly (fileName, peinfo, pectxtEager, pev, mdfile: BinaryFile, reduceMemoryUsage, ilGlobals) = - openMetadataReader (fileName, mdfile, 0, peinfo, pectxtEager, pev, None, reduceMemoryUsage, ilGlobals) +let openPEMetadataOnly (fileName, peinfo, pectxtEager, pevEager, mdfile: BinaryFile, reduceMemoryUsage, ilGlobals) = + openMetadataReader (fileName, mdfile, 0, peinfo, pectxtEager, pevEager, None, reduceMemoryUsage, ilGlobals) let ClosePdbReader pdb = #if FX_NO_PDB_READER @@ -3992,13 +3905,12 @@ let OpenILModuleReader fileName opts = // We do however care about avoiding locks on files that prevent their deletion during a // multi-proc build. So use memory mapping, but only for stable files. Other files // still use an in-memory ByteFile - let _disposer, pefile = + let pefile = if alwaysMemoryMapFSC || stableFileHeuristicApplies fullPath then - createMemoryMapFile fullPath + let _, pefile = createMemoryMapFile fullPath + pefile else - let pefile = createByteFileChunk opts fullPath None - let disposer = { new IDisposable with member __.Dispose() = () } - disposer, pefile + createByteFileChunk opts fullPath None let ilModule, ilAssemblyRefs, pdb = openPE (fullPath, pefile, opts.pdbDirPath, reduceMemoryUsage, opts.ilGlobals, false) let ilModuleReader = new ILModuleReaderImpl(ilModule, ilAssemblyRefs, (fun () -> ClosePdbReader pdb)) diff --git a/src/absil/ilreflect.fs b/src/absil/ilreflect.fs index 69209100b98..c6a79dd5b8a 100644 --- a/src/absil/ilreflect.fs +++ b/src/absil/ilreflect.fs @@ -2070,11 +2070,9 @@ let buildModuleFragment cenv emEnv (asmB: AssemblyBuilder) (modB: ModuleBuilder) m.Resources.AsList |> List.iter (fun r -> let attribs = (match r.Access with ILResourceAccess.Public -> ResourceAttributes.Public | ILResourceAccess.Private -> ResourceAttributes.Private) match r.Location with - | ILResourceLocation.LocalIn (file, start, len) -> - let bytes = FileSystem.ReadAllBytesShim(file).[start .. start + len - 1] - modB.DefineManifestResourceAndLog (r.Name, new MemoryStream(bytes), attribs) - | ILResourceLocation.LocalOut bytes -> - modB.DefineManifestResourceAndLog (r.Name, new MemoryStream(bytes), attribs) + | ILResourceLocation.Local bytes -> + use stream = bytes.AsStream() + modB.DefineManifestResourceAndLog (r.Name, stream, attribs) | ILResourceLocation.File (mr, _) -> asmB.AddResourceFileAndLog (r.Name, mr.Name, attribs) | ILResourceLocation.Assembly _ -> diff --git a/src/absil/ilwrite.fs b/src/absil/ilwrite.fs index 409c91d590b..802ad3d3a04 100644 --- a/src/absil/ilwrite.fs +++ b/src/absil/ilwrite.fs @@ -2689,7 +2689,7 @@ and GenEventPass3 cenv env (md: ILEventDef) = let rec GetResourceAsManifestResourceRow cenv r = let data, impl = - let embedManagedResources (bytes: byte[]) = + let embedManagedResources (bytes: ReadOnlyByteMemory) = // Embedded managed resources must be word-aligned. However resource format is // not specified in ECMA. Some mscorlib resources appear to be non-aligned - it seems it doesn't matter.. let offset = cenv.resources.Position @@ -2698,12 +2698,11 @@ let rec GetResourceAsManifestResourceRow cenv r = let resourceSize = bytes.Length cenv.resources.EmitPadding pad cenv.resources.EmitInt32 resourceSize - cenv.resources.EmitBytes bytes + cenv.resources.EmitByteMemory bytes Data (alignedOffset, true), (i_File, 0) match r.Location with - | ILResourceLocation.LocalIn _ -> embedManagedResources (r.GetBytes()) - | ILResourceLocation.LocalOut bytes -> embedManagedResources bytes + | ILResourceLocation.Local bytes -> embedManagedResources bytes | ILResourceLocation.File (mref, offset) -> ULong offset, (i_File, GetModuleRefAsFileIdx cenv mref) | ILResourceLocation.Assembly aref -> ULong 0x0, (i_AssemblyRef, GetAssemblyRefAsIdx cenv aref) diff --git a/src/fsharp/CompileOps.fs b/src/fsharp/CompileOps.fs index ab20bfcf139..f82b6551ac0 100644 --- a/src/fsharp/CompileOps.fs +++ b/src/fsharp/CompileOps.fs @@ -1893,9 +1893,9 @@ type IRawFSharpAssemblyData = /// in the language service abstract TryGetILModuleDef: unit -> ILModuleDef option /// The raw F# signature data in the assembly, if any - abstract GetRawFSharpSignatureData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> byte[])) list + abstract GetRawFSharpSignatureData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> ReadOnlyByteMemory)) list /// The raw F# optimization data in the assembly, if any - abstract GetRawFSharpOptimizationData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> byte[])) list + abstract GetRawFSharpOptimizationData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> ReadOnlyByteMemory)) list /// The table of type forwarders in the assembly abstract GetRawTypeForwarders: unit -> ILExportedTypesAndForwarders /// The identity of the module @@ -3589,7 +3589,7 @@ let IsReflectedDefinitionsResource (r: ILResource) = let MakeILResource rName bytes = { Name = rName - Location = ILResourceLocation.LocalOut bytes + Location = ILResourceLocation.Local(ByteMemory.FromArray(bytes).AsReadOnly()) Access = ILResourceAccess.Public CustomAttrsStored = storeILCustomAttrs emptyILCustomAttrs MetadataIndex = NoMetadataIdx } @@ -3598,7 +3598,7 @@ let PickleToResource inMem file (g: TcGlobals) scope rName p x = let file = PathMap.apply g.pathMap file { Name = rName - Location = (let bytes = pickleObjWithDanglingCcus inMem file g scope p x in ILResourceLocation.LocalOut bytes) + Location = (let bytes = pickleObjWithDanglingCcus inMem file g scope p x in ILResourceLocation.Local(ByteMemory.FromArray(bytes).AsReadOnly())) Access = ILResourceAccess.Public CustomAttrsStored = storeILCustomAttrs emptyILCustomAttrs MetadataIndex = NoMetadataIdx } @@ -3656,7 +3656,7 @@ type RawFSharpAssemblyDataBackedByFileOnDisk (ilModule: ILModuleDef, ilAssemblyR let sigFileName = Path.ChangeExtension(filename, "sigdata") if not (FileSystem.SafeExists sigFileName) then error(Error(FSComp.SR.buildExpectedSigdataFile (FileSystem.GetFullPathShim sigFileName), m)) - [ (ilShortAssemName, fun () -> FileSystem.ReadAllBytesShim sigFileName)] + [ (ilShortAssemName, fun () -> ByteMemory.FromFile(sigFileName, FileAccess.Read, canShadowCopy=true).AsReadOnly())] else sigDataReaders sigDataReaders @@ -3671,7 +3671,7 @@ type RawFSharpAssemblyDataBackedByFileOnDisk (ilModule: ILModuleDef, ilAssemblyR let optDataFile = Path.ChangeExtension(filename, "optdata") if not (FileSystem.SafeExists optDataFile) then error(Error(FSComp.SR.buildExpectedFileAlongSideFSharpCore(optDataFile, FileSystem.GetFullPathShim optDataFile), m)) - [ (ilShortAssemName, (fun () -> FileSystem.ReadAllBytesShim optDataFile))] + [ (ilShortAssemName, (fun () -> ByteMemory.FromFile(optDataFile, FileAccess.Read, canShadowCopy=true).AsReadOnly()))] else optDataReaders optDataReaders diff --git a/src/fsharp/CompileOps.fsi b/src/fsharp/CompileOps.fsi index 51b10bc0287..230bd508de5 100644 --- a/src/fsharp/CompileOps.fsi +++ b/src/fsharp/CompileOps.fsi @@ -11,6 +11,7 @@ open FSharp.Compiler.AbstractIL open FSharp.Compiler.AbstractIL.IL open FSharp.Compiler.AbstractIL.ILBinaryReader open FSharp.Compiler.AbstractIL.ILPdbWriter +open FSharp.Compiler.AbstractIL.Internal open FSharp.Compiler.AbstractIL.Internal.Library open FSharp.Compiler open FSharp.Compiler.TypeChecker @@ -154,9 +155,9 @@ type IRawFSharpAssemblyData = abstract HasAnyFSharpSignatureDataAttribute: bool abstract HasMatchingFSharpSignatureDataAttribute: ILGlobals -> bool /// The raw F# signature data in the assembly, if any - abstract GetRawFSharpSignatureData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> byte[])) list + abstract GetRawFSharpSignatureData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> ReadOnlyByteMemory)) list /// The raw F# optimization data in the assembly, if any - abstract GetRawFSharpOptimizationData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> byte[])) list + abstract GetRawFSharpOptimizationData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> ReadOnlyByteMemory)) list /// The table of type forwarders in the assembly abstract GetRawTypeForwarders: unit -> ILExportedTypesAndForwarders /// The identity of the module diff --git a/src/fsharp/TastPickle.fs b/src/fsharp/TastPickle.fs index 97a723f6cfd..5098b16ecd0 100644 --- a/src/fsharp/TastPickle.fs +++ b/src/fsharp/TastPickle.fs @@ -284,10 +284,13 @@ let u_int32 st = assert(b0 = 0xFF) prim_u_int32 st -let u_bytes st = +let u_byte_memory st = let n = (u_int32 st) st.is.ReadBytes n +let u_bytes st = + (u_byte_memory st).ToArray() + let u_prim_string st = let len = (u_int32 st) st.is.ReadUtf8String len @@ -832,7 +835,7 @@ let check (ilscope: ILScopeRef) (inMap : NodeInTable<_, _>) = // an identical copy of the source for the DLL containing the data being unpickled. A message will // then be printed indicating the name of the item. -let unpickleObjWithDanglingCcus file ilscope (iILModule: ILModuleDef option) u (phase2bytes: byte[]) = +let unpickleObjWithDanglingCcus file ilscope (iILModule: ILModuleDef option) u (phase2bytes: ReadOnlyByteMemory) = let st2 = { is = ByteStream.FromBytes (phase2bytes, 0, phase2bytes.Length) iilscope= ilscope @@ -858,7 +861,7 @@ let unpickleObjWithDanglingCcus file ilscope (iILModule: ILModuleDef option) u ( (u_array u_encoded_pubpath) (u_array u_encoded_nleref) (u_array u_encoded_simpletyp) - u_bytes + u_byte_memory st2 let ccuTab = new_itbl "iccus" (Array.map (CcuThunk.CreateDelayed) ccuNameTab) let stringTab = new_itbl "istrings" (Array.map decode_string stringTab) diff --git a/src/fsharp/TastPickle.fsi b/src/fsharp/TastPickle.fsi index 2b659b0598d..e3d9379c07d 100644 --- a/src/fsharp/TastPickle.fsi +++ b/src/fsharp/TastPickle.fsi @@ -142,7 +142,7 @@ val internal u_ty : unpickler val internal unpickleCcuInfo : ReaderState -> PickledCcuInfo /// Deserialize an arbitrary object which may have holes referring to other compilation units -val internal unpickleObjWithDanglingCcus : file:string -> viewedScope:ILScopeRef -> ilModule:ILModuleDef option -> ('T unpickler) -> byte[] -> PickledDataWithReferences<'T> +val internal unpickleObjWithDanglingCcus : file:string -> viewedScope:ILScopeRef -> ilModule:ILModuleDef option -> ('T unpickler) -> ReadOnlyByteMemory -> PickledDataWithReferences<'T> diff --git a/src/fsharp/fsc.fs b/src/fsharp/fsc.fs index bb46f75e1db..61cc6e8547d 100644 --- a/src/fsharp/fsc.fs +++ b/src/fsharp/fsc.fs @@ -442,7 +442,9 @@ let EncodeInterfaceData(tcConfig: TcConfig, tcGlobals, exportRemapping, generate let useDataFiles = (tcConfig.useOptimizationDataFile || tcGlobals.compilingFslib) && not isIncrementalBuild if useDataFiles then let sigDataFileName = (Filename.chopExtension outfile)+".sigdata" - File.WriteAllBytes(sigDataFileName, resource.GetBytes()) + let bytes = resource.GetBytes() + use fileStream = File.Create(sigDataFileName, bytes.Length) + bytes.CopyTo fileStream let resources = [ resource ] let sigAttr = mkSignatureDataVersionAttr tcGlobals (IL.parseILVersion Internal.Utilities.FSharpEnvironment.FSharpBinaryMetadataFormatRevision) @@ -892,7 +894,7 @@ module MainModuleBuilder = [ ] let reflectedDefinitionResource = { Name=reflectedDefinitionResourceName - Location = ILResourceLocation.LocalOut reflectedDefinitionBytes + Location = ILResourceLocation.Local(ByteMemory.FromArray(reflectedDefinitionBytes).AsReadOnly()) Access= ILResourceAccess.Public CustomAttrsStored = storeILCustomAttrs emptyILCustomAttrs MetadataIndex = NoMetadataIdx } @@ -936,7 +938,7 @@ module MainModuleBuilder = let bytes = FileSystem.ReadAllBytesShim file name, bytes, pub yield { Name=name - Location=ILResourceLocation.LocalOut bytes + Location=ILResourceLocation.Local(ByteMemory.FromArray(bytes).AsReadOnly()) Access=pub CustomAttrsStored=storeILCustomAttrs emptyILCustomAttrs MetadataIndex = NoMetadataIdx } diff --git a/src/fsharp/service/IncrementalBuild.fs b/src/fsharp/service/IncrementalBuild.fs index 774fd78e108..4cb4c23ea4a 100755 --- a/src/fsharp/service/IncrementalBuild.fs +++ b/src/fsharp/service/IncrementalBuild.fs @@ -14,6 +14,7 @@ open FSharp.Compiler.Lib open FSharp.Compiler.AbstractIL open FSharp.Compiler.AbstractIL.IL open FSharp.Compiler.AbstractIL.ILBinaryReader +open FSharp.Compiler.AbstractIL.Internal open FSharp.Compiler.AbstractIL.Internal.Library open FSharp.Compiler.CompileOps open FSharp.Compiler.CompileOptions diff --git a/src/fsharp/service/service.fs b/src/fsharp/service/service.fs index 254491005a5..a98cb2966b9 100644 --- a/src/fsharp/service/service.fs +++ b/src/fsharp/service/service.fs @@ -225,7 +225,7 @@ module CompileHelpers = // Register the reflected definitions for the dynamically generated assembly for resource in ilxMainModule.Resources.AsList do if IsReflectedDefinitionsResource resource then - Quotations.Expr.RegisterReflectedDefinitions (assemblyBuilder, moduleBuilder.Name, resource.GetBytes()) + Quotations.Expr.RegisterReflectedDefinitions (assemblyBuilder, moduleBuilder.Name, resource.GetBytes().ToArray()) // Save the result assemblyBuilderRef := Some assemblyBuilder