diff --git a/.paket/Paket.Restore.targets b/.paket/Paket.Restore.targets index d93abfef..e33a731f 100644 --- a/.paket/Paket.Restore.targets +++ b/.paket/Paket.Restore.targets @@ -17,23 +17,43 @@ native /Library/Frameworks/Mono.framework/Commands/mono mono - - $(PaketToolsPath)paket - $(PaketRootPath)paket.exe - $(PaketToolsPath)paket.exe - $(PaketToolsPath)paket.exe - $(PaketToolsPath)paket + + $(PaketRootPath)paket.bootstrapper.exe + $(PaketToolsPath)paket.bootstrapper.exe + $([System.IO.Path]::GetDirectoryName("$(PaketBootStrapperExePath)"))\ + + + + + $(PaketRootPath)paket.exe + $(PaketToolsPath)paket.exe + $(PaketToolsPath)paket.exe + $(_PaketBootStrapperExeDir)paket.exe + paket.exe + + + $(PaketRootPath)paket + $(PaketToolsPath)paket + $(PaketToolsPath)paket + + + $(PaketRootPath)paket.exe + $(PaketToolsPath)paket.exe + + + $(PaketBootStrapperExeDir)paket.exe + + + paket <_PaketExeExtension>$([System.IO.Path]::GetExtension("$(PaketExePath)")) dotnet "$(PaketExePath)" - "$(PaketExePath)" $(MonoPath) --runtime=v4.0.30319 "$(PaketExePath)" - "$(PaketExePath)" + "$(PaketExePath)" + - $(PaketRootPath)paket.bootstrapper.exe - $(PaketToolsPath)paket.bootstrapper.exe "$(PaketBootStrapperExePath)" $(MonoPath) --runtime=v4.0.30319 "$(PaketBootStrapperExePath)" @@ -42,6 +62,9 @@ true true + + + True @@ -82,7 +105,11 @@ true - + + true @@ -163,6 +190,7 @@ runtime runtime true + true @@ -200,6 +228,7 @@ + <_NuspecFilesNewLocation Include="$(BaseIntermediateOutputPath)$(Configuration)\*.nuspec"/> @@ -209,12 +238,14 @@ $(MSBuildProjectDirectory)/$(MSBuildProjectFile) true + false + true false - true + true false - true + true false - true + true $(BaseIntermediateOutputPath)$(Configuration) $(BaseIntermediateOutputPath) @@ -230,6 +261,53 @@ + + ReleaseNotesHelper.parseReleaseNotes let useMsBuildToolchain = environVar "USE_MSBUILD" <> null -let dotnetSdkVersion = "2.1.403" +let dotnetSdkVersion = "2.2.105" let sdkPath = lazy DotNetCli.InstallDotNetSDK dotnetSdkVersion let getSdkPath() = sdkPath.Value diff --git a/src/ProvidedTypes.fs b/src/ProvidedTypes.fs index cc66f353..cb33f4e7 100644 --- a/src/ProvidedTypes.fs +++ b/src/ProvidedTypes.fs @@ -312,6 +312,7 @@ namespace ProviderImplementation.ProvidedTypes let qTy = typeof.Assembly.GetType("Microsoft.FSharp.Quotations.ExprConstInfo") assert (not (isNull qTy)) + let pTy = typeof.Assembly.GetType("Microsoft.FSharp.Quotations.PatternsModule") assert (not (isNull pTy)) @@ -319,12 +320,16 @@ namespace ProviderImplementation.ProvidedTypes // these function names have been stable since F# 2.0. let mkFE0 = pTy.GetMethod("mkFE0", bindAll) assert (not (isNull mkFE0)) + let mkFE1 = pTy.GetMethod("mkFE1", bindAll) assert (not (isNull mkFE1)) + let mkFE2 = pTy.GetMethod("mkFE2", bindAll) assert (mkFE2 |> isNull |> not) + let mkFE3 = pTy.GetMethod("mkFE3", bindAll) assert (mkFE3 |> isNull |> not) + let mkFEN = pTy.GetMethod("mkFEN", bindAll) assert (mkFEN |> isNull |> not) @@ -332,44 +337,64 @@ namespace ProviderImplementation.ProvidedTypes // these function names have been stable since F# 2.0. let newDelegateOp = qTy.GetMethod("NewNewDelegateOp", bindAll) assert (newDelegateOp |> isNull |> not) + let instanceCallOp = qTy.GetMethod("NewInstanceMethodCallOp", bindAll) assert (instanceCallOp |> isNull |> not) + let staticCallOp = qTy.GetMethod("NewStaticMethodCallOp", bindAll) assert (staticCallOp |> isNull |> not) + let newObjectOp = qTy.GetMethod("NewNewObjectOp", bindAll) assert (newObjectOp |> isNull |> not) + let newArrayOp = qTy.GetMethod("NewNewArrayOp", bindAll) assert (newArrayOp |> isNull |> not) + let appOp = qTy.GetMethod("get_AppOp", bindAll) assert (appOp |> isNull |> not) + let instancePropGetOp = qTy.GetMethod("NewInstancePropGetOp", bindAll) assert (instancePropGetOp |> isNull |> not) + let staticPropGetOp = qTy.GetMethod("NewStaticPropGetOp", bindAll) assert (staticPropGetOp |> isNull |> not) + let instancePropSetOp = qTy.GetMethod("NewInstancePropSetOp", bindAll) assert (instancePropSetOp |> isNull |> not) + let staticPropSetOp = qTy.GetMethod("NewStaticPropSetOp", bindAll) assert (staticPropSetOp |> isNull |> not) + let instanceFieldGetOp = qTy.GetMethod("NewInstanceFieldGetOp", bindAll) assert (instanceFieldGetOp |> isNull |> not) + let staticFieldGetOp = qTy.GetMethod("NewStaticFieldGetOp", bindAll) assert (staticFieldGetOp |> isNull |> not) + let instanceFieldSetOp = qTy.GetMethod("NewInstanceFieldSetOp", bindAll) assert (instanceFieldSetOp |> isNull |> not) + let staticFieldSetOp = qTy.GetMethod("NewStaticFieldSetOp", bindAll) assert (staticFieldSetOp |> isNull |> not) + let tupleGetOp = qTy.GetMethod("NewTupleGetOp", bindAll) assert (tupleGetOp |> isNull |> not) + let letOp = qTy.GetMethod("get_LetOp", bindAll) assert (letOp |> isNull |> not) + let forIntegerRangeLoopOp = qTy.GetMethod("get_ForIntegerRangeLoopOp", bindAll) assert (forIntegerRangeLoopOp |> isNull |> not) + let whileLoopOp = qTy.GetMethod("get_WhileLoopOp", bindAll) assert (whileLoopOp |> isNull |> not) + let ifThenElseOp = qTy.GetMethod("get_IfThenElseOp", bindAll) assert (ifThenElseOp |> isNull |> not) + let newUnionCaseOp = qTy.GetMethod("NewNewUnionCaseOp", bindAll) assert (newUnionCaseOp |> isNull |> not) + let newRecordOp = qTy.GetMethod("NewNewRecordOp", bindAll) assert (newRecordOp |> isNull |> not) @@ -460,11 +485,11 @@ namespace ProviderImplementation.ProvidedTypes static member NewUnionCaseUnchecked (uci:Reflection.UnionCaseInfo, args:Expr list) = let op = newUnionCaseOp.Invoke(null, [| box uci |]) mkFEN.Invoke(null, [| box op; box args |]) :?> Expr - - static member NewRecordUnchecked (ty:Type, args:Expr list) = + + static member NewRecordUnchecked (ty:Type, args:Expr list) = let op = newRecordOp.Invoke(null, [| box ty |]) mkFEN.Invoke(null, [| box op; box args |]) :?> Expr - + type Shape = Shape of (Expr list -> Expr) let (|ShapeCombinationUnchecked|ShapeVarUnchecked|ShapeLambdaUnchecked|) e = @@ -6628,8 +6653,11 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader with err -> failwithf "FAILED decodeILCustomAttribData, data.Length = %d, data = %A, meth = %A, argtypes = %A, fixedArgs=%A, nnamed = %A, sigptr before named = %A, innerError = %A" bytes.Length bytes ca.Method.EnclosingType ca.Method.FormalArgTypes fixedArgs nnamed sigptr (err.ToString()) - // Share DLLs across providers by caching them - let readerCache = ConcurrentDictionary<(string * string), DateTime * int * ILModuleReader>() + // Share DLLs within a provider by weak-caching them. + let readerWeakCache = ConcurrentDictionary<(string * string), DateTime * WeakReference>(HashIdentity.Structural) + + // Share DLLs across providers by strong-caching them, but flushing regularly + let readerStrongCache = ConcurrentDictionary<(string * string), DateTime * int * ILModuleReader>(HashIdentity.Structural) type File with static member ReadBinaryChunk (fileName: string, start, len) = @@ -6650,23 +6678,66 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader let reader = ILModuleReader(fileName, mdfile, ilGlobals, true) reader - let GetReaderCache () = ReadOnlyDictionary(readerCache) + let GetWeakReaderCache () = readerWeakCache + let GetStrongReaderCache () = readerStrongCache + + // Auto-clear the cache every 30.0 seconds. + // We would use System.Runtime.Caching but some version constraints make this difficult. + let enableAutoClear = try Environment.GetEnvironmentVariable("FSHARP_TPREADER_AUTOCLEAR_OFF") = null with _ -> true + let clearSpanDefault = 30000 + let clearSpan = try (match Environment.GetEnvironmentVariable("FSHARP_TPREADER_AUTOCLEAR_SPAN") with null -> clearSpanDefault | s -> int32 s) with _ -> clearSpanDefault + let lastAccessLock = obj() + let mutable lastAccess = DateTime.Now + + let StartClearReaderCache() = + if enableAutoClear then + async { + while true do + do! Async.Sleep clearSpan + let timeSinceLastAccess = DateTime.Now - lock lastAccessLock (fun () -> lastAccess) + if timeSinceLastAccess > TimeSpan.FromMilliseconds(float clearSpan) then + readerStrongCache.Clear() + } + |> Async.Start + + do StartClearReaderCache() + + let (|WeakReference|_|) (x: WeakReference<'T>) = + match x.TryGetTarget() with + | true, v -> Some v + | _ -> None let ILModuleReaderAfterReadingAllBytes (file:string, ilGlobals: ILGlobals) = let key = (file, ilGlobals.systemRuntimeScopeRef.QualifiedName) - let add _ = - let lastWriteTime = File.GetLastWriteTime(file) - let reader = createReader ilGlobals file - (lastWriteTime, 1, reader) - let update _ (currentLastWriteTime, count, reader) = - let lastWriteTime = File.GetLastWriteTime(file) - if currentLastWriteTime <> lastWriteTime then + lock lastAccessLock (fun () -> lastAccess <- DateTime.Now) + + // Check the weak cache, to enable sharing within a provider, even if the strong cache is flushed. + match readerWeakCache.TryGetValue(key) with + | true, (currentLastWriteTime, WeakReference(reader)) when + let lastWriteTime = File.GetLastWriteTime(file) + currentLastWriteTime = lastWriteTime -> + + reader + + | _ -> + let add _ = + let lastWriteTime = File.GetLastWriteTime(file) let reader = createReader ilGlobals file - (lastWriteTime, count + 1, reader) - else - (lastWriteTime, count, reader) - let _, _, reader = readerCache.AddOrUpdate(key, add, update) - reader + // record in the weak cache, to enable sharing within a provider, even if the strong cache is flushed. + readerWeakCache.[key] <- (lastWriteTime, WeakReference<_>(reader)) + (lastWriteTime, 1, reader) + + let update _ (currentLastWriteTime, count, reader) = + let lastWriteTime = File.GetLastWriteTime(file) + if currentLastWriteTime <> lastWriteTime then + let reader = createReader ilGlobals file + // record in the weak cache, to enable sharing within a provider, even if the strong cache is flushed. + readerWeakCache.[key] <- (lastWriteTime, WeakReference<_>(reader)) + (lastWriteTime, count + 1, reader) + else + (lastWriteTime, count, reader) + let _, _, reader = readerStrongCache.AddOrUpdate(key, add, update) + reader (* NOTE: ecma_ prefix refers to the standard "mscorlib" *) let EcmaPublicKey = PublicKeyToken ([|0xdeuy; 0xaduy; 0xbeuy; 0xefuy; 0xcauy; 0xfeuy; 0xfauy; 0xceuy |]) @@ -14576,7 +14647,6 @@ namespace ProviderImplementation.ProvidedTypes let ctxt = ProvidedTypesContext.Create (config, assemblyReplacementMap, sourceAssemblies) - #if !NO_GENERATIVE let theTable = ConcurrentDictionary() @@ -14701,11 +14771,14 @@ namespace ProviderImplementation.ProvidedTypes AppDomain.CurrentDomain.remove_AssemblyResolve handler #endif - member __.AddNamespace (namespaceName, types) = namespacesT.Add (makeProvidedNamespace namespaceName types) + member __.AddNamespace (namespaceName, types) = + namespacesT.Add (makeProvidedNamespace namespaceName types) - member __.Namespaces = namespacesT.ToArray() + member __.Namespaces = + namespacesT.ToArray() - member this.Invalidate() = invalidateE.Trigger(this,EventArgs()) + member this.Invalidate() = + invalidateE.Trigger(this,EventArgs()) member __.GetStaticParametersForMethod(mb: MethodBase) = match mb with diff --git a/src/ProvidedTypes.fsi b/src/ProvidedTypes.fsi index 295ccd47..20fffed5 100644 --- a/src/ProvidedTypes.fsi +++ b/src/ProvidedTypes.fsi @@ -20,7 +20,8 @@ module internal Reader = type ILModuleReader = class end - val GetReaderCache : unit -> ReadOnlyDictionary<(string * string), DateTime * int * ILModuleReader> + val GetWeakReaderCache : unit -> System.Collections.Concurrent.ConcurrentDictionary<(string * string), DateTime * WeakReference> + val GetStrongReaderCache : unit -> System.Collections.Concurrent.ConcurrentDictionary<(string * string), DateTime * int * ILModuleReader> #endif diff --git a/tests/AssemblyReaderTests.fs b/tests/AssemblyReaderTests.fs index 9a16f6c2..bf6779cf 100644 --- a/tests/AssemblyReaderTests.fs +++ b/tests/AssemblyReaderTests.fs @@ -3,7 +3,7 @@ //#load "../src/AssemblyReaderReflection.fs" (strangely fails to bind) #else -module FSharp.TypeProviders.SDK.Tests.AssemblyReaderTests +module TPSDK.AssemblyReaderTests #endif open System diff --git a/tests/BasicErasedProvisionTests.fs b/tests/BasicErasedProvisionTests.fs index 344e1a89..78e0f594 100644 --- a/tests/BasicErasedProvisionTests.fs +++ b/tests/BasicErasedProvisionTests.fs @@ -4,7 +4,7 @@ #else -module FSharp.TypeProviders.SDK.Tests.StaticProperty +module TPSDK.BasicErasedTests #endif open System @@ -545,9 +545,7 @@ let ``test basic symbol type ops``() = #if INTERNAL_FSHARP_TYPEPROVIDERS_SDK_TESTS -[] -let ``test reader cache actually caches``() = - for i = 1 to 1000 do +let stressTestCore() = let refs = Targets.DotNet45FSharp40Refs() let config = Testing.MakeSimulatedTypeProviderConfig (resolutionFolder=__SOURCE_DIRECTORY__, runtimeAssembly="whatever.dll", runtimeAssemblyRefs=refs) use tp = new TypeProviderForNamespaces(config) @@ -567,16 +565,50 @@ let ``test reader cache actually caches``() = match t1T with :? ProvidedTypeSymbol as st -> Assert.True(st.IsFSharpUnitAnnotated) | _ -> failwith "expected a ProvidedTypeSymbol#4" let _ = ProvidedTypeBuilder.MakeTupleType([ t1; t1 ]) - () - - let dict = AssemblyReader.Reader.GetReaderCache() - Assert.True(dict.Count > 0, "Reader Cache has not count") - dict - |> Seq.iter (fun pair -> - let _, count, _ = pair.Value - Assert.False(count > 500, "Too many instances of an assembly") + tp + +let stressTestLoop() = + let mutable latestTp = None + let weakDict = AssemblyReader.Reader.GetWeakReaderCache() + let strongDict = AssemblyReader.Reader.GetStrongReaderCache() + weakDict.Clear() + strongDict.Clear() + + for i = 1 to 1000 do + latestTp <- Some (stressTestCore()) + + Assert.True(weakDict.Count > 0, "Weak Reader Cache has zero count") + Assert.True(strongDict.Count > 0, "Strong Reader Cache has zero count") + + // We created 1000 TP instances rapidly but we should not be re-creating readers. + strongDict + |> Seq.iter (fun (KeyValue(_, (_, count, _))) -> + Assert.False(count > 5, "Too many instances of an assembly") ) + // After we are done the weak handles should still be populated as we have a handle to the last TP + System.GC.Collect() + for (KeyValue(_, (_, wh))) in weakDict do + let alive = fst(wh.TryGetTarget()) + Assert.True(alive, "Weak handle should still be populated as latest TP still alive") + + latestTp <- None + strongDict.Clear() + +[] +let ``test reader cache actually caches``() = + let weakDict = AssemblyReader.Reader.GetWeakReaderCache() + // We factor this test into another ethod to ensure that things get collecte properly on all version of .NET + // i.e. that the stack frame isn't keeping any strong handles to anything. + stressTestLoop() + System.GC.Collect (2, GCCollectionMode.Forced, true, true) + System.GC.WaitForPendingFinalizers() + System.GC.Collect (2, GCCollectionMode.Forced, true, true) + let runningOnMono = try Type.GetType("Mono.Runtime") <> null with _ -> false + if not runningOnMono then + for (KeyValue(key, (_, wh))) in weakDict do + let alive = fst(wh.TryGetTarget()) + Assert.False(alive, sprintf "Weak handle for %A should no longer be populated as latest TP no longer alive" key) #endif [] diff --git a/tests/BasicGenerativeProvisionTests.fs b/tests/BasicGenerativeProvisionTests.fs index d831e96c..d8bff597 100644 --- a/tests/BasicGenerativeProvisionTests.fs +++ b/tests/BasicGenerativeProvisionTests.fs @@ -3,7 +3,7 @@ #load "../src/ProvidedTypesTesting.fs" #else -module FSharp.TypeProviders.SDK.Tests.BasicGenerativeTests +module TPSDK.Tests.BasicGenerativeTests #endif open System diff --git a/tests/FSharp.TypeProviders.SDK.Tests.fsproj b/tests/FSharp.TypeProviders.SDK.Tests.fsproj index d1bea949..d39c317c 100644 --- a/tests/FSharp.TypeProviders.SDK.Tests.fsproj +++ b/tests/FSharp.TypeProviders.SDK.Tests.fsproj @@ -3,12 +3,7 @@ netcoreapp2.0;net461 false - - - TRACE;INTERNAL_FSHARP_TYPEPROVIDERS_SDK_TESTS - - - TRACE;INTERNAL_FSHARP_TYPEPROVIDERS_SDK_TESTS + $(DefineConstants);INTERNAL_FSHARP_TYPEPROVIDERS_SDK_TESTS diff --git a/tests/GenerativeEnumsProvisionTests.fs b/tests/GenerativeEnumsProvisionTests.fs index 27300f35..f298d2ad 100644 --- a/tests/GenerativeEnumsProvisionTests.fs +++ b/tests/GenerativeEnumsProvisionTests.fs @@ -4,7 +4,7 @@ #else -module FSharp.TypeProviders.SDK.Tests.GenerativeEnumsProvisionTests +module TPSDK.GenerativeEnumsProvisionTests #endif #nowarn "760" // IDisposable needs new diff --git a/tests/Program.fs b/tests/Program.fs index 2ad94bd4..f297a55c 100644 --- a/tests/Program.fs +++ b/tests/Program.fs @@ -1,2 +1,4 @@ module Program -let [] main _ = 0 +let [] main _ = + //TPSDK.BasicErasedTests.``test reader cache actually caches``() + 0 \ No newline at end of file