[C#] Clean up TargetFramework defines, enable IAsyncEnumerable on net…

…standard2.0/net461 (#377)

* Clean up targetframework stuff:
- Change the order in the csproj so netstandard2.1 is first, which appears to be the one that's used by the IDE for #if <TargetFramework> resolution
- Switch to the TargetFramework preprocessor symbols already defined by the build system (we use NETSTANDARD and NETSTANDARD2_1)
- Add Microsoft.Bcl.AsyncInterfaces for pre-netstandard2.1 frameworks, which allows IAsyncEnumerable to work
- Added tests for FasterLogIterator.GetAsyncEnumerable()
- Enable netcore2.2 in FASTER.tests.csproj
* Fix DeviceFasterLogTests hang with IAsyncEnumerator; add tests for FasterLogIterator.GetAsyncEnumerable(MemoryPool<T>)
* Update azure-pipelines.yml
Put UseDotNet@2 task in the right place
* Drop FASTER.test netcoreapp v2.2 to 2.1 (2.2 isn't needed and 2.1 is on the pipeline by default)
* Add VSTest2 pipeline task for Linux
* use DotNetCoreCLI@2 task to run Linux tests
* Specify framework version on C# Linux dotnetcorecli task
* Replace directory separator in tests with front slash (/) for Linux compatibility.
* Fixed SharedDirectoryTests to work on Linux
* change !NET461 to NETCOREAPP; restore previous LocalStorageDevice GetSectorSize behavior

Co-authored-by: Badrish Chandramouli <badrishc@microsoft.com>

azure-pipelines.yml

@@ -64,8 +64,20 @@ jobs:
   - task: VSTest@2
     inputs:
       testAssemblyVer2: |
-       **\$(BuildConfiguration)\netcoreapp*\*test.dll
-       **\$(BuildConfiguration)\**\netcoreapp*\*test.dll
+       **\$(BuildConfiguration)\netcoreapp2*\*test.dll
+       **\$(BuildConfiguration)\**\netcoreapp2*\*test.dll
+       !**\*Microsoft.VisualStudio.TestPlatform*
+       !**\obj\**
+       !**\*TestAdapter.dll
+      platform: '$(buildPlatform)'
+      configuration: '$(buildConfiguration)'
+      otherConsoleOptions: '/Framework:.NETCoreApp,Version=v2.2'
+
+  - task: VSTest@2
+    inputs:
+      testAssemblyVer2: |
+       **\$(BuildConfiguration)\netcoreapp3*\*test.dll
+       **\$(BuildConfiguration)\**\netcoreapp3*\*test.dll
        !**\*Microsoft.VisualStudio.TestPlatform*
        !**\obj\**
        !**\*TestAdapter.dll
@@ -169,7 +181,13 @@ jobs:
       msbuild /t:restore $(solution)
       msbuild /p:Configuration=Release $(solution)
     displayName: 'Build'
-    
+
+  - task: DotNetCoreCLI@2
+    inputs:
+      command: test
+      projects: '**/test/*.csproj'
+      arguments: '--configuration $(buildConfiguration) --framework netcoreapp3.1'
+
 # - job: 'cppBlobsWindows'
 #   pool:
 #     vmImage: vs2017-win2016

cs/benchmark/ConcurrentDictionaryBenchmark.cs

@@ -409,8 +409,8 @@ void DoContinuousMeasurements()
 
         private void LoadDataFromFile(string filePath)
         {
-            string init_filename = filePath + "\\load_" + distribution + "_250M_raw.dat";
-            string txn_filename = filePath + "\\run_" + distribution + "_250M_1000M_raw.dat";
+            string init_filename = filePath + "/load_" + distribution + "_250M_raw.dat";
+            string txn_filename = filePath + "/run_" + distribution + "_250M_1000M_raw.dat";
 
             long count = 0;
             using (FileStream stream = File.Open(init_filename, FileMode.Open, FileAccess.Read,

cs/benchmark/FasterSpanByteYcsbBenchmark.cs

@@ -506,8 +506,8 @@ void DoContinuousMeasurements()
 
         private unsafe void LoadDataFromFile(string filePath)
         {
-            string init_filename = filePath + "\\load_" + distribution + "_250M_raw.dat";
-            string txn_filename = filePath + "\\run_" + distribution + "_250M_1000M_raw.dat";
+            string init_filename = filePath + "/load_" + distribution + "_250M_raw.dat";
+            string txn_filename = filePath + "/run_" + distribution + "_250M_1000M_raw.dat";
 
             long count = 0;
             using (FileStream stream = File.Open(init_filename, FileMode.Open, FileAccess.Read,

cs/benchmark/FasterYcsbBenchmark.cs

@@ -498,8 +498,8 @@ void DoContinuousMeasurements()
 
         private unsafe void LoadDataFromFile(string filePath)
         {
-            string init_filename = filePath + "\\load_" + distribution + "_250M_raw.dat";
-            string txn_filename = filePath + "\\run_" + distribution + "_250M_1000M_raw.dat";
+            string init_filename = filePath + "/load_" + distribution + "_250M_raw.dat";
+            string txn_filename = filePath + "/run_" + distribution + "_250M_1000M_raw.dat";
 
             long count = 0;
             using (FileStream stream = File.Open(init_filename, FileMode.Open, FileAccess.Read,

cs/playground/CacheStoreConcurrent/Program.cs

@@ -34,7 +34,7 @@ static void Main()
             // Number of caches and number of threads can be varied.
 
             h = new FasterKV<CacheKey, CacheValue>[kNumTables];
-            var path = Path.GetTempPath() + "CacheStoreConcurrent\\";
+            var path = Path.GetTempPath() + "CacheStoreConcurrent/";
 
             for (int ht = 0; ht < kNumTables; ht++)
             {

cs/playground/SumStore/ConcurrencyTest.cs

@@ -26,7 +26,7 @@ public ConcurrencyTest(int threadCount)
             this.threadCount = threadCount;
 
             // Create FASTER index
-            var log = Devices.CreateLogDevice("logs\\hlog");
+            var log = Devices.CreateLogDevice("logs/hlog");
             fht = new FasterKV<AdId, NumClicks>
                 (keySpace, new LogSettings { LogDevice = log }, 
                 new CheckpointSettings { CheckpointDir = "logs" });

cs/playground/SumStore/RecoveryTest.cs

@@ -28,7 +28,7 @@ public RecoveryTest(int threadCount)
             this.threadCount = threadCount;
 
             // Create FASTER index
-            var log = Devices.CreateLogDevice("logs\\hlog");
+            var log = Devices.CreateLogDevice("logs/hlog");
             fht = new FasterKV<AdId, NumClicks>
                 (indexSize, new LogSettings { LogDevice = log },
                 new CheckpointSettings { CheckpointDir = "logs" });

cs/samples/CacheStore/Program.cs

@@ -22,7 +22,7 @@ static void Main()
             // Use blittable structs for *much* better performance
 
             // Create files for storing data
-            var path = Path.GetTempPath() + "ClassCache\\";
+            var path = Path.GetTempPath() + "ClassCache/";
             var log =  Devices.CreateLogDevice(path + "hlog.log");
 
             // Log for storing serialized objects; needed only for class keys/values

cs/samples/FasterLogPubSub/Program.cs

@@ -14,7 +14,7 @@ class Program
     {
         const int commitPeriodMs = 2000;
         const int restorePeriodMs = 1000;
-        static string path = Path.GetTempPath() + "FasterLogPubSub\\";
+        static string path = Path.GetTempPath() + "FasterLogPubSub/";
 
         static async Task Main()
         {

cs/samples/FasterLogSample/Program.cs

@@ -31,7 +31,7 @@ static void Main()
                 staticEntry[i] = (byte)i;
             }
 
-            var path = Path.GetTempPath() + "FasterLogSample\\";
+            var path = Path.GetTempPath() + "FasterLogSample/";
             IDevice device = Devices.CreateLogDevice(path + "hlog.log");
 
             // FasterLog will recover and resume if there is a previous commit found

cs/samples/HelloWorld/Program.cs

@@ -17,7 +17,7 @@ static void Main()
             // This is a simple in-memory sample of FASTER using value types
 
             // Create device for FASTER log
-            var path = Path.GetTempPath() + "HelloWorld\\";
+            var path = Path.GetTempPath() + "HelloWorld/";
             var log = Devices.CreateLogDevice(Path.GetTempPath() + "hlog.log");
 
             // Create store instance

cs/samples/SecondaryReaderStore/Program.cs

@@ -19,7 +19,7 @@ class Program
         static void Main()
         {
             // Create files for storing data
-            var path = Path.GetTempPath() + "SecondaryReaderStore\\";
+            var path = Path.GetTempPath() + "SecondaryReaderStore/";
             if (Directory.Exists(path))
                 new DirectoryInfo(path).Delete(true);
 

cs/samples/StoreAsyncApi/Program.cs

@@ -20,7 +20,7 @@ public class Program
         /// </summary>
         static void Main()
         {
-            var path = Path.GetTempPath() + "StoreAsyncApi\\";
+            var path = Path.GetTempPath() + "StoreAsyncApi/";
 
             // Since the example runs forever, we set option to auto-delete files on close
             // Note that this setting precludes recovery

cs/samples/StoreCheckpointRecover/Program.cs

@@ -14,7 +14,7 @@ static void Main()
             // We use classes in this example, replace with blittable structs for 
             // much better performance
 
-            var path = Path.GetTempPath() + "StoreCheckpointRecover\\";
+            var path = Path.GetTempPath() + "StoreCheckpointRecover/";
 
             // Main hybrid log device                        
             var log = Devices.CreateLogDevice(path + "hlog.log");

cs/samples/StoreCustomTypes/Program.cs

@@ -18,7 +18,7 @@ static void Main()
             // (2) Provide IFasterEqualityComparer<Key> instance as param to FASTER constructor
 
             // Main hybrid log device
-            var path = Path.GetTempPath() + "StoreCustomTypes\\";
+            var path = Path.GetTempPath() + "StoreCustomTypes/";
             var log = Devices.CreateLogDevice(path + "hlog.log");
 
             // With non-blittable types, you need an object log device in addition to the

cs/samples/StoreDiskReadBenchmark/Program.cs

@@ -31,7 +31,7 @@ public class Program
         /// </summary>
         static void Main()
         {
-            var path = Path.GetTempPath() + "FasterKVDiskReadBenchmark\\";
+            var path = Path.GetTempPath() + "FasterKVDiskReadBenchmark/";
             if (Directory.Exists(path))
                 new DirectoryInfo(path).Delete(true);
 

cs/src/core/Device/Devices.cs

@@ -28,7 +28,7 @@ public static IDevice CreateLogDevice(string logPath, bool preallocateFile = fal
         {
             IDevice logDevice;
 
-#if DOTNETCORE
+#if NETSTANDARD
             if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
             {
                 logDevice = new ManagedLocalStorageDevice(logPath, preallocateFile, deleteOnClose, capacity, recoverDevice);

cs/src/core/Device/ManagedLocalStorageDevice.cs

@@ -109,7 +109,7 @@ public override unsafe void ReadAsync(int segmentId, ulong sourceAddress,
             logReadHandle.Seek((long)sourceAddress, SeekOrigin.Begin);
             Interlocked.Increment(ref numPending);
 
-#if NETSTANDARD21
+#if NETSTANDARD2_1
             var umm = new UnmanagedMemoryManager<byte>((byte*)destinationAddress, (int)readLength);
             logReadHandle.ReadAsync(umm.Memory).AsTask()
 #else
@@ -121,16 +121,7 @@ public override unsafe void ReadAsync(int segmentId, ulong sourceAddress,
                     Interlocked.Decrement(ref numPending);
 
                     uint errorCode = 0;
-                    if (!t.IsFaulted)
-                    {
-#if !NETSTANDARD21
-                        fixed (void* source = memory.buffer)
-                        {
-                            Buffer.MemoryCopy(source, (void*)destinationAddress, readLength, readLength);
-                        }
-#endif
-                    }
-                    else
+                    if (t.IsFaulted)
                     {
                         if (t.Exception.InnerException is IOException)
                         {
@@ -142,12 +133,9 @@ public override unsafe void ReadAsync(int segmentId, ulong sourceAddress,
                             errorCode = uint.MaxValue;
                         }
                     }
-#if !NETSTANDARD21
-                    memory.Return();
-#endif
 
                     // Sequentialize all reads from same handle on non-windows
-#if DOTNETCORE
+#if NETSTANDARD
                     if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                     {
                         if (offset >= 0) streampool?.Return(offset);
@@ -158,7 +146,7 @@ public override unsafe void ReadAsync(int segmentId, ulong sourceAddress,
                 }
                 );
 
-#if DOTNETCORE
+#if NETSTANDARD
             if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                 if (offset >= 0) streampool?.Return(offset);
 #else
@@ -194,7 +182,7 @@ public override unsafe void WriteAsync(IntPtr sourceAddress,
             logWriteHandle.Seek((long)destinationAddress, SeekOrigin.Begin);
             Interlocked.Increment(ref numPending);
 
-#if NETSTANDARD21
+#if NETSTANDARD2_1
             var umm = new UnmanagedMemoryManager<byte>((byte*)sourceAddress, (int)numBytesToWrite);
             logWriteHandle.WriteAsync(umm.Memory).AsTask()
 #else
@@ -222,12 +210,9 @@ public override unsafe void WriteAsync(IntPtr sourceAddress,
                             errorCode = uint.MaxValue;
                         }
                     }
-#if !NETSTANDARD21
-                    memory.Return();
-#endif
 
                     // Sequentialize all writes to same handle on non-windows
-#if DOTNETCORE
+#if NETSTANDARD
                     if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                     {
                         ((FileStream)logWriteHandle).Flush(true);
@@ -239,7 +224,7 @@ public override unsafe void WriteAsync(IntPtr sourceAddress,
                 }
                 );
 
-#if DOTNETCORE
+#if NETSTANDARD
             if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                 if (offset >= 0) streampool?.Return(offset);
 #else
@@ -296,7 +281,7 @@ private string GetSegmentName(int segmentId)
 
         private static uint GetSectorSize(string filename)
         {
-#if DOTNETCORE
+#if NETSTANDARD
             if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
             {
                 Debug.WriteLine("Assuming 512 byte sector alignment for disk with file " + filename);

cs/src/core/FASTER.core.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>net461;netstandard2.0;netstandard2.1</TargetFrameworks>
+    <TargetFrameworks>netstandard2.1;netstandard2.0;net461</TargetFrameworks>
     <Platforms>AnyCPU;x64</Platforms>
     <LangVersion>8</LangVersion>
   </PropertyGroup>
@@ -32,14 +32,6 @@
     <OutputPath>bin\$(Platform)\Release\</OutputPath>
   </PropertyGroup>
 
-  <PropertyGroup Condition="'$(TargetFramework)' != 'net461'">
-    <DefineConstants>$(DefineConstants);DOTNETCORE</DefineConstants>
-  </PropertyGroup>
-
-  <PropertyGroup Condition="'$(TargetFramework)' == 'netstandard2.1'">
-    <DefineConstants>$(DefineConstants);NETSTANDARD21</DefineConstants>
-  </PropertyGroup>
-
   <ItemGroup>
     <PackageReference Include="System.Runtime.CompilerServices.Unsafe" Version="4.7.1" />
   </ItemGroup>
@@ -51,6 +43,7 @@
   <ItemGroup Condition="'$(TargetFramework)' != 'netstandard2.1'">
     <PackageReference Include="System.Memory" Version="4.5.4" />
     <PackageReference Include="System.Threading.Tasks.Extensions" Version="4.5.4" />
+    <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="5.0.0" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(TargetFramework)' != 'net461'">

cs/src/core/Index/FasterLog/FasterLogIterator.cs

@@ -95,7 +95,6 @@ internal unsafe FasterLogScanIterator(FasterLog fasterLog, BlittableAllocator<Em
             }
         }
 
-#if DOTNETCORE
         /// <summary>
         /// Async enumerable for iterator
         /// </summary>
@@ -123,7 +122,7 @@ internal unsafe FasterLogScanIterator(FasterLog fasterLog, BlittableAllocator<Em
         /// Async enumerable for iterator (memory pool based version)
         /// </summary>
         /// <returns>Entry, actual entry length, logical address of entry, logical address of next entry</returns>
-        public async IAsyncEnumerable<(IMemoryOwner<byte>, int entryLength, long currentAddress, long nextAddress)> GetAsyncEnumerable(MemoryPool<byte> pool, [EnumeratorCancellation] CancellationToken token = default)
+        public async IAsyncEnumerable<(IMemoryOwner<byte> entry, int entryLength, long currentAddress, long nextAddress)> GetAsyncEnumerable(MemoryPool<byte> pool, [EnumeratorCancellation] CancellationToken token = default)
         {
             while (!disposed)
             {
@@ -141,7 +140,6 @@ internal unsafe FasterLogScanIterator(FasterLog fasterLog, BlittableAllocator<Em
                 yield return (result, length, currentAddress, nextAddress);
             }
         }
-#endif
 
         /// <summary>
         /// Wait for iteration to be ready to continue

cs/src/core/Index/Interfaces/ICompactionFunctions.cs

@@ -43,7 +43,7 @@ public interface ICompactionFunctions<Key, Value>
         /// <param name="valueLength">[Can be null] Variable length struct functions</param>
         /// <returns></returns>
         void Copy(ref Value src, ref Value dst, IVariableLengthStruct<Value> valueLength)
-#if NETSTANDARD21
+#if NETSTANDARD2_1
             { }
 #else
             ;
@@ -64,7 +64,7 @@ void Copy(ref Value src, ref Value dst, IVariableLengthStruct<Value> valueLength
         /// managed pointer to new record.
         /// </returns>
         bool CopyInPlace(ref Value src, ref Value dst, IVariableLengthStruct<Value> valueLength)
-#if NETSTANDARD21
+#if NETSTANDARD2_1
             => true
 #endif
             ;

cs/src/core/Index/Interfaces/IFunctions.cs

@@ -70,7 +70,7 @@ public interface IFunctions<Key, Value, Input, Output, Context>
         /// <param name="input"></param>
         /// <param name="oldValue"></param>
         bool NeedCopyUpdate(ref Key key, ref Input input, ref Value oldValue)
-#if NETSTANDARD21            
+#if NETSTANDARD2_1
             => true
 #endif
             ;

cs/src/core/Utilities/Native32.cs

@@ -272,7 +272,7 @@ public static void AffinitizeThreadShardedNuma(uint threadIdx, ushort nrOfProces
         /// <returns></returns>
         public static bool EnableProcessPrivileges()
         {
-#if DOTNETCORE
+#if NETSTANDARD
             if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                 return false;
 #endif
@@ -308,7 +308,7 @@ private static uint CTL_CODE(uint DeviceType, uint Function, uint Method, uint A
 
         internal static bool EnableVolumePrivileges(string filename, SafeFileHandle handle)
         {
-#if DOTNETCORE
+#if NETSTANDARD
             if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                 return false;
 #endif
@@ -346,7 +346,7 @@ internal static bool EnableVolumePrivileges(string filename, SafeFileHandle hand
         /// <returns></returns>
         public static bool SetFileSize(SafeFileHandle file_handle, long file_size)
         {
-#if DOTNETCORE
+#if NETSTANDARD
             if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                 return false;
 #endif