Add directory deletion retry in test teardown (#398)

Co-authored-by: Badrish Chandramouli <badrishc@microsoft.com>

cs/test/AsyncLargeObjectTests.cs

@@ -2,11 +2,7 @@
 // Licensed under the MIT license.
 
 using System;
-using System.Text;
-using System.Threading;
 using System.Threading.Tasks;
-using System.Collections.Generic;
-using System.Linq;
 using FASTER.core;
 using System.IO;
 using NUnit.Framework;
@@ -37,7 +33,7 @@ public void Setup()
         [TearDown]
         public void TearDown()
         {
-            Directory.Delete(test_path, true);
+            TestUtils.DeleteDirectory(test_path);
         }
 
         [TestCase(CheckpointType.FoldOver)]

cs/test/FasterLogRecoverReadOnlyTests.cs

@@ -29,14 +29,14 @@ public void Setup()
             path = Path.GetTempPath() + "RecoverReadOnlyTest/";
             deviceName = path + "testlog";
             if (Directory.Exists(path))
-                Directory.Delete(path, true);
+                TestUtils.DeleteDirectory(path);
             cts = new CancellationTokenSource();
         }
 
         [TearDown]
         public void TearDown()
         {
-            Directory.Delete(path, true);
+            TestUtils.DeleteDirectory(path);
             cts.Dispose();
         }
 

cs/test/FasterLogResumeTests.cs

@@ -22,7 +22,7 @@ public void Setup()
             commitPath = TestContext.CurrentContext.TestDirectory + "/" + TestContext.CurrentContext.Test.Name + "/";
 
             if (Directory.Exists(commitPath))
-                Directory.Delete(commitPath, true);
+                TestUtils.DeleteDirectory(commitPath);
 
             device = Devices.CreateLogDevice(commitPath + "fasterlog.log", deleteOnClose: true);
         }
@@ -33,7 +33,7 @@ public void TearDown()
             device.Dispose();
 
             if (Directory.Exists(commitPath))
-                Directory.Delete(commitPath, true);
+                TestUtils.DeleteDirectory(commitPath);
         }
 
         [Test]

cs/test/LargeObjectTests.cs

@@ -36,7 +36,7 @@ public void Setup()
         [TearDown]
         public void TearDown()
         {
-            Directory.Delete(test_path, true);
+            TestUtils.DeleteDirectory(test_path);
         }
 
         [TestCase(CheckpointType.FoldOver)]

cs/test/ObjectRecoveryTest.cs

@@ -57,7 +57,7 @@ public void TearDown()
             fht = null;
             log.Dispose();
             objlog.Dispose();
-            Directory.Delete(test_path, true);
+            TestUtils.DeleteDirectory(test_path);
         }
 
         [Test]

cs/test/ObjectRecoveryTest2.cs

@@ -26,7 +26,7 @@ public void Setup()
         [TearDown]
         public void TearDown()
         {
-            Directory.Delete(FasterFolderPath, true);
+            TestUtils.DeleteDirectory(FasterFolderPath);
         }
 
         [Test]

cs/test/ObjectRecoveryTest3.cs

@@ -29,7 +29,7 @@ public void Setup()
         [TearDown]
         public void TearDown()
         {
-            Directory.Delete(FasterFolderPath, true);
+            TestUtils.DeleteDirectory(FasterFolderPath);
         }
 
         [Test]

cs/test/RecoverContinueTests.cs

@@ -22,32 +22,33 @@ internal class RecoverContinueTests
         private FasterKV<AdId, NumClicks> fht3;
         private IDevice log;
         private int numOps;
+        private string checkpointDir = TestContext.CurrentContext.TestDirectory + "/checkpoints3";
 
         [SetUp]
         public void Setup()
         {
             log = Devices.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "/RecoverContinueTests.log", deleteOnClose: true);
-            Directory.CreateDirectory(TestContext.CurrentContext.TestDirectory + "/checkpoints3");
+            Directory.CreateDirectory(checkpointDir);
 
             fht1 = new FasterKV
                 <AdId, NumClicks>
                 (128,
                 logSettings: new LogSettings { LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29 },
-                checkpointSettings: new CheckpointSettings { CheckpointDir = TestContext.CurrentContext.TestDirectory + "/checkpoints3", CheckPointType = CheckpointType.Snapshot }
+                checkpointSettings: new CheckpointSettings { CheckpointDir = checkpointDir, CheckPointType = CheckpointType.Snapshot }
                 );
 
             fht2 = new FasterKV
                 <AdId, NumClicks>
                 (128,
                 logSettings: new LogSettings { LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29 },
-                checkpointSettings: new CheckpointSettings { CheckpointDir = TestContext.CurrentContext.TestDirectory + "/checkpoints3", CheckPointType = CheckpointType.Snapshot }
+                checkpointSettings: new CheckpointSettings { CheckpointDir = checkpointDir, CheckPointType = CheckpointType.Snapshot }
                 );
 
             fht3 = new FasterKV
                 <AdId, NumClicks>
                 (128,
                 logSettings: new LogSettings { LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29 },
-                checkpointSettings: new CheckpointSettings { CheckpointDir = TestContext.CurrentContext.TestDirectory + "/checkpoints3", CheckPointType = CheckpointType.Snapshot }
+                checkpointSettings: new CheckpointSettings { CheckpointDir = checkpointDir, CheckPointType = CheckpointType.Snapshot }
                 );
 
             numOps = 5000;
@@ -63,7 +64,7 @@ public void TearDown()
             fht2 = null;
             fht3 = null;
             log.Dispose();
-            Directory.Delete(TestContext.CurrentContext.TestDirectory + "/checkpoints3", true);
+            TestUtils.DeleteDirectory(checkpointDir);
         }
 
         [Test]

cs/test/RecoveryTests.cs

@@ -49,7 +49,7 @@ public void TearDown()
             fht.Dispose();
             fht = null;
             log.Dispose();
-            Directory.Delete(test_path, true);
+            TestUtils.DeleteDirectory(test_path);
         }
 
         [Test]

cs/test/SharedDirectoryTests.cs

@@ -45,7 +45,7 @@ public void TearDown()
             this.clone.TearDown();
             try
             {
-                Directory.Delete(this.rootPath, recursive: true);
+                TestUtils.DeleteDirectory(this.rootPath);
             }
             catch
             {

cs/test/TestUtils.cs

@@ -0,0 +1,34 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+using System;
+using System.IO;
+
+namespace FASTER.test
+{
+    internal static class TestUtils
+    {
+        internal static void DeleteDirectory(string path)
+        {
+            foreach (string directory in Directory.GetDirectories(path))
+            {
+                DeleteDirectory(directory);
+            }
+
+            // Exceptions may happen due to a handle briefly remaining held after Dispose().
+            try
+            {
+                Directory.Delete(path, true);
+            }
+            catch (Exception ex) when (ex is IOException ||
+                                       ex is UnauthorizedAccessException)
+            {
+                try
+                {
+                    Directory.Delete(path, true);
+                }
+                catch { }
+            }
+        }
+    }
+}