Merge pull request #1363: Handle corrupt objects on Mac

Handle corrupt objects on Mac

GVFS/GVFS.FunctionalTests/Categories.cs

@@ -29,9 +29,6 @@ public static class MacTODO
             // Tests requires code updates so that we lock the file instead of looking for a .lock file
             public const string TestNeedsToLockFile = "TestNeedsToLockFile";
 
-            // Corrupt Objects are not redownloaded
-            public const string NeedsCorruptObjectFix = "NeedsCorruptObjectFix";
-
             // Tests that have been flaky on build servers and need additional logging and\or
             // investigation
             public const string FlakyTest = "MacFlakyTest";

GVFS/GVFS.FunctionalTests/Program.cs

@@ -90,7 +90,6 @@ public static void Main(string[] args)
                 excludeCategories.Add(Categories.MacTODO.NeedsDehydrate);
                 excludeCategories.Add(Categories.MacTODO.NeedsServiceVerb);
                 excludeCategories.Add(Categories.MacTODO.NeedsStatusCache);
-                excludeCategories.Add(Categories.MacTODO.NeedsCorruptObjectFix);
                 excludeCategories.Add(Categories.MacTODO.TestNeedsToLockFile);
                 excludeCategories.Add(Categories.WindowsOnly);
             }

GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs

@@ -498,9 +498,7 @@ public void AllNullObjectRedownloaded()
             (badObject as FileInfo).ShouldNotBeNull().Length.ShouldEqual(objectFileInfo.Length);
         }
 
-        // TODO(#1218): Figure out why git for Mac is not requesting a redownload of the truncated object
         [TestCase, Order(17)]
-        [Category(Categories.MacTODO.NeedsCorruptObjectFix)]
         public void TruncatedObjectRedownloaded()
         {
             GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout " + this.Enlistment.Commitish);
@@ -521,6 +519,8 @@ public void TruncatedObjectRedownloaded()
             GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "rev-parse :Test_EPF_WorkingDirectoryTests/TruncatedObjectRedownloaded_copy.txt").Output.Trim().ShouldEqual(sha);
             string testFileContents = this.Enlistment.GetVirtualPathTo("Test_EPF_WorkingDirectoryTests", "TruncatedObjectRedownloaded_copy.txt").ShouldBeAFile(this.fileSystem).WithContents();
             objectPath.ShouldBeAFile(this.fileSystem);
+            string modifedFile = "Test_EPF_WorkingDirectoryTests/TruncatedObjectRedownloaded.txt";
+            GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, modifedFile);
 
             // Truncate the contents of objectPath
             string tempTruncatedObjectPath = objectPath + "truncated";
@@ -541,11 +541,29 @@ public void TruncatedObjectRedownloaded()
             objectPath.ShouldBeAFile(this.fileSystem);
             new FileInfo(objectPath).Length.ShouldEqual(objectLength - 16);
 
-            // Read the original path and verify its contents are correct
-            this.Enlistment.GetVirtualPathTo("Test_EPF_WorkingDirectoryTests", "TruncatedObjectRedownloaded.txt").ShouldBeAFile(this.fileSystem).WithContents(testFileContents);
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+            {
+                // Mac can't correct corrupt objects, but it should detect them and add to ModifiedPaths.dat
+                this.Enlistment.GetVirtualPathTo("Test_EPF_WorkingDirectoryTests", "TruncatedObjectRedownloaded.txt").ShouldBeAFile(this.fileSystem);
+
+                GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, modifedFile);
+                GitHelpers.CheckGitCommandAgainstGVFSRepo(
+                    this.Enlistment.RepoRoot,
+                    "status",
+                    $"modified:   {modifedFile}");
+            }
+            else
+            {
+                // Windows should correct a corrupt obect
+                // Read the original path and verify its contents are correct
+                this.Enlistment.GetVirtualPathTo("Test_EPF_WorkingDirectoryTests", "TruncatedObjectRedownloaded.txt").ShouldBeAFile(this.fileSystem).WithContents(testFileContents);
 
-            // Confirm there's a new item in the corrupt objects folder
-            corruptObjectFolderPath.ShouldBeADirectory(this.fileSystem).WithItems().Count().ShouldEqual(initialCorruptObjectCount + 1);
+                // Confirm there's a new item in the corrupt objects folder
+                corruptObjectFolderPath.ShouldBeADirectory(this.fileSystem).WithItems().Count().ShouldEqual(initialCorruptObjectCount + 1);
+
+                // File should not be in ModifiedPaths.dat
+                GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, "Test_EPF_WorkingDirectoryTests/TruncatedObjectRedownloaded.txt");
+            }
         }
 
         [TestCase, Order(18)]

GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs

@@ -386,6 +386,7 @@ private Result OnGetFileStream(
                             byte[] buffer = new byte[4096];
                             uint bufferIndex = 0;
                             int nextByte = stream.ReadByte();
+                            int bytesWritten = 0;
                             while (nextByte != -1)
                             {
                                 while (bufferIndex < buffer.Length && nextByte != -1)
@@ -408,8 +409,19 @@ private Result OnGetFileStream(
                                 if (bufferIndex == buffer.Length)
                                 {
                                     bufferIndex = 0;
+                                    bytesWritten += buffer.Length;
                                 }
                             }
+                            bytesWritten += Convert.ToInt32(bufferIndex);
+
+                            if (bytesWritten != blobLength)
+                            {
+                                // If the read size does not match the expected size print an error and add the file to ModifiedPaths.dat
+                                // This allows the user to see that something went wrong with file hydration
+                                // Unfortunitely we must do this check *after* the file is hydrated since the header isn't corrupt for trunctated objects on mac
+                                this.Context.Tracer.RelatedError($"Read {relativePath} to {bytesWritten}, not expected size of {blobLength}");
+                                this.FileSystemCallbacks.OnFailedFileHydration(relativePath);
+                            }
                         }))
                     {
                         activity.RelatedError(metadata, $"{nameof(this.OnGetFileStream)}: TryCopyBlobContentStream failed");

GVFS/GVFS.Virtualization/Background/FileSystemTask.cs

@@ -33,6 +33,7 @@ public enum OperationType
             OnFilePreDelete,
             OnFolderPreDelete,
             OnFileSymLinkCreated,
+            OnFailedFileHydration,
         }
 
         public OperationType Operation { get; }
@@ -95,6 +96,11 @@ public static FileSystemTask OnFailedPlaceholderUpdate(string virtualPath)
             return new FileSystemTask(OperationType.OnFailedPlaceholderUpdate, virtualPath, oldVirtualPath: null);
         }
 
+        public static FileSystemTask OnFailedFileHydration(string virtualPath)
+        {
+            return new FileSystemTask(OperationType.OnFailedFileHydration, virtualPath, oldVirtualPath: null);
+        }
+
         public static FileSystemTask OnFolderCreated(string virtualPath)
         {
             return new FileSystemTask(OperationType.OnFolderCreated, virtualPath, oldVirtualPath: null);

GVFS/GVFS.Virtualization/FileSystemCallbacks.cs

@@ -383,6 +383,11 @@ public void OnFileConvertedToFull(string relativePath)
             this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileConvertedToFull(relativePath));
         }
 
+        public void OnFailedFileHydration(string relativePath)
+        {
+            this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFailedFileHydration(relativePath));
+        }
+
         public virtual void OnFileRenamed(string oldRelativePath, string newRelativePath)
         {
             this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileRenamed(oldRelativePath, newRelativePath));
@@ -687,6 +692,7 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate
                 case FileSystemTask.OperationType.OnFileSuperseded:
                 case FileSystemTask.OperationType.OnFileConvertedToFull:
                 case FileSystemTask.OperationType.OnFailedPlaceholderUpdate:
+                case FileSystemTask.OperationType.OnFailedFileHydration:
                     metadata.Add("virtualPath", gitUpdate.VirtualPath);
                     result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath);
                     break;