diff --git a/Directory.Build.props b/Directory.Build.props
index f951fb564e..5650be6635 100644
--- a/Directory.Build.props
+++ b/Directory.Build.props
@@ -36,7 +36,7 @@
0.2.173.2
- 2.20191211.2
+ 2.20191217.2
https://github.com/facebook/watchman/suites/307436006/artifacts/304557
https://github.com/microsoft/Git-Credential-Manager-Core/releases/download/v2.0.79-beta/gcmcore-osx-2.0.79.64449.pkg
diff --git a/Scalar.Common/Git/GitObjects.cs b/Scalar.Common/Git/GitObjects.cs
index 2af97e3182..c3339a7ebd 100644
--- a/Scalar.Common/Git/GitObjects.cs
+++ b/Scalar.Common/Git/GitObjects.cs
@@ -1,17 +1,10 @@
using Scalar.Common.FileSystem;
using Scalar.Common.Http;
-using Scalar.Common.NetworkStreams;
using Scalar.Common.Tracing;
using System;
-using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Linq;
-using System.Net;
-using System.Net.Http;
-using System.Net.Http.Headers;
-using System.Threading;
-using System.Threading.Tasks;
namespace Scalar.Common.Git
{
@@ -84,230 +77,6 @@ public virtual void DeleteTemporaryFiles()
}
}
- public virtual bool TryDownloadPrefetchPacks(GitProcess gitProcess, long latestTimestamp, out List packIndexes)
- {
- EventMetadata metadata = CreateEventMetadata();
- metadata.Add("latestTimestamp", latestTimestamp);
-
- using (ITracer activity = this.Tracer.StartActivity("TryDownloadPrefetchPacks", EventLevel.Informational, Keywords.Telemetry, metadata))
- {
- long bytesDownloaded = 0;
-
- long requestId = HttpRequestor.GetNewRequestId();
- List innerPackIndexes = null;
- RetryWrapper.InvocationResult result = this.GitObjectRequestor.TrySendProtocolRequest(
- requestId: requestId,
- onSuccess: (tryCount, response) => this.DeserializePrefetchPacks(response, ref latestTimestamp, ref bytesDownloaded, ref innerPackIndexes, gitProcess),
- onFailure: RetryWrapper.StandardErrorHandler(activity, requestId, "TryDownloadPrefetchPacks"),
- method: HttpMethod.Get,
- endPointGenerator: () => new Uri(
- string.Format(
- "{0}?lastPackTimestamp={1}",
- this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl,
- latestTimestamp)),
- requestBodyGenerator: () => null,
- cancellationToken: CancellationToken.None,
- acceptType: new MediaTypeWithQualityHeaderValue(ScalarConstants.MediaTypes.PrefetchPackFilesAndIndexesMediaType));
-
- packIndexes = innerPackIndexes;
-
- if (!result.Succeeded)
- {
- if (result.Result != null && result.Result.HttpStatusCodeResult == HttpStatusCode.NotFound)
- {
- EventMetadata warning = CreateEventMetadata();
- warning.Add(TracingConstants.MessageKey.WarningMessage, "The server does not support " + ScalarConstants.Endpoints.ScalarPrefetch);
- warning.Add(nameof(this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl), this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl);
- activity.RelatedEvent(EventLevel.Warning, "CommandNotSupported", warning);
- }
- else
- {
- EventMetadata error = CreateEventMetadata(result.Error);
- error.Add("latestTimestamp", latestTimestamp);
- error.Add(nameof(this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl), this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl);
- activity.RelatedWarning(error, "DownloadPrefetchPacks failed.", Keywords.Telemetry);
- }
- }
-
- activity.Stop(new EventMetadata
- {
- { "Area", EtwArea },
- { "Success", result.Succeeded },
- { "Attempts", result.Attempts },
- { "BytesDownloaded", bytesDownloaded },
- });
-
- return result.Succeeded;
- }
- }
-
- public virtual string WriteLooseObject(Stream responseStream, string sha, byte[] bufToCopyWith)
- {
- try
- {
- LooseObjectToWrite toWrite = this.GetLooseObjectDestination(sha);
-
- using (Stream fileStream = this.OpenTempLooseObjectStream(toWrite.TempFile))
- {
- StreamUtil.CopyToWithBuffer(responseStream, fileStream, bufToCopyWith);
- }
-
- this.FinalizeTempFile(sha, toWrite);
-
- return toWrite.ActualFile;
- }
- catch (IOException e)
- {
- throw new RetryableException("IOException while writing loose object. See inner exception for details.", e);
- }
- catch (UnauthorizedAccessException e)
- {
- throw new RetryableException("UnauthorizedAccessException while writing loose object. See inner exception for details.", e);
- }
- catch (Win32Exception e)
- {
- throw new RetryableException("Win32Exception while writing loose object. See inner exception for details.", e);
- }
- }
-
- public virtual string WriteTempPackFile(Stream stream)
- {
- string fileName = Path.GetRandomFileName();
- string fullPath = Path.Combine(this.Enlistment.GitPackRoot, fileName);
-
- Task flushTask;
- long fileLength;
- this.TryWriteTempFile(
- tracer: null,
- source: stream,
- tempFilePath: fullPath,
- fileLength: out fileLength,
- flushTask: out flushTask,
- throwOnError: true);
-
- flushTask?.Wait();
-
- return fullPath;
- }
-
- public virtual bool TryWriteTempFile(
- ITracer tracer,
- Stream source,
- string tempFilePath,
- out long fileLength,
- out Task flushTask,
- bool throwOnError = false)
- {
- fileLength = 0;
- flushTask = null;
- try
- {
- Stream fileStream = null;
-
- try
- {
- fileStream = this.fileSystem.OpenFileStream(
- tempFilePath,
- FileMode.OpenOrCreate,
- FileAccess.Write,
- FileShare.Read,
- callFlushFileBuffers: false); // Any flushing to disk will be done asynchronously
-
- StreamUtil.CopyToWithBuffer(source, fileStream);
- fileLength = fileStream.Length;
-
- if (this.Enlistment.FlushFileBuffersForPacks)
- {
- // Flush any data buffered in FileStream to the file system
- fileStream.Flush();
-
- // FlushFileBuffers using FlushAsync
- // Do this last to ensure that the stream is not being accessed after it's been disposed
- flushTask = fileStream.FlushAsync().ContinueWith((result) => fileStream.Dispose());
- }
- }
- finally
- {
- if (flushTask == null && fileStream != null)
- {
- fileStream.Dispose();
- }
- }
-
- this.ValidateTempFile(tempFilePath, tempFilePath);
- }
- catch (Exception ex)
- {
- if (flushTask != null)
- {
- flushTask.Wait();
- flushTask = null;
- }
-
- this.CleanupTempFile(this.Tracer, tempFilePath);
-
- if (tracer != null)
- {
- EventMetadata metadata = CreateEventMetadata(ex);
- metadata.Add("tempFilePath", tempFilePath);
- tracer.RelatedWarning(metadata, $"{nameof(this.TryWriteTempFile)}: Exception caught while writing temp file", Keywords.Telemetry);
- }
-
- if (throwOnError)
- {
- throw;
- }
- else
- {
- return false;
- }
- }
-
- return true;
- }
-
- public virtual GitProcess.Result IndexTempPackFile(string tempPackPath, GitProcess gitProcess = null)
- {
- string packfilePath = GetRandomPackName(this.Enlistment.GitPackRoot);
-
- Exception moveFileException = null;
- try
- {
- // We're indexing a pack file that was saved to a temp file name, and so it must be renamed
- // to its final name before indexing ('git index-pack' requires that the pack file name end with .pack)
- this.fileSystem.MoveFile(tempPackPath, packfilePath);
- }
- catch (IOException e)
- {
- moveFileException = e;
- }
- catch (UnauthorizedAccessException e)
- {
- moveFileException = e;
- }
-
- if (moveFileException != null)
- {
- EventMetadata failureMetadata = CreateEventMetadata(moveFileException);
- failureMetadata.Add("tempPackPath", tempPackPath);
- failureMetadata.Add("packfilePath", packfilePath);
-
- this.fileSystem.TryDeleteFile(tempPackPath, metadataKey: nameof(tempPackPath), metadata: failureMetadata);
-
- this.Tracer.RelatedWarning(failureMetadata, $"{nameof(this.IndexTempPackFile): Exception caught while trying to move temp pack file}");
-
- return new GitProcess.Result(
- string.Empty,
- moveFileException != null ? moveFileException.Message : "Failed to move temp pack file to final path",
- GitProcess.Result.GenericFailureCode);
- }
-
- // TryBuildIndex will delete the pack file if indexing fails
- GitProcess.Result result;
- this.TryBuildIndex(this.Tracer, packfilePath, out result, gitProcess);
- return result;
- }
-
public virtual GitProcess.Result IndexPackFile(string packfilePath, GitProcess gitProcess)
{
string tempIdxPath = Path.ChangeExtension(packfilePath, TempIdxExtension);
@@ -410,12 +179,6 @@ public virtual bool IsUsingCacheServer()
return !this.GitObjectRequestor.CacheServer.IsNone(this.Enlistment.RepoUrl);
}
- private static string GetRandomPackName(string packRoot)
- {
- string packName = "pack-" + Guid.NewGuid().ToString("N") + ".pack";
- return Path.Combine(packRoot, packName);
- }
-
private static EventMetadata CreateEventMetadata(Exception e = null)
{
EventMetadata metadata = new EventMetadata();
@@ -428,40 +191,6 @@ private static EventMetadata CreateEventMetadata(Exception e = null)
return metadata;
}
- private bool TryMovePackAndIdxFromTempFolder(string packName, string packTempPath, string idxName, string idxTempPath, out Exception exception)
- {
- exception = null;
- string finalPackPath = Path.Combine(this.Enlistment.GitPackRoot, packName);
- string finalIdxPath = Path.Combine(this.Enlistment.GitPackRoot, idxName);
-
- try
- {
- this.fileSystem.MoveAndOverwriteFile(packTempPath, finalPackPath);
- this.fileSystem.MoveAndOverwriteFile(idxTempPath, finalIdxPath);
- }
- catch (Win32Exception e)
- {
- exception = e;
-
- EventMetadata metadata = CreateEventMetadata(e);
- metadata.Add("packName", packName);
- metadata.Add("packTempPath", packTempPath);
- metadata.Add("idxName", idxName);
- metadata.Add("idxTempPath", idxTempPath);
-
- this.fileSystem.TryDeleteFile(idxTempPath, metadataKey: nameof(idxTempPath), metadata: metadata);
- this.fileSystem.TryDeleteFile(finalIdxPath, metadataKey: nameof(finalIdxPath), metadata: metadata);
- this.fileSystem.TryDeleteFile(packTempPath, metadataKey: nameof(packTempPath), metadata: metadata);
- this.fileSystem.TryDeleteFile(finalPackPath, metadataKey: nameof(finalPackPath), metadata: metadata);
-
- this.Tracer.RelatedWarning(metadata, $"{nameof(this.TryMovePackAndIdxFromTempFolder): Failed to move pack and idx from temp folder}");
-
- return false;
- }
-
- return true;
- }
-
private bool TryFlushFileBuffers(string path, out Exception exception, out string error)
{
error = null;
@@ -551,395 +280,5 @@ private bool TrySetAttributes(string path, FileAttributes attributes, out Except
return false;
}
-
- private Stream OpenTempLooseObjectStream(string path)
- {
- return this.fileSystem.OpenFileStream(
- path,
- FileMode.Create,
- FileAccess.Write,
- FileShare.None,
- FileOptions.SequentialScan,
- callFlushFileBuffers: false);
- }
-
- private LooseObjectToWrite GetLooseObjectDestination(string sha)
- {
- string firstTwoDigits = sha.Substring(0, 2);
- string remainingDigits = sha.Substring(2);
- string twoLetterFolderName = Path.Combine(this.Enlistment.GitObjectsRoot, firstTwoDigits);
- this.fileSystem.CreateDirectory(twoLetterFolderName);
-
- return new LooseObjectToWrite(
- tempFile: Path.Combine(twoLetterFolderName, Path.GetRandomFileName()),
- actualFile: Path.Combine(twoLetterFolderName, remainingDigits));
- }
-
- ///
- /// Uses a to read the packs from the stream.
- ///
- private RetryWrapper.CallbackResult DeserializePrefetchPacks(
- GitEndPointResponseData response,
- ref long latestTimestamp,
- ref long bytesDownloaded,
- ref List packIndexes,
- GitProcess gitProcess)
- {
- if (packIndexes == null)
- {
- packIndexes = new List();
- }
-
- using (ITracer activity = this.Tracer.StartActivity("DeserializePrefetchPacks", EventLevel.Informational))
- {
- PrefetchPacksDeserializer deserializer = new PrefetchPacksDeserializer(response.Stream);
-
- string tempPackFolderPath = Path.Combine(this.Enlistment.GitPackRoot, TempPackFolder);
- this.fileSystem.CreateDirectory(tempPackFolderPath);
-
- List tempPacks = new List();
- foreach (PrefetchPacksDeserializer.PackAndIndex pack in deserializer.EnumeratePacks())
- {
- // The advertised size may not match the actual, on-disk size.
- long indexLength = 0;
- long packLength;
-
- // Write the temp and index to a temp folder to avoid putting corrupt files in the pack folder
- // Once the files are validated and flushed they can be moved to the pack folder
- string packName = string.Format("{0}-{1}-{2}.pack", ScalarConstants.PrefetchPackPrefix, pack.Timestamp, pack.UniqueId);
- string packTempPath = Path.Combine(tempPackFolderPath, packName);
- string idxName = string.Format("{0}-{1}-{2}.idx", ScalarConstants.PrefetchPackPrefix, pack.Timestamp, pack.UniqueId);
- string idxTempPath = Path.Combine(tempPackFolderPath, idxName);
-
- EventMetadata data = CreateEventMetadata();
- data["timestamp"] = pack.Timestamp.ToString();
- data["uniqueId"] = pack.UniqueId;
- activity.RelatedEvent(EventLevel.Informational, "Receiving Pack/Index", data);
-
- // Write the pack
- // If it fails, TryWriteTempFile cleans up the file and we retry the fetch-commits-and-trees
- Task packFlushTask;
- if (!this.TryWriteTempFile(activity, pack.PackStream, packTempPath, out packLength, out packFlushTask))
- {
- bytesDownloaded += packLength;
- return new RetryWrapper.CallbackResult(null, true);
- }
-
- bytesDownloaded += packLength;
-
- // We will try to build an index if the server does not send one
- if (pack.IndexStream == null)
- {
- GitProcess.Result result;
- if (!this.TryBuildIndex(activity, packTempPath, out result, gitProcess))
- {
- if (packFlushTask != null)
- {
- packFlushTask.Wait();
- }
-
- // Move whatever has been successfully downloaded so far
- Exception moveException;
- this.TryFlushAndMoveTempPacks(tempPacks, ref latestTimestamp, out moveException);
-
- return new RetryWrapper.CallbackResult(null, true);
- }
-
- tempPacks.Add(new TempPrefetchPackAndIdx(pack.Timestamp, packName, packTempPath, packFlushTask, idxName, idxTempPath, idxFlushTask: null));
- }
- else
- {
- Task indexFlushTask;
- if (this.TryWriteTempFile(activity, pack.IndexStream, idxTempPath, out indexLength, out indexFlushTask))
- {
- tempPacks.Add(new TempPrefetchPackAndIdx(pack.Timestamp, packName, packTempPath, packFlushTask, idxName, idxTempPath, indexFlushTask));
- }
- else
- {
- bytesDownloaded += indexLength;
-
- // Try to build the index manually, then retry the fetch-commits-and-trees
- GitProcess.Result result;
- if (this.TryBuildIndex(activity, packTempPath, out result, gitProcess))
- {
- // If we were able to recreate the failed index
- // we can start the fetch-commits-and-trees at the next timestamp
- tempPacks.Add(new TempPrefetchPackAndIdx(pack.Timestamp, packName, packTempPath, packFlushTask, idxName, idxTempPath, idxFlushTask: null));
- }
- else
- {
- if (packFlushTask != null)
- {
- packFlushTask.Wait();
- }
- }
-
- // Move whatever has been successfully downloaded so far
- Exception moveException;
- this.TryFlushAndMoveTempPacks(tempPacks, ref latestTimestamp, out moveException);
-
- // The download stream will not be in a good state if the index download fails.
- // So we have to restart the fetch-commits-and-trees
- return new RetryWrapper.CallbackResult(null, true);
- }
- }
-
- bytesDownloaded += indexLength;
- }
-
- Exception exception = null;
- if (!this.TryFlushAndMoveTempPacks(tempPacks, ref latestTimestamp, out exception))
- {
- return new RetryWrapper.CallbackResult(exception, true);
- }
-
- foreach (TempPrefetchPackAndIdx tempPack in tempPacks)
- {
- packIndexes.Add(tempPack.IdxName);
- }
-
- return new RetryWrapper.CallbackResult(
- new GitObjectsHttpRequestor.GitObjectTaskResult(success: true));
- }
- }
-
- private bool TryFlushAndMoveTempPacks(List tempPacks, ref long latestTimestamp, out Exception exception)
- {
- exception = null;
- bool moveFailed = false;
- foreach (TempPrefetchPackAndIdx tempPack in tempPacks)
- {
- if (tempPack.PackFlushTask != null)
- {
- tempPack.PackFlushTask.Wait();
- }
-
- if (tempPack.IdxFlushTask != null)
- {
- tempPack.IdxFlushTask.Wait();
- }
-
- // If we've hit a failure moving temp files, we should stop trying to move them (but we still need to wait for all outstanding
- // flush tasks)
- if (!moveFailed)
- {
- if (this.TryMovePackAndIdxFromTempFolder(tempPack.PackName, tempPack.PackFullPath, tempPack.IdxName, tempPack.IdxFullPath, out exception))
- {
- latestTimestamp = tempPack.Timestamp;
- }
- else
- {
- moveFailed = true;
- }
- }
- }
-
- return !moveFailed;
- }
-
- ///
- /// Attempts to build an index for the specified path. If building the index fails, the pack file is deleted
- ///
- private bool TryBuildIndex(
- ITracer activity,
- string packFullPath,
- out GitProcess.Result result,
- GitProcess gitProcess)
- {
- result = this.IndexPackFile(packFullPath, gitProcess);
-
- if (result.ExitCodeIsFailure)
- {
- EventMetadata errorMetadata = CreateEventMetadata();
- Exception exception;
- if (!this.fileSystem.TryDeleteFile(packFullPath, exception: out exception))
- {
- if (exception != null)
- {
- errorMetadata.Add("deleteException", exception.ToString());
- }
-
- errorMetadata.Add("deletedBadPack", "false");
- }
-
- errorMetadata.Add("Operation", nameof(this.TryBuildIndex));
- errorMetadata.Add("packFullPath", packFullPath);
- activity.RelatedWarning(errorMetadata, result.Errors, Keywords.Telemetry);
- }
-
- return result.ExitCodeIsSuccess;
- }
-
- private void CleanupTempFile(ITracer activity, string fullPath)
- {
- Exception e;
- if (!this.fileSystem.TryDeleteFile(fullPath, exception: out e))
- {
- EventMetadata info = CreateEventMetadata(e);
- info.Add("file", fullPath);
- activity.RelatedWarning(info, "Failed to cleanup temp file");
- }
- }
-
- private void FinalizeTempFile(string sha, LooseObjectToWrite toWrite)
- {
- try
- {
- // Checking for existence reduces warning outputs when the same object is being downloaded
- // concurrently
- if (!this.fileSystem.FileExists(toWrite.ActualFile))
- {
- this.ValidateTempFile(toWrite.TempFile, sha);
-
- try
- {
- this.fileSystem.MoveFile(toWrite.TempFile, toWrite.ActualFile);
- }
- catch (IOException ex)
- {
- // IOExceptions happen when someone else is writing to our object.
- // That implies they are doing what we're doing, which should be a success
- EventMetadata info = CreateEventMetadata(ex);
- info.Add("file", toWrite.ActualFile);
- this.Tracer.RelatedWarning(info, $"{nameof(this.FinalizeTempFile)}: Exception moving temp file");
- }
- }
- }
- finally
- {
- this.CleanupTempFile(this.Tracer, toWrite.TempFile);
- }
- }
-
- private void ValidateTempFile(string tempFilePath, string finalFilePath)
- {
- using (Stream fs = this.fileSystem.OpenFileStream(tempFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, callFlushFileBuffers: false))
- {
- if (fs.Length == 0)
- {
- throw new RetryableException($"Temp file '{tempFilePath}' for '{finalFilePath}' was written with 0 bytes");
- }
- else
- {
- byte[] buffer = new byte[10];
-
- // Temp files should always have at least one non-zero byte
- int bytesRead = fs.Read(buffer, 0, buffer.Length);
- if (buffer.All(b => b == 0))
- {
- RetryableException ex = new RetryableException(
- $"Temp file '{tempFilePath}' for '{finalFilePath}' was written with {bytesRead} null bytes");
-
- EventMetadata eventInfo = CreateEventMetadata(ex);
- eventInfo.Add("file", tempFilePath);
- eventInfo.Add("finalFilePath", finalFilePath);
- this.Tracer.RelatedWarning(eventInfo, $"{nameof(this.ValidateTempFile)}: Temp file invalid");
-
- throw ex;
- }
- }
- }
- }
-
- private RetryWrapper.CallbackResult TrySavePackOrLooseObject(
- IEnumerable objectShas,
- bool unpackObjects,
- GitEndPointResponseData responseData,
- GitProcess gitProcess)
- {
- if (responseData.ContentType == GitObjectContentType.LooseObject)
- {
- List objectShaList = objectShas.Distinct().ToList();
- if (objectShaList.Count != 1)
- {
- return new RetryWrapper.CallbackResult(new InvalidOperationException("Received loose object when multiple objects were requested."), shouldRetry: false);
- }
-
- // To reduce allocations, reuse the same buffer when writing objects in this batch
- byte[] bufToCopyWith = new byte[StreamUtil.DefaultCopyBufferSize];
-
- this.WriteLooseObject(responseData.Stream, objectShaList[0], bufToCopyWith: bufToCopyWith);
- }
- else if (responseData.ContentType == GitObjectContentType.BatchedLooseObjects)
- {
- // To reduce allocations, reuse the same buffer when writing objects in this batch
- byte[] bufToCopyWith = new byte[StreamUtil.DefaultCopyBufferSize];
-
- BatchedLooseObjectDeserializer deserializer = new BatchedLooseObjectDeserializer(
- responseData.Stream,
- (stream, sha) => this.WriteLooseObject(stream, sha, bufToCopyWith: bufToCopyWith));
- deserializer.ProcessObjects();
- }
- else
- {
- GitProcess.Result result = this.TryAddPackFile(responseData.Stream, unpackObjects, gitProcess);
- if (result.ExitCodeIsFailure)
- {
- return new RetryWrapper.CallbackResult(new InvalidOperationException("Could not add pack file: " + result.Errors), shouldRetry: false);
- }
- }
-
- return new RetryWrapper.CallbackResult(new GitObjectsHttpRequestor.GitObjectTaskResult(true));
- }
-
- private GitProcess.Result TryAddPackFile(Stream contents, bool unpackObjects, GitProcess gitProcess)
- {
- GitProcess.Result result;
-
- this.fileSystem.CreateDirectory(this.Enlistment.GitPackRoot);
-
- if (unpackObjects)
- {
- result = new GitProcess(this.Enlistment).UnpackObjects(contents);
- }
- else
- {
- string tempPackPath = this.WriteTempPackFile(contents);
- return this.IndexTempPackFile(tempPackPath, gitProcess);
- }
-
- return result;
- }
-
- private struct LooseObjectToWrite
- {
- public readonly string TempFile;
- public readonly string ActualFile;
-
- public LooseObjectToWrite(string tempFile, string actualFile)
- {
- this.TempFile = tempFile;
- this.ActualFile = actualFile;
- }
- }
-
- private class TempPrefetchPackAndIdx
- {
- public TempPrefetchPackAndIdx(
- long timestamp,
- string packName,
- string packFullPath,
- Task packFlushTask,
- string idxName,
- string idxFullPath,
- Task idxFlushTask)
- {
- this.Timestamp = timestamp;
- this.PackName = packName;
- this.PackFullPath = packFullPath;
- this.PackFlushTask = packFlushTask;
- this.IdxName = idxName;
- this.IdxFullPath = idxFullPath;
- this.IdxFlushTask = idxFlushTask;
- }
-
- public long Timestamp { get; }
- public string PackName { get; }
- public string PackFullPath { get; }
- public Task PackFlushTask { get; }
- public string IdxName { get; }
- public string IdxFullPath { get; }
- public Task IdxFlushTask { get; }
- }
}
}
diff --git a/Scalar.Common/Git/GitProcess.cs b/Scalar.Common/Git/GitProcess.cs
index 3eedfe29b4..0e6c864a2c 100644
--- a/Scalar.Common/Git/GitProcess.cs
+++ b/Scalar.Common/Git/GitProcess.cs
@@ -449,6 +449,11 @@ public Result GvfsHelperDownloadCommit(string commitId)
});
}
+ public Result GvfsHelperPrefetch()
+ {
+ return this.InvokeGitInWorkingDirectoryRoot("gvfs-helper prefetch", fetchMissingObjects: false);
+ }
+
public Result Status(bool allowObjectDownloads, bool useStatusCache, bool showUntracked = false)
{
string command = "status";
diff --git a/Scalar.Common/Http/GitObjectsHttpRequestor.cs b/Scalar.Common/Http/GitObjectsHttpRequestor.cs
index 72017ec22f..3f4bd59c49 100644
--- a/Scalar.Common/Http/GitObjectsHttpRequestor.cs
+++ b/Scalar.Common/Http/GitObjectsHttpRequestor.cs
@@ -4,7 +4,6 @@
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
-using System.Net.Http.Headers;
using System.Threading;
namespace Scalar.Common.Http
@@ -58,92 +57,6 @@ public virtual GitRefs QueryInfoRefs(string branch)
return output.Result;
}
- public virtual RetryWrapper.InvocationResult TrySendProtocolRequest(
- long requestId,
- Func.CallbackResult> onSuccess,
- Action.ErrorEventArgs> onFailure,
- HttpMethod method,
- Uri endPoint,
- CancellationToken cancellationToken,
- string requestBody = null,
- MediaTypeWithQualityHeaderValue acceptType = null,
- bool retryOnFailure = true)
- {
- return this.TrySendProtocolRequest(
- requestId,
- onSuccess,
- onFailure,
- method,
- endPoint,
- cancellationToken,
- () => requestBody,
- acceptType,
- retryOnFailure);
- }
-
- public virtual RetryWrapper.InvocationResult TrySendProtocolRequest(
- long requestId,
- Func.CallbackResult> onSuccess,
- Action.ErrorEventArgs> onFailure,
- HttpMethod method,
- Uri endPoint,
- CancellationToken cancellationToken,
- Func requestBodyGenerator,
- MediaTypeWithQualityHeaderValue acceptType = null,
- bool retryOnFailure = true)
- {
- return this.TrySendProtocolRequest(
- requestId,
- onSuccess,
- onFailure,
- method,
- () => endPoint,
- requestBodyGenerator,
- cancellationToken,
- acceptType,
- retryOnFailure);
- }
-
- public virtual RetryWrapper.InvocationResult TrySendProtocolRequest(
- long requestId,
- Func.CallbackResult> onSuccess,
- Action.ErrorEventArgs> onFailure,
- HttpMethod method,
- Func endPointGenerator,
- Func requestBodyGenerator,
- CancellationToken cancellationToken,
- MediaTypeWithQualityHeaderValue acceptType = null,
- bool retryOnFailure = true)
- {
- RetryWrapper retrier = new RetryWrapper(
- retryOnFailure ? this.RetryConfig.MaxAttempts : 1,
- cancellationToken);
- if (onFailure != null)
- {
- retrier.OnFailure += onFailure;
- }
-
- return retrier.Invoke(
- tryCount =>
- {
- using (GitEndPointResponseData response = this.SendRequest(
- requestId,
- endPointGenerator(),
- method,
- requestBodyGenerator(),
- cancellationToken,
- acceptType))
- {
- if (response.HasErrors)
- {
- return new RetryWrapper.CallbackResult(response.Error, response.ShouldRetry, new GitObjectTaskResult(response.StatusCode));
- }
-
- return onSuccess(tryCount, response);
- }
- });
- }
-
public class GitObjectTaskResult
{
public GitObjectTaskResult(bool success)
diff --git a/Scalar.Common/Maintenance/FetchCommitsAndTreesStep.cs b/Scalar.Common/Maintenance/FetchCommitsAndTreesStep.cs
index 1c4e657d17..00d126783d 100644
--- a/Scalar.Common/Maintenance/FetchCommitsAndTreesStep.cs
+++ b/Scalar.Common/Maintenance/FetchCommitsAndTreesStep.cs
@@ -35,8 +35,6 @@ public bool TryFetchCommitsAndTrees(out string error, GitProcess gitProcess = nu
gitProcess = new GitProcess(this.Context.Enlistment);
}
- List packIndexes;
-
// We take our own lock here to keep background and foreground fetches
// (i.e. a user running 'scalar maintenance --fetch-commits-and-trees')
// from running at the same time.
@@ -46,25 +44,22 @@ public bool TryFetchCommitsAndTrees(out string error, GitProcess gitProcess = nu
Path.Combine(this.Context.Enlistment.GitPackRoot, FetchCommitsAndTreesLock)))
{
WaitUntilLockIsAcquired(this.Context.Tracer, fetchLock);
- long maxGoodTimeStamp;
this.GitObjects.DeleteStaleTempPrefetchPackAndIdxs();
this.GitObjects.DeleteTemporaryFiles();
- if (!this.TryGetMaxGoodPrefetchPackTimestamp(out maxGoodTimeStamp, out error))
- {
- return false;
- }
+ GitProcess.Result result = gitProcess.GvfsHelperPrefetch();
- if (!this.GitObjects.TryDownloadPrefetchPacks(gitProcess, maxGoodTimeStamp, out packIndexes))
+ if (result.ExitCodeIsFailure)
{
- error = "Failed to download prefetch packs";
+ error = result.Errors;
return false;
}
this.UpdateKeepPacks();
}
+ error = null;
return true;
}
diff --git a/Scalar.Common/NetworkStreams/PrefetchPacksDeserializer.cs b/Scalar.Common/NetworkStreams/PrefetchPacksDeserializer.cs
deleted file mode 100644
index f8fc45bf4c..0000000000
--- a/Scalar.Common/NetworkStreams/PrefetchPacksDeserializer.cs
+++ /dev/null
@@ -1,124 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Text;
-
-namespace Scalar.Common.NetworkStreams
-{
- ///
- /// Deserializer for packs and indexes for prefetch packs.
- ///
- public class PrefetchPacksDeserializer
- {
- private const int NumPackHeaderBytes = 3 * sizeof(long);
-
- private static readonly byte[] PrefetchPackExpectedHeader =
- new byte[]
- {
- (byte)'G', (byte)'P', (byte)'R', (byte)'E', (byte)' ', // Magic
- 1 // Version
- };
-
- private readonly Stream source;
-
- public PrefetchPacksDeserializer(Stream source)
- {
- this.source = source;
- }
-
- ///
- /// Read all the packs and indexes from the source stream and return a for each pack
- /// and index. Caller must consume pack stream fully before the index stream.
- ///
- public IEnumerable EnumeratePacks()
- {
- this.ValidateHeader();
-
- byte[] buffer = new byte[NumPackHeaderBytes];
-
- int packCount = this.ReadPackCount(buffer);
-
- for (int i = 0; i < packCount; i++)
- {
- long timestamp;
- long packLength;
- long indexLength;
- this.ReadPackHeader(buffer, out timestamp, out packLength, out indexLength);
-
- using (Stream packData = new RestrictedStream(this.source, packLength))
- using (Stream indexData = indexLength > 0 ? new RestrictedStream(this.source, indexLength) : null)
- {
- yield return new PackAndIndex(packData, indexData, timestamp);
- }
- }
- }
-
- ///
- /// Read the ushort pack count
- ///
- private ushort ReadPackCount(byte[] buffer)
- {
- StreamUtil.TryReadGreedy(this.source, buffer, 0, 2);
- return BitConverter.ToUInt16(buffer, 0);
- }
-
- ///
- /// Parse the current pack header
- ///
- private void ReadPackHeader(
- byte[] buffer,
- out long timestamp,
- out long packLength,
- out long indexLength)
- {
- int totalBytes = StreamUtil.TryReadGreedy(
- this.source,
- buffer,
- 0,
- NumPackHeaderBytes);
-
- if (totalBytes == NumPackHeaderBytes)
- {
- timestamp = BitConverter.ToInt64(buffer, 0);
- packLength = BitConverter.ToInt64(buffer, 8);
- indexLength = BitConverter.ToInt64(buffer, 16);
- }
- else
- {
- throw new RetryableException(
- string.Format(
- "Reached end of stream before expected {0} bytes. Got {1}. Buffer: {2}",
- NumPackHeaderBytes,
- totalBytes,
- SHA1Util.HexStringFromBytes(buffer)));
- }
- }
-
- private void ValidateHeader()
- {
- byte[] headerBuf = new byte[PrefetchPackExpectedHeader.Length];
- StreamUtil.TryReadGreedy(this.source, headerBuf, 0, headerBuf.Length);
- if (!headerBuf.SequenceEqual(PrefetchPackExpectedHeader))
- {
- throw new InvalidDataException("Unexpected header: " + Encoding.UTF8.GetString(headerBuf));
- }
- }
-
- public class PackAndIndex
- {
- public PackAndIndex(Stream packStream, Stream idxStream, long timestamp)
- {
- this.PackStream = packStream;
- this.IndexStream = idxStream;
- this.Timestamp = timestamp;
- this.UniqueId = Guid.NewGuid().ToString("N");
- }
-
- public Stream PackStream { get; }
- public Stream IndexStream { get; }
- public long Timestamp { get; }
- public string UniqueId { get; }
- }
- }
-}
diff --git a/Scalar.Common/Paths.Shared.cs b/Scalar.Common/Paths.Shared.cs
index c3c062da61..91fd83e218 100644
--- a/Scalar.Common/Paths.Shared.cs
+++ b/Scalar.Common/Paths.Shared.cs
@@ -1,51 +1,9 @@
-using System;
using System.IO;
-using System.Linq;
namespace Scalar.Common
{
public static class Paths
{
- public static string GetRoot(string startingDirectory, string rootName)
- {
- startingDirectory = startingDirectory.TrimEnd(Path.DirectorySeparatorChar);
- DirectoryInfo dirInfo;
-
- try
- {
- dirInfo = new DirectoryInfo(startingDirectory);
- }
- catch (Exception)
- {
- return null;
- }
-
- while (dirInfo != null)
- {
- if (dirInfo.Exists)
- {
- DirectoryInfo[] dotScalarDirs = new DirectoryInfo[0];
-
- try
- {
- dotScalarDirs = dirInfo.GetDirectories(rootName);
- }
- catch (IOException)
- {
- }
-
- if (dotScalarDirs.Count() == 1)
- {
- return dirInfo.FullName;
- }
- }
-
- dirInfo = dirInfo.Parent;
- }
-
- return null;
- }
-
public static string ConvertPathToGitFormat(string path)
{
return path.Replace(Path.DirectorySeparatorChar, ScalarConstants.GitPathSeparator);
diff --git a/Scalar.FunctionalTests/Tests/EnlistmentPerFixture/FetchCommitsAndTreesWithoutSharedCacheTests.cs b/Scalar.FunctionalTests/Tests/EnlistmentPerFixture/FetchCommitsAndTreesWithoutSharedCacheTests.cs
index 39b0b8f1fe..4b11716e40 100644
--- a/Scalar.FunctionalTests/Tests/EnlistmentPerFixture/FetchCommitsAndTreesWithoutSharedCacheTests.cs
+++ b/Scalar.FunctionalTests/Tests/EnlistmentPerFixture/FetchCommitsAndTreesWithoutSharedCacheTests.cs
@@ -102,36 +102,6 @@ public void FetchCommitsAndTreesCleansUpBadPrefetchPack()
}
[TestCase, Order(4)]
- public void FetchCommitsAndTreesCleansUpOldPrefetchPack()
- {
- this.Enlistment.UnregisterRepo();
-
- string[] prefetchPacks = this.ReadPrefetchPackFileNames();
- long oldestPackTimestamp = this.GetOldestPackTimestamp(prefetchPacks);
-
- // Create a bad pack that is older than the oldest pack
- string badContents = "BADPACK";
- string badPackPath = Path.Combine(this.PackRoot, $"{PrefetchPackPrefix}-{oldestPackTimestamp - 1}-{Guid.NewGuid().ToString("N")}.pack");
- this.fileSystem.WriteAllText(badPackPath, badContents);
- badPackPath.ShouldBeAFile(this.fileSystem).WithContents(badContents);
-
- // fetch-commits-and-trees should delete the bad pack and all packs after it
- this.Enlistment.FetchCommitsAndTrees();
-
- badPackPath.ShouldNotExistOnDisk(this.fileSystem);
- foreach (string packPath in prefetchPacks)
- {
- string idxPath = Path.ChangeExtension(packPath, ".idx");
- badPackPath.ShouldNotExistOnDisk(this.fileSystem);
- idxPath.ShouldNotExistOnDisk(this.fileSystem);
- }
-
- string[] newPrefetchPacks = this.ReadPrefetchPackFileNames();
- this.AllPrefetchPacksShouldHaveIdx(newPrefetchPacks);
- this.TempPackRoot.ShouldBeADirectory(this.fileSystem).WithNoItems();
- }
-
- [TestCase, Order(5)]
[Category(Categories.MacTODO.TestNeedsToLockFile)]
public void FetchCommitsAndTreesFailsWhenItCannotRemoveABadPrefetchPack()
{
@@ -164,79 +134,7 @@ public void FetchCommitsAndTreesFailsWhenItCannotRemoveABadPrefetchPack()
this.TempPackRoot.ShouldBeADirectory(this.fileSystem).WithNoItems();
}
- [TestCase, Order(6)]
- [Category(Categories.MacTODO.TestNeedsToLockFile)]
- public void FetchCommitsAndTreesFailsWhenItCannotRemoveAPrefetchPackNewerThanBadPrefetchPack()
- {
- this.Enlistment.UnregisterRepo();
-
- string[] prefetchPacks = this.ReadPrefetchPackFileNames();
- long oldestPackTimestamp = this.GetOldestPackTimestamp(prefetchPacks);
-
- // Create a bad pack that is older than the oldest pack
- string badContents = "BADPACK";
- string badPackPath = Path.Combine(this.PackRoot, $"{PrefetchPackPrefix}-{oldestPackTimestamp - 1}-{Guid.NewGuid().ToString("N")}.pack");
- this.fileSystem.WriteAllText(badPackPath, badContents);
- badPackPath.ShouldBeAFile(this.fileSystem).WithContents(badContents);
-
- // Open a handle to a good pack that is newer than the bad pack, which will prevent fetch-commits-and-trees from being able to delete it
- using (FileStream stream = new FileStream(prefetchPacks[0], FileMode.Open, FileAccess.Read, FileShare.None))
- {
- string output = this.Enlistment.FetchCommitsAndTrees(failOnError: false);
- output.ShouldContain($"Unable to delete {prefetchPacks[0]}");
- }
-
- // After handle is closed fetching commits and trees should succeed
- this.Enlistment.FetchCommitsAndTrees();
-
- // The bad pack and all packs newer than it should not be on disk
- badPackPath.ShouldNotExistOnDisk(this.fileSystem);
-
- string[] newPrefetchPacks = this.ReadPrefetchPackFileNames();
- newPrefetchPacks.ShouldNotContain(prefetchPacks, (item, expectedValue) => { return string.Equals(item, expectedValue); });
- this.AllPrefetchPacksShouldHaveIdx(newPrefetchPacks);
- this.TempPackRoot.ShouldBeADirectory(this.fileSystem).WithNoItems();
- }
-
- [TestCase, Order(7)]
- [Category(Categories.MacTODO.TestNeedsToLockFile)]
- public void FetchCommitsAndTreesFailsWhenItCannotRemoveAPrefetchIdxNewerThanBadPrefetchPack()
- {
- this.Enlistment.UnregisterRepo();
-
- string[] prefetchPacks = this.ReadPrefetchPackFileNames();
- long oldestPackTimestamp = this.GetOldestPackTimestamp(prefetchPacks);
-
- // Create a bad pack that is older than the oldest pack
- string badContents = "BADPACK";
- string badPackPath = Path.Combine(this.PackRoot, $"{PrefetchPackPrefix}-{oldestPackTimestamp - 1}-{Guid.NewGuid().ToString("N")}.pack");
- this.fileSystem.WriteAllText(badPackPath, badContents);
- badPackPath.ShouldBeAFile(this.fileSystem).WithContents(badContents);
-
- string newerIdxPath = Path.ChangeExtension(prefetchPacks[0], ".idx");
- newerIdxPath.ShouldBeAFile(this.fileSystem);
-
- // Open a handle to a good idx that is newer than the bad pack, which will prevent fetch-commits-and-trees from being able to delete it
- using (FileStream stream = new FileStream(newerIdxPath, FileMode.Open, FileAccess.Read, FileShare.None))
- {
- string output = this.Enlistment.FetchCommitsAndTrees(failOnError: false);
- output.ShouldContain($"Unable to delete {newerIdxPath}");
- }
-
- // After handle is closed fetching commits and trees should succeed
- this.Enlistment.FetchCommitsAndTrees();
-
- // The bad pack and all packs newer than it should not be on disk
- badPackPath.ShouldNotExistOnDisk(this.fileSystem);
- newerIdxPath.ShouldNotExistOnDisk(this.fileSystem);
-
- string[] newPrefetchPacks = this.ReadPrefetchPackFileNames();
- newPrefetchPacks.ShouldNotContain(prefetchPacks, (item, expectedValue) => { return string.Equals(item, expectedValue); });
- this.AllPrefetchPacksShouldHaveIdx(newPrefetchPacks);
- this.TempPackRoot.ShouldBeADirectory(this.fileSystem).WithNoItems();
- }
-
- [TestCase, Order(8)]
+ [TestCase, Order(5)]
public void FetchCommitsAndTreesCleansUpStaleTempPrefetchPacks()
{
this.Enlistment.UnregisterRepo();
diff --git a/Scalar.UnitTests/Mock/Common/MockPhysicalGitObjects.cs b/Scalar.UnitTests/Mock/Common/MockPhysicalGitObjects.cs
deleted file mode 100644
index 2a7725da77..0000000000
--- a/Scalar.UnitTests/Mock/Common/MockPhysicalGitObjects.cs
+++ /dev/null
@@ -1,44 +0,0 @@
-using Scalar.Common;
-using Scalar.Common.FileSystem;
-using Scalar.Common.Git;
-using Scalar.Common.Http;
-using Scalar.Common.Tracing;
-using System.Diagnostics;
-using System.IO;
-
-namespace Scalar.UnitTests.Mock.Common
-{
- public class MockPhysicalGitObjects : GitObjects
- {
- public MockPhysicalGitObjects(ITracer tracer, PhysicalFileSystem fileSystem, Enlistment enlistment, GitObjectsHttpRequestor objectRequestor)
- : base(tracer, enlistment, objectRequestor, fileSystem)
- {
- }
-
- public override string WriteLooseObject(Stream responseStream, string sha, byte[] sharedBuf = null)
- {
- using (StreamReader reader = new StreamReader(responseStream))
- {
- // Return "file contents" as "file name". Weird, but proves we got the right thing.
- return reader.ReadToEnd();
- }
- }
-
- public override string WriteTempPackFile(Stream stream)
- {
- Debug.Assert(stream != null, "WriteTempPackFile should not receive a null stream");
-
- using (stream)
- using (StreamReader reader = new StreamReader(stream))
- {
- // Return "file contents" as "file name". Weird, but proves we got the right thing.
- return reader.ReadToEnd();
- }
- }
-
- public override GitProcess.Result IndexTempPackFile(string tempPackPath, GitProcess gitProcess = null)
- {
- return new GitProcess.Result(string.Empty, "TestFailure", GitProcess.Result.GenericFailureCode);
- }
- }
-}
diff --git a/Scalar.UnitTests/Prefetch/PrefetchPacksDeserializerTests.cs b/Scalar.UnitTests/Prefetch/PrefetchPacksDeserializerTests.cs
deleted file mode 100644
index e30bdc88e7..0000000000
--- a/Scalar.UnitTests/Prefetch/PrefetchPacksDeserializerTests.cs
+++ /dev/null
@@ -1,181 +0,0 @@
-using NUnit.Framework;
-using Scalar.Common.NetworkStreams;
-using Scalar.Tests.Should;
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-
-namespace Scalar.UnitTests.Prefetch
-{
- [TestFixture]
- public class PrefetchPacksDeserializerTests
- {
- private static readonly byte[] PrefetchPackExpectedHeader
- = new byte[]
- {
- (byte)'G', (byte)'P', (byte)'R', (byte)'E', (byte)' ',
- 1 // Version
- };
-
- [TestCase]
- public void PrefetchPacksDeserializer_No_Packs_Succeeds()
- {
- this.RunPrefetchPacksDeserializerTest(0, false);
- }
-
- [TestCase]
- public void PrefetchPacksDeserializer_Single_Pack_With_Index_Receives_Both()
- {
- this.RunPrefetchPacksDeserializerTest(1, true);
- }
-
- [TestCase]
- public void PrefetchPacksDeserializer_Single_Pack_Without_Index_Receives_Only_Pack()
- {
- this.RunPrefetchPacksDeserializerTest(1, false);
- }
-
- [TestCase]
- public void PrefetchPacksDeserializer_Multiple_Packs_With_Indexes()
- {
- this.RunPrefetchPacksDeserializerTest(10, true);
- }
-
- [TestCase]
- public void PrefetchPacksDeserializer_Multiple_Packs_Without_Indexes()
- {
- this.RunPrefetchPacksDeserializerTest(10, false);
- }
-
- ///
- /// A deterministic way to create somewhat unique packs
- ///
- private static byte[] PackForTimestamp(long timestamp)
- {
- unchecked
- {
- Random rand = new Random((int)timestamp);
- byte[] data = new byte[100];
- rand.NextBytes(data);
- return data;
- }
- }
-
- ///
- /// A deterministic way to create somewhat unique indexes
- ///
- private static byte[] IndexForTimestamp(long timestamp)
- {
- unchecked
- {
- Random rand = new Random((int)-timestamp);
- byte[] data = new byte[50];
- rand.NextBytes(data);
- return data;
- }
- }
-
- ///
- /// Implementation of the PrefetchPack spec to generate data for tests
- ///
- private void WriteToSpecs(Stream stream, long[] packTimestamps, bool withIndexes)
- {
- // Header
- stream.Write(PrefetchPackExpectedHeader, 0, PrefetchPackExpectedHeader.Length);
-
- // PackCount
- stream.Write(BitConverter.GetBytes((ushort)packTimestamps.Length), 0, 2);
-
- for (int i = 0; i < packTimestamps.Length; i++)
- {
- byte[] packContents = PackForTimestamp(packTimestamps[i]);
- byte[] indexContents = IndexForTimestamp(packTimestamps[i]);
-
- // Pack Header
- // Timestamp
- stream.Write(BitConverter.GetBytes(packTimestamps[i]), 0, 8);
-
- // Pack length
- stream.Write(BitConverter.GetBytes((long)packContents.Length), 0, 8);
-
- // Pack index length
- if (withIndexes)
- {
- stream.Write(BitConverter.GetBytes((long)indexContents.Length), 0, 8);
- }
- else
- {
- stream.Write(BitConverter.GetBytes(-1L), 0, 8);
- }
-
- // Pack data
- stream.Write(packContents, 0, packContents.Length);
-
- if (withIndexes)
- {
- stream.Write(indexContents, 0, indexContents.Length);
- }
- }
- }
-
- private void RunPrefetchPacksDeserializerTest(int packCount, bool withIndexes)
- {
- using (MemoryStream ms = new MemoryStream())
- {
- long[] packTimestamps = Enumerable.Range(0, packCount).Select(x => (long)x).ToArray();
-
- // Write the data to the memory stream.
- this.WriteToSpecs(ms, packTimestamps, withIndexes);
- ms.Position = 0;
-
- Dictionary>> receivedPacksAndIndexes = new Dictionary>>();
-
- foreach (PrefetchPacksDeserializer.PackAndIndex pack in new PrefetchPacksDeserializer(ms).EnumeratePacks())
- {
- List> packsAndIndexesByUniqueId;
- if (!receivedPacksAndIndexes.TryGetValue(pack.UniqueId, out packsAndIndexesByUniqueId))
- {
- packsAndIndexesByUniqueId = new List>();
- receivedPacksAndIndexes.Add(pack.UniqueId, packsAndIndexesByUniqueId);
- }
-
- using (MemoryStream packContent = new MemoryStream())
- using (MemoryStream idxContent = new MemoryStream())
- {
- pack.PackStream.CopyTo(packContent);
- byte[] packData = packContent.ToArray();
- packData.ShouldMatchInOrder(PackForTimestamp(pack.Timestamp));
- packsAndIndexesByUniqueId.Add(Tuple.Create("pack", pack.Timestamp));
-
- if (pack.IndexStream != null)
- {
- pack.IndexStream.CopyTo(idxContent);
- byte[] idxData = idxContent.ToArray();
- idxData.ShouldMatchInOrder(IndexForTimestamp(pack.Timestamp));
- packsAndIndexesByUniqueId.Add(Tuple.Create("idx", pack.Timestamp));
- }
- }
- }
-
- receivedPacksAndIndexes.Count.ShouldEqual(packCount, "UniqueId count");
-
- foreach (List> groupedByUniqueId in receivedPacksAndIndexes.Values)
- {
- if (withIndexes)
- {
- groupedByUniqueId.Count.ShouldEqual(2, "Both Pack and Index for UniqueId");
-
- // Should only contain 1 index file
- groupedByUniqueId.ShouldContainSingle(x => x.Item1 == "idx");
- }
-
- // should only contain 1 pack file
- groupedByUniqueId.ShouldContainSingle(x => x.Item1 == "pack");
-
- groupedByUniqueId.Select(x => x.Item2).Distinct().Count().ShouldEqual(1, "Same timestamps for a uniqueId");
- }
- }
- }
- }
-}