From 91261ac35e881d6c65988fd6f89ad92ad730e205 Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Sat, 18 Dec 2021 18:08:36 -0800 Subject: [PATCH 1/5] First pass of fixes --- cs/src/core/FasterLog/FasterLog.cs | 27 +++++++++++++++++++-------- cs/test/DeviceFasterLogTests.cs | 2 +- cs/test/TestUtils.cs | 4 ++-- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/cs/src/core/FasterLog/FasterLog.cs b/cs/src/core/FasterLog/FasterLog.cs index 475644ffe..3829a7266 100644 --- a/cs/src/core/FasterLog/FasterLog.cs +++ b/cs/src/core/FasterLog/FasterLog.cs @@ -120,12 +120,21 @@ public sealed class FasterLog : IDisposable /// Used to determine disposability of log /// internal int logRefCount = 1; - + /// /// Create new log instance /// /// Log settings public FasterLog(FasterLogSettings logSettings) + : this(logSettings, logSettings.TryRecoverLatest) + { } + + /// + /// Create new log instance + /// + /// Log settings + /// Recover synchronously + private FasterLog(FasterLogSettings logSettings, bool syncRecover) { logCommitManager = logSettings.LogCommitManager ?? new DeviceLogCommitCheckpointManager @@ -167,7 +176,8 @@ public FasterLog(FasterLogSettings logSettings) commitPolicy.OnAttached(this); tolerateDeviceFailure = logSettings.TolerateDeviceFailure; - if (logSettings.TryRecoverLatest) + + if (syncRecover) { try { @@ -201,11 +211,13 @@ public void Recover(long requestedCommitNum = -1) /// public static async ValueTask CreateAsync(FasterLogSettings logSettings, CancellationToken cancellationToken = default) { - var fasterLog = new FasterLog(logSettings); - var (it, cookie) = await fasterLog.RestoreLatestAsync(cancellationToken).ConfigureAwait(false); - fasterLog.RecoveredIterators = it; - fasterLog.RecoveredCookie = cookie; - + var fasterLog = new FasterLog(logSettings, false); + if (logSettings.TryRecoverLatest) + { + var (it, cookie) = await fasterLog.RestoreLatestAsync(cancellationToken).ConfigureAwait(false); + fasterLog.RecoveredIterators = it; + fasterLog.RecoveredCookie = cookie; + } return fasterLog; } @@ -1499,7 +1511,6 @@ private void RestoreLatest(out Dictionary iterators, out byte[] co cookie = info.Cookie; commitNum = info.CommitNum; beginAddress = allocator.BeginAddress; - if (commitNum == long.MaxValue) throw new FasterException("Attempting to enqueue into a completed log"); if (readOnlyMode) allocator.HeadAddress = long.MaxValue; diff --git a/cs/test/DeviceFasterLogTests.cs b/cs/test/DeviceFasterLogTests.cs index 4d294034b..b70a57bd8 100644 --- a/cs/test/DeviceFasterLogTests.cs +++ b/cs/test/DeviceFasterLogTests.cs @@ -96,7 +96,7 @@ public void BasicHighLatencyDeviceTest() private async ValueTask FasterLogTest1(LogChecksumType logChecksum, IDevice device, ILogCommitManager logCommitManager, FasterLogTestBase.IteratorType iteratorType) { - var logSettings = new FasterLogSettings { PageSizeBits = 20, SegmentSizeBits = 20, LogDevice = device, LogChecksum = logChecksum, LogCommitManager = logCommitManager }; + var logSettings = new FasterLogSettings { PageSizeBits = 20, SegmentSizeBits = 20, LogDevice = device, LogChecksum = logChecksum, LogCommitManager = logCommitManager, TryRecoverLatest = false }; log = FasterLogTestBase.IsAsync(iteratorType) ? await FasterLog.CreateAsync(logSettings) : new FasterLog(logSettings); byte[] entry = new byte[entryLength]; diff --git a/cs/test/TestUtils.cs b/cs/test/TestUtils.cs index 73e130cb1..ce9bc591a 100644 --- a/cs/test/TestUtils.cs +++ b/cs/test/TestUtils.cs @@ -87,8 +87,8 @@ internal static void RecreateDirectory(string path) internal static void IgnoreIfNotRunningAzureTests() { // Need this environment variable set AND Azure Storage Emulator running - if (!IsRunningAzureTests) - Assert.Ignore("Environment variable RunAzureTests is not defined"); + //if (!IsRunningAzureTests) + // Assert.Ignore("Environment variable RunAzureTests is not defined"); } // Used to test the various devices by using the same test with VALUES parameter From 0ce875ce41d421879a581c61d5486e20cfc81872 Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Sat, 18 Dec 2021 18:13:00 -0800 Subject: [PATCH 2/5] undo local change --- cs/test/TestUtils.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cs/test/TestUtils.cs b/cs/test/TestUtils.cs index ce9bc591a..c4c2216ac 100644 --- a/cs/test/TestUtils.cs +++ b/cs/test/TestUtils.cs @@ -86,9 +86,9 @@ internal static void RecreateDirectory(string path) internal static void IgnoreIfNotRunningAzureTests() { - // Need this environment variable set AND Azure Storage Emulator running - //if (!IsRunningAzureTests) - // Assert.Ignore("Environment variable RunAzureTests is not defined"); + Need this environment variable set AND Azure Storage Emulator running + if (!IsRunningAzureTests) + Assert.Ignore("Environment variable RunAzureTests is not defined"); } // Used to test the various devices by using the same test with VALUES parameter From 144c5183ed9dfc5befa51abd0e2ff5e4bacc995d Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Sat, 18 Dec 2021 18:13:18 -0800 Subject: [PATCH 3/5] umm --- cs/test/TestUtils.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cs/test/TestUtils.cs b/cs/test/TestUtils.cs index c4c2216ac..73e130cb1 100644 --- a/cs/test/TestUtils.cs +++ b/cs/test/TestUtils.cs @@ -86,7 +86,7 @@ internal static void RecreateDirectory(string path) internal static void IgnoreIfNotRunningAzureTests() { - Need this environment variable set AND Azure Storage Emulator running + // Need this environment variable set AND Azure Storage Emulator running if (!IsRunningAzureTests) Assert.Ignore("Environment variable RunAzureTests is not defined"); } From df84b5c2d2321d3776b9bade4f070a1ed42e212b Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Sat, 18 Dec 2021 18:29:51 -0800 Subject: [PATCH 4/5] fixes --- cs/src/core/FasterLog/FasterLogIterator.cs | 2 +- cs/test/FasterLogRecoverReadOnlyTests.cs | 2 +- cs/test/FasterLogTests.cs | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cs/src/core/FasterLog/FasterLogIterator.cs b/cs/src/core/FasterLog/FasterLogIterator.cs index 496c8be37..77744a2c9 100644 --- a/cs/src/core/FasterLog/FasterLogIterator.cs +++ b/cs/src/core/FasterLog/FasterLogIterator.cs @@ -36,7 +36,7 @@ public sealed class FasterLogScanIterator : ScanIteratorBase, IDisposable /// Whether iteration has ended, either because we reached the end address of iteration, or because /// we reached the end of a completed log. /// - public bool Ended => (currentAddress >= endAddress) || (fasterLog.LogCompleted && currentAddress == fasterLog.TailAddress); + public bool Ended => (nextAddress >= endAddress) || (fasterLog.LogCompleted && nextAddress == fasterLog.TailAddress); /// /// Constructor diff --git a/cs/test/FasterLogRecoverReadOnlyTests.cs b/cs/test/FasterLogRecoverReadOnlyTests.cs index c56191028..126dfc53b 100644 --- a/cs/test/FasterLogRecoverReadOnlyTests.cs +++ b/cs/test/FasterLogRecoverReadOnlyTests.cs @@ -51,7 +51,7 @@ public void TearDown() public async Task RecoverReadOnlyCheck1([Values] bool isAsync) { using var device = Devices.CreateLogDevice(deviceName); - var logSettings = new FasterLogSettings { LogDevice = device, MemorySizeBits = 11, PageSizeBits = 9, MutableFraction = 0.5, SegmentSizeBits = 9 }; + var logSettings = new FasterLogSettings { LogDevice = device, MemorySizeBits = 11, PageSizeBits = 9, MutableFraction = 0.5, SegmentSizeBits = 9, TryRecoverLatest = false }; using var log = isAsync ? await FasterLog.CreateAsync(logSettings) : new FasterLog(logSettings); await Task.WhenAll(ProducerAsync(log, cts), diff --git a/cs/test/FasterLogTests.cs b/cs/test/FasterLogTests.cs index 2cafdc1f7..c67585d94 100644 --- a/cs/test/FasterLogTests.cs +++ b/cs/test/FasterLogTests.cs @@ -188,7 +188,7 @@ internal class FasterLogGeneralTests : FasterLogTestBase public async ValueTask FasterLogTest1([Values] LogChecksumType logChecksum, [Values] IteratorType iteratorType) { device = Devices.CreateLogDevice(path + "fasterlog.log", deleteOnClose: true); - var logSettings = new FasterLogSettings { LogDevice = device, LogChecksum = logChecksum, LogCommitManager = manager }; + var logSettings = new FasterLogSettings { LogDevice = device, LogChecksum = logChecksum, LogCommitManager = manager, TryRecoverLatest = false }; log = IsAsync(iteratorType) ? await FasterLog.CreateAsync(logSettings) : new FasterLog(logSettings); byte[] entry = new byte[entryLength]; @@ -255,7 +255,7 @@ public async ValueTask TryEnqueue1([Values] LogChecksumType logChecksum, [Values CancellationToken token = cts.Token; device = Devices.CreateLogDevice(path + "fasterlog.log", deleteOnClose: true); - var logSettings = new FasterLogSettings { LogDevice = device, LogChecksum = logChecksum, LogCommitManager = manager }; + var logSettings = new FasterLogSettings { LogDevice = device, LogChecksum = logChecksum, LogCommitManager = manager, TryRecoverLatest = false }; log = IsAsync(iteratorType) ? await FasterLog.CreateAsync(logSettings) : new FasterLog(logSettings); const int dataLength = 1000; @@ -302,7 +302,7 @@ public async ValueTask TryEnqueue2([Values] LogChecksumType logChecksum, [Values string filename = path + "TryEnqueue2" + deviceType.ToString() + ".log"; device = TestUtils.CreateTestDevice(deviceType, filename); - var logSettings = new FasterLogSettings { LogDevice = device, PageSizeBits = 14, LogChecksum = logChecksum, LogCommitManager = manager, SegmentSizeBits = 22 }; + var logSettings = new FasterLogSettings { LogDevice = device, PageSizeBits = 14, LogChecksum = logChecksum, LogCommitManager = manager, SegmentSizeBits = 22, TryRecoverLatest = false }; log = IsAsync(iteratorType) ? await FasterLog.CreateAsync(logSettings) : new FasterLog(logSettings); const int dataLength = 10000; @@ -375,7 +375,7 @@ public async ValueTask TruncateUntilBasic([Values] LogChecksumType logChecksum, string filename = path + "TruncateUntilBasic" + deviceType.ToString() + ".log"; device = TestUtils.CreateTestDevice(deviceType, filename); - var logSettings = new FasterLogSettings { LogDevice = device, PageSizeBits = 14, LogChecksum = logChecksum, LogCommitManager = manager, SegmentSizeBits = 22 }; + var logSettings = new FasterLogSettings { LogDevice = device, PageSizeBits = 14, LogChecksum = logChecksum, LogCommitManager = manager, SegmentSizeBits = 22, TryRecoverLatest = false }; log = IsAsync(iteratorType) ? await FasterLog.CreateAsync(logSettings) : new FasterLog(logSettings); byte[] data1 = new byte[100]; @@ -463,7 +463,7 @@ public async ValueTask EnqueueAndWaitForCommitAsyncBasicTest([Values] LogChecksu public async ValueTask TruncateUntil2([Values] LogChecksumType logChecksum, [Values] IteratorType iteratorType) { device = Devices.CreateLogDevice(path + "fasterlog.log", deleteOnClose: true); - var logSettings = new FasterLogSettings { LogDevice = device, MemorySizeBits = 20, PageSizeBits = 14, LogChecksum = logChecksum, LogCommitManager = manager }; + var logSettings = new FasterLogSettings { LogDevice = device, MemorySizeBits = 20, PageSizeBits = 14, LogChecksum = logChecksum, LogCommitManager = manager, TryRecoverLatest = false }; log = IsAsync(iteratorType) ? await FasterLog.CreateAsync(logSettings) : new FasterLog(logSettings); byte[] data1 = new byte[1000]; @@ -658,7 +658,7 @@ public async ValueTask CommitAsyncPrevTask([Values] TestUtils.DeviceType deviceT string filename = $"{path}/CommitAsyncPrevTask_{deviceType}.log"; device = TestUtils.CreateTestDevice(deviceType, filename); - var logSettings = new FasterLogSettings { LogDevice = device, LogCommitManager = manager, SegmentSizeBits = 22 }; + var logSettings = new FasterLogSettings { LogDevice = device, LogCommitManager = manager, SegmentSizeBits = 22, TryRecoverLatest = false }; log = await FasterLog.CreateAsync(logSettings); // make it small since launching each on separate threads From 0fa29895e1411a170858c503a163d512ed7ca4b4 Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Sat, 18 Dec 2021 18:43:32 -0800 Subject: [PATCH 5/5] fix --- cs/src/core/FasterLog/FasterLogIterator.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cs/src/core/FasterLog/FasterLogIterator.cs b/cs/src/core/FasterLog/FasterLogIterator.cs index 77744a2c9..7af79bb73 100644 --- a/cs/src/core/FasterLog/FasterLogIterator.cs +++ b/cs/src/core/FasterLog/FasterLogIterator.cs @@ -517,8 +517,8 @@ internal unsafe bool ScanForwardForCommit(ref FasterLogRecoveryInfo info, long c try { // Continue looping until we find a record that is a commit record - while (GetNextInternal(out long physicalAddress, out var entryLength, out currentAddress, - out nextAddress, + while (GetNextInternal(out long physicalAddress, out var entryLength, out long currentAddress, + out long nextAddress, out var isCommitRecord)) { if (!isCommitRecord) continue;