Skip to content

Commit 2094cad

Browse files
JakeGinnivanJakeGinnivan
authored
Merge pull request #920 from simonejsing/simon/concurrentcachewrite
Gracefully handle IOException on concurrent cache writes
2 parents 2e3d5e5 + 83becc0 commit 2094cad

File tree

9 files changed

+251
-13
lines changed

9 files changed

+251
-13
lines changed

‎src/GitVersionCore.Tests/GitVersionCore.Tests.csproj‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,8 @@
113113
<Compile Include="ConfigProviderTests.cs" />
114114
<Compile Include="GitVersionContextTests.cs" />
115115
<Compile Include="Helpers\DirectoryHelper.cs" />
116+
<Compile Include="Mocks\MockThreadSleep.cs" />
117+
<Compile Include="OperationWithExponentialBackoffTests.cs" />
116118
<Compile Include="Init\InitScenarios.cs" />
117119
<Compile Include="Init\InitStepsDefaultResponsesDoNotThrow.cs" />
118120
<Compile Include="Init\TestConsole.cs" />

‎src/GitVersionCore.Tests/Mocks/MockThreadSleep.cs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
using System;
2+
using GitVersion.Helpers;
3+
4+
public class MockThreadSleep : IThreadSleep
5+
{
6+
private Action<int> Validator;
7+
8+
public MockThreadSleep(Action<int> validator = null)
9+
{
10+
this.Validator = validator;
11+
}
12+
13+
public void Sleep(int milliseconds)
14+
{
15+
if (Validator != null)
16+
{
17+
Validator(milliseconds);
18+
}
19+
}
20+
}

‎src/GitVersionCore.Tests/OperationWithExponentialBackoffTests.cs‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
using System;
2+
using System.IO;
3+
using GitVersion.Helpers;
4+
using NUnit.Framework;
5+
using Shouldly;
6+
7+
[TestFixture]
8+
public class OperationWithExponentialBackoffTests
9+
{
10+
[Test]
11+
public void RetryOperationThrowsWhenNegativeMaxRetries()
12+
{
13+
Action action = () => new OperationWithExponentialBackoff<IOException>(new MockThreadSleep(), () => { }, -1);
14+
action.ShouldThrow<ArgumentOutOfRangeException>();
15+
}
16+
17+
[Test]
18+
public void RetryOperationThrowsWhenThreadSleepIsNull()
19+
{
20+
Action action = () => new OperationWithExponentialBackoff<IOException>(null, () => { });
21+
action.ShouldThrow<ArgumentNullException>();
22+
}
23+
24+
[Test]
25+
public void OperationIsNotRetriedOnInvalidException()
26+
{
27+
Action operation = () =>
28+
{
29+
throw new Exception();
30+
};
31+
32+
var retryOperation = new OperationWithExponentialBackoff<IOException>(new MockThreadSleep(), operation);
33+
Action action = () => retryOperation.Execute();
34+
action.ShouldThrow<Exception>();
35+
}
36+
37+
[Test]
38+
public void OperationIsRetriedOnIOException()
39+
{
40+
var operationCount = 0;
41+
42+
Action operation = () =>
43+
{
44+
operationCount++;
45+
if (operationCount < 2)
46+
{
47+
throw new IOException();
48+
}
49+
};
50+
51+
var retryOperation = new OperationWithExponentialBackoff<IOException>(new MockThreadSleep(), operation);
52+
retryOperation.Execute();
53+
54+
operationCount.ShouldBe(2);
55+
}
56+
57+
[Test]
58+
public void OperationIsRetriedAMaximumNumberOfTimes()
59+
{
60+
const int numberOfRetries = 3;
61+
var operationCount = 0;
62+
63+
Action operation = () =>
64+
{
65+
operationCount++;
66+
throw new IOException();
67+
};
68+
69+
var retryOperation = new OperationWithExponentialBackoff<IOException>(new MockThreadSleep(), operation, numberOfRetries);
70+
Action action = () => retryOperation.Execute();
71+
action.ShouldThrow<AggregateException>();
72+
73+
operationCount.ShouldBe(numberOfRetries + 1);
74+
}
75+
76+
[Test]
77+
public void OperationDelayDoublesBetweenRetries()
78+
{
79+
const int numberOfRetries = 3;
80+
var expectedSleepMSec = 500;
81+
var sleepCount = 0;
82+
83+
Action operation = () =>
84+
{
85+
throw new IOException();
86+
};
87+
88+
Action<int> validator = u =>
89+
{
90+
sleepCount++;
91+
u.ShouldBe(expectedSleepMSec);
92+
expectedSleepMSec *= 2;
93+
};
94+
95+
var retryOperation = new OperationWithExponentialBackoff<IOException>(new MockThreadSleep(validator), operation, numberOfRetries);
96+
Action action = () => retryOperation.Execute();
97+
action.ShouldThrow<AggregateException>();
98+
99+
sleepCount.ShouldBe(numberOfRetries);
100+
}
101+
102+
[Test]
103+
public void TotalSleepTimeForSixRetriesIsAboutThirtySeconds()
104+
{
105+
const int numberOfRetries = 6;
106+
int totalSleep = 0;
107+
108+
Action operation = () =>
109+
{
110+
throw new IOException();
111+
};
112+
113+
Action<int> validator = u =>
114+
{
115+
totalSleep += u;
116+
};
117+
118+
var retryOperation = new OperationWithExponentialBackoff<IOException>(new MockThreadSleep(validator), operation, numberOfRetries);
119+
Action action = () => retryOperation.Execute();
120+
action.ShouldThrow<AggregateException>();
121+
122+
// Exact number is 31,5 seconds
123+
totalSleep.ShouldBe(31500);
124+
}
125+
}

‎src/GitVersionCore/ExecuteCore.cs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ namespace GitVersion
22
{
33
using System;
44
using System.ComponentModel;
5+
using System.IO;
56
using System.Linq;
67
using GitVersion.Helpers;
78

@@ -57,7 +58,14 @@ public VersionVariables ExecuteGitVersion(string targetUrl, string dynamicReposi
5758
if (versionVariables == null)
5859
{
5960
versionVariables = ExecuteInternal(targetBranch, commitId, repo, gitPreparer, projectRoot, buildServer, overrideConfig: overrideConfig);
60-
gitVersionCache.WriteVariablesToDiskCache(repo, dotGitDirectory, versionVariables);
61+
try
62+
{
63+
gitVersionCache.WriteVariablesToDiskCache(repo, dotGitDirectory, versionVariables);
64+
}
65+
catch (AggregateException e)
66+
{
67+
Logger.WriteWarning(string.Format("One or more exceptions during cache write:{0}{1}", Environment.NewLine, e));
68+
}
6169
}
6270

6371
return versionVariables;

‎src/GitVersionCore/GitVersionCache.cs‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -24,23 +24,29 @@ public void WriteVariablesToDiskCache(IRepository repo, string gitDir, VersionVa
2424
var cacheFileName = GetCacheFileName(GetKey(repo, gitDir), GetCacheDir(gitDir));
2525
variablesFromCache.FileName = cacheFileName;
2626

27-
using (var stream = fileSystem.OpenWrite(cacheFileName))
27+
Dictionary<string, string> dictionary;
28+
using (Logger.IndentLog("Creating dictionary"))
2829
{
29-
using (var sw = new StreamWriter(stream))
30-
{
31-
Dictionary<string, string> dictionary;
32-
using (Logger.IndentLog("Creating dictionary"))
33-
{
34-
dictionary = variablesFromCache.ToDictionary(x => x.Key, x => x.Value);
35-
}
30+
dictionary = variablesFromCache.ToDictionary(x => x.Key, x => x.Value);
31+
}
3632

37-
using (Logger.IndentLog("Storing version variables to cache file " + cacheFileName))
33+
Action writeCacheOperation = () =>
34+
{
35+
using (var stream = fileSystem.OpenWrite(cacheFileName))
36+
{
37+
using (var sw = new StreamWriter(stream))
3838
{
39-
var serializer = new Serializer();
40-
serializer.Serialize(sw, dictionary);
39+
using (Logger.IndentLog("Storing version variables to cache file " + cacheFileName))
40+
{
41+
var serializer = new Serializer();
42+
serializer.Serialize(sw, dictionary);
43+
}
4144
}
4245
}
43-
}
46+
};
47+
48+
var retryOperation = new OperationWithExponentialBackoff<IOException>(new ThreadSleep(), writeCacheOperation, maxRetries: 6);
49+
retryOperation.Execute();
4450
}
4551

4652
public VersionVariables LoadVersionVariablesFromDiskCache(IRepository repo, string gitDir)

‎src/GitVersionCore/GitVersionCore.csproj‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,10 @@
122122
<Compile Include="GitVersionCache.cs" />
123123
<Compile Include="Helpers\FileSystem.cs" />
124124
<Compile Include="Helpers\IFileSystem.cs" />
125+
<Compile Include="Helpers\IThreadSleep.cs" />
126+
<Compile Include="Helpers\OperationWithExponentialBackoff.cs" />
125127
<Compile Include="Helpers\ServiceMessageEscapeHelper.cs" />
128+
<Compile Include="Helpers\ThreadSleep.cs" />
126129
<Compile Include="IncrementStrategyFinder.cs" />
127130
<Compile Include="OutputVariables\VersionVariables.cs" />
128131
<Compile Include="SemanticVersionExtensions.cs" />

‎src/GitVersionCore/Helpers/IThreadSleep.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
namespace GitVersion.Helpers
2+
{
3+
public interface IThreadSleep
4+
{
5+
void Sleep(int milliseconds);
6+
}
7+
}

‎src/GitVersionCore/Helpers/OperationWithExponentialBackoff.cs‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
using System;
2+
using System.Collections.Generic;
3+
4+
namespace GitVersion.Helpers
5+
{
6+
internal class OperationWithExponentialBackoff<T> where T : Exception
7+
{
8+
private IThreadSleep ThreadSleep;
9+
private Action Operation;
10+
private int MaxRetries;
11+
12+
public OperationWithExponentialBackoff(IThreadSleep threadSleep, Action operation, int maxRetries = 5)
13+
{
14+
if (threadSleep == null)
15+
throw new ArgumentNullException("threadSleep");
16+
if (maxRetries < 0)
17+
throw new ArgumentOutOfRangeException("maxRetries");
18+
19+
this.ThreadSleep = threadSleep;
20+
this.Operation = operation;
21+
this.MaxRetries = maxRetries;
22+
}
23+
24+
public void Execute()
25+
{
26+
var exceptions = new List<Exception>();
27+
28+
int tries = 0;
29+
int sleepMSec = 500;
30+
31+
while (tries <= MaxRetries)
32+
{
33+
tries++;
34+
35+
try
36+
{
37+
Operation();
38+
break;
39+
}
40+
catch (T e)
41+
{
42+
exceptions.Add(e);
43+
if (tries > MaxRetries)
44+
{
45+
throw new AggregateException("Operation failed after maximum number of retries were exceeded.", exceptions);
46+
}
47+
}
48+
49+
Logger.WriteInfo(string.Format("Operation failed, retrying in {0} milliseconds.", sleepMSec));
50+
ThreadSleep.Sleep(sleepMSec);
51+
sleepMSec *= 2;
52+
}
53+
}
54+
}
55+
}

‎src/GitVersionCore/Helpers/ThreadSleep.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
namespace GitVersion.Helpers
2+
{
3+
using System.Threading;
4+
5+
internal class ThreadSleep : IThreadSleep
6+
{
7+
public void Sleep(int milliseconds)
8+
{
9+
Thread.Sleep(milliseconds);
10+
}
11+
}
12+
}

0 commit comments

Comments
 (0)