Skip to content

Commit

Permalink
Add a retry to get deployment lock
Browse files Browse the repository at this point in the history
  • Loading branch information
sanchitmehta committed Dec 10, 2020
1 parent 7c9975c commit 3e1b46b
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 11 deletions.
22 changes: 17 additions & 5 deletions Kudu.Core/AllSafeLinuxLock.cs
Original file line number Diff line number Diff line change
Expand Up @@ -41,20 +41,26 @@ public bool IsHeld
}
catch (Exception ex)
{
_traceFactory.GetTracer().Trace("Error determining if deployment lock is valid");
_traceFactory.GetTracer().TraceError(ex);
// Exception where file is corrupt
// Wait for a second, if the file is being written
//Console.WriteLine("IsHeld - There was an Exception - Sleeping for a second ");
Thread.Sleep(1000);
exception = ex;
return IsLockValid();
if(IsLockValid())
{
exception = null;
return true;
}
}
finally
{
// There is some problem with reading the lock info file
// Avoid deadlock by releasing this lock/removing the dir
if (exception!=null)
{
//Console.WriteLine("IsHeld - there were exceptions twice -releasing the lock - ie deleting the lock directory");
_traceFactory.GetTracer().Trace("IsHeld - there were exceptions twice -releasing the lock - ie deleting the lock directory");
FileSystemHelpers.DeleteDirectorySafe(locksPath+"/deployment");
}
}
Expand Down Expand Up @@ -132,8 +138,14 @@ public bool Lock(string operationName)
return false;
}
}
_traceFactory.GetTracer().Trace("Acquired Deployment Lock");
CreateLockInfoFile(operationName);
if (!FileSystemHelpers.DirectoryExists(locksPath + "/deployment"))
{
_traceFactory.GetTracer().Trace("Deployment lock directory doesn't exist retrying");
CreateLockInfoFile(operationName);
}

_traceFactory.GetTracer().Trace("Acquired Deployment Lock");
return true;
}

Expand All @@ -155,13 +167,13 @@ public void Release()
if (FileSystemHelpers.DirectoryExists(locksPath+"/deployment"))
{
//Console.WriteLine("Releasing Lock - RemovingDir");
_traceFactory.GetTracer().Trace("Releasing Lock ");
_traceFactory.GetTracer().Trace("Releasing Deployment Lock");
FileSystemHelpers.DeleteDirectorySafe(locksPath+"/deployment");

}
else
{
Console.WriteLine("ReleasingLock - There is NO LOCK HELD | ERROR");
_traceFactory.GetTracer().Trace("Releasing Deployment Lock - There is NO LOCK HELD | ERROR");
}
}

Expand Down
17 changes: 11 additions & 6 deletions Kudu.Core/Deployment/DeploymentManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -895,15 +895,20 @@ private IDeploymentStatusFile VerifyDeployment(string id, bool isDeploying)
{
return statusFile;
}

// There's an incomplete deployment, yet nothing is going on, mark this deployment as failed
// since it probably means something died
// since it probably means something died, give a 60 seconds wait before marking it failed
if (!isDeploying)
{
_traceFactory.GetTracer().Step("Deployment Lock Failure");
ILogger logger = GetLogger(id);
logger.LogUnexpectedError();
//_traceFactory.GetTracer().Step("Unexpected Error "+statusFile.Message);
MarkStatusComplete(statusFile, success: false);
if (statusFile != null
&& statusFile.ReceivedTime != null
&& statusFile.ReceivedTime < DateTime.Now.AddSeconds(-60))
{
_traceFactory.GetTracer().Step("Deployment Lock Failure");
ILogger logger = GetLogger(id);
logger.LogUnexpectedError();
MarkStatusComplete(statusFile, success: false);
}
}

return statusFile;
Expand Down

0 comments on commit 3e1b46b

Please sign in to comment.