Skip to content

Cleanup of async log posts and retry logic #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 14, 2017
Merged

Cleanup of async log posts and retry logic #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 11 additions & 24 deletions Src/StackifyLib/Internal/Logs/LogClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,11 @@
using System;
using System.Collections.Concurrent;
using Newtonsoft.Json;
using StackifyLib.Models;
using StackifyLib.Utils;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
//using System.Web.Configuration;
using Newtonsoft.Json;
using StackifyLib.Models;
using StackifyLib.Utils;

namespace StackifyLib.Internal.Logs
{
Expand Down Expand Up @@ -295,7 +291,7 @@ private Models.LogMsgGroup CreateDefaultMsgGroup()
}


internal Task<HttpClient.StackifyWebResponse> SendLogsByGroups(LogMsg[] messages)
internal HttpClient.StackifyWebResponse SendLogsByGroups(LogMsg[] messages)
{
try
{
Expand All @@ -309,16 +305,12 @@ private Models.LogMsgGroup CreateDefaultMsgGroup()

if (_HttpClient.IsRecentError())
{
var tcs = new TaskCompletionSource<HttpClient.StackifyWebResponse>();
tcs.SetResult(new HttpClient.StackifyWebResponse() { Exception = new Exception("Unable to send logs at this time due to recent error: " + (_HttpClient.LastErrorMessage ?? "")) });
return tcs.Task;
return new HttpClient.StackifyWebResponse() { Exception = new Exception("Unable to send logs at this time due to recent error: " + (_HttpClient.LastErrorMessage ?? "")) };
}

if (!identified)
{
var tcs = new TaskCompletionSource<HttpClient.StackifyWebResponse>();
tcs.SetResult(new HttpClient.StackifyWebResponse() { Exception = new Exception("Unable to send logs at this time. Unable to identify app") });
return tcs.Task;
return new HttpClient.StackifyWebResponse() { Exception = new Exception("Unable to send logs at this time. Unable to identify app") };
}

var groups = SplitLogsToGroups(messages);
Expand All @@ -338,28 +330,23 @@ private Models.LogMsgGroup CreateDefaultMsgGroup()
}

StackifyAPILogger.Log("Sending " + messages.Length.ToString() + " log messages via send multi groups");
var task =
_HttpClient.SendJsonAndGetResponseAsync(
var response =
_HttpClient.SendJsonAndGetResponse(
urlToUse,
jsonData, jsonData.Length > 5000);


messages = null;
groups = null;

return task;
return response;

}
catch (Exception ex)
{
Utils.StackifyAPILogger.Log(ex.ToString());

var tcs = new TaskCompletionSource<HttpClient.StackifyWebResponse>();
tcs.SetResult(new HttpClient.StackifyWebResponse() { Exception = ex });
return tcs.Task;
return new HttpClient.StackifyWebResponse() { Exception = ex };
}

return null;
}
}
}
121 changes: 46 additions & 75 deletions Src/StackifyLib/Internal/Logs/LogQueue.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
using System;
using StackifyLib.Models;
using StackifyLib.Utils;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Threading.Tasks;
using StackifyLib.Models;
using StackifyLib.Utils;

#if NET451 || NET45 || NET40
using System.Runtime.Remoting.Messaging;
Expand Down Expand Up @@ -298,19 +297,11 @@ private int FlushLoop()

bool keepGoing = false;

var tasks = new List<Task>();

int flushTimes = 0;
//Keep flushing
do
{
int count;
var task = FlushOnceAsync(out count);

if (task != null)
{
tasks.Add(task);
}
int count = FlushOnce();

if (count >= 100)
{
Expand All @@ -324,14 +315,6 @@ private int FlushLoop()
processedCount += count;
} while (keepGoing && flushTimes < 25);


if (_StopRequested && tasks.Any())
{
StackifyLib.Utils.StackifyAPILogger.Log("Waiting to ensure final log send. Waiting on " + tasks.Count + " tasks");
Task.WaitAll(tasks.ToArray(), 5000);
StackifyLib.Utils.StackifyAPILogger.Log("Final log flush complete");
}

_QueueTooBig = _MessageBuffer.Count < Logger.MaxLogBufferSize;
}
}
Expand All @@ -343,12 +326,12 @@ private int FlushLoop()
return processedCount;
}

private Task FlushOnceAsync(out int messageSize)
private int FlushOnce()
{

// StackifyLib.Utils.StackifyAPILogger.Log("Calling FlushOnceAsync");

messageSize = 0;
int messageSize = 0;
var chunk = new List<LogMsg>();

//we only want to do this once at a time but the actual send is done async
Expand Down Expand Up @@ -393,85 +376,44 @@ private Task FlushOnceAsync(out int messageSize)

if (chunk.Any())
{
var response = _LogClient.SendLogsByGroups(chunk.ToArray());

return _LogClient.SendLogsByGroups(chunk.ToArray()).ContinueWith((continuation) =>
if (response != null && response.Exception != null)
{
Utils.StackifyAPILogger.Log("Requeueing log messages due to error: " + response.Exception.ToString(), true);

if (continuation.Exception != null)
if (response.IsClientError())
{
Utils.StackifyAPILogger.Log("Requeueing log messages due to error: " + continuation.Exception.ToString(), true);
Utils.StackifyAPILogger.Log("Not requeueing log messages due to client error: " + response.StatusCode, true);
}

if (continuation.Result != null && continuation.Result.Exception != null)
{
Utils.StackifyAPILogger.Log("Requeueing log messages due to error: " + continuation.Result.Exception.ToString(), true);
}

if (continuation.Exception != null ||
(continuation.Result != null && continuation.Result.Exception != null))
else
{
try
{
bool messagesSentTooManyTimes = false;

foreach (var item in chunk)
{
item.UploadErrors++;

// try to upload up to 10 times
if (item.UploadErrors < 100)
{
_MessageBuffer.Enqueue(item);
}
else
{
messagesSentTooManyTimes = true;
}
}
bool messagesSentTooManyTimes = EnqueueForRetransmission(chunk);

if (messagesSentTooManyTimes)
{
Utils.StackifyAPILogger.Log(
"Some messages not queued again due to too many failures uploading");
}

}
catch (Exception ex2)
{
Utils.StackifyAPILogger.Log("Error trying to requeue messages " + ex2.ToString());
}
}
});
}
}
}
catch (Exception ex)
{
Utils.StackifyAPILogger.Log(ex.ToString());


//requeue the messages that errored trying to upload
try
{
foreach (var item in chunk)
{
item.UploadErrors++;

// try to upload up to 10 times
if (item.UploadErrors < 10)
{
_MessageBuffer.Enqueue(item);
}
}

}
catch (Exception ex2)
{
Utils.StackifyAPILogger.Log(ex2.ToString());
}
EnqueueForRetransmission(chunk);
}


return null;
return messageSize;
}


Expand Down Expand Up @@ -506,5 +448,34 @@ public void Pause(bool isPaused)
{
_PauseUpload = isPaused;
}

private bool EnqueueForRetransmission(List<LogMsg> chunk)
{
bool skippedMessage = false;

try
{
foreach (var item in chunk)
{
++item.UploadErrors;

// retry up to 5 times
if (item.UploadErrors < 5)
{
_MessageBuffer.Enqueue(item);
}
else
{
skippedMessage = true;
}
}
}
catch (Exception e)
{
Utils.StackifyAPILogger.Log(e.ToString());
}

return skippedMessage;
}
}
}
6 changes: 6 additions & 0 deletions Src/StackifyLib/Utils/HttpClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,12 @@ public class StackifyWebResponse
public string ResponseText { get; set; }
public System.Net.HttpStatusCode StatusCode { get; set; }
public Exception Exception { get; set; }

// return true if 4xx status code
public bool IsClientError()
{
return (HttpStatusCode.BadRequest <= StatusCode) && (StatusCode < HttpStatusCode.InternalServerError);
}
}

static HttpClient()
Expand Down