Skip to content

Commit

Permalink
optimize trimming list and show retry info in dashboard
Browse files Browse the repository at this point in the history 
fix Dashboard does not show exception for retries (#375)
fix Slow operations reported by Atlas (#374)
  • Loading branch information
@gottscj
gottscj committed Nov 28, 2023
1 parent b411aa1 commit ff7b3df
Show file tree
Hide file tree
Showing 5 changed files with 207 additions and 33 deletions.
23 changes: 23 additions & 0 deletions src/Hangfire.Mongo.Sample.ASPNetCore/Controllers/HomeController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,23 @@ public void Execute(int index)
}
}

public class RandomExceptionJobHandler
{
public void Execute(int index)
{
Thread.Sleep(200);
var rand = new Random();
var next = rand.Next(1, 100);
var shouldThrow = next > 50;
if (shouldThrow)
{
throw new InvalidOperationException($"Throwing exception as {next} > 50 = true");
}
Debug.WriteLine(
$@"Hangfire random exception task started ({index}) - [{Thread.CurrentThread.ManagedThreadId}]");
}
}

public ActionResult Index()
{
return View();
Expand All @@ -42,6 +59,12 @@ public ActionResult LongRunning(int id, int iterations)
BackgroundJob.Enqueue<LongRunningJobHandler>(j => j.Execute(id, iterations));
return RedirectToAction("Index");
}

public ActionResult RandomException(int id)
{
BackgroundJob.Enqueue<RandomExceptionJobHandler>(j => j.Execute(id));
return RedirectToAction("Index");
}

public ActionResult FireAndForget(int id)
{
Expand Down
14 changes: 5 additions & 9 deletions src/Hangfire.Mongo.Sample.ASPNetCore/Views/Home/Index.cshtml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@
</div>

<div class="row">
<div class="col-md-4">
<h2>Long running tasks</h2>
<p><a class="btn btn-success" href="@Url.Action("LongRunning", new {id = 1, iterations = 100})">
Run 1 long running task &raquo;
</a></p>
<div class="col-md-4">
<h2>Misc tasks</h2>
<p><a class="btn btn-success" href="@Url.Action("LongRunning", new {id = 1, iterations = 100})">Run 1 long running task &raquo;</a></p>
<p><a class="btn btn-success" href="@Url.Action("RandomException", new {id = 1})">Run 1 random failing task &raquo;</a></p>
<p><a class="btn btn-success" href="@Url.Action("Recurring")">Run 1 recurring task &raquo;</a></p>
</div>
<div class="col-md-4">
<h2>Fire-and-forget tasks</h2>
Expand All @@ -31,8 +31,4 @@
<p><a class="btn btn-success" href="@Url.Action("Delayed", new {id = 250})">Run 250 task &raquo;</a></p>
<p><a class="btn btn-success" href="@Url.Action("Delayed", new {id = 1000})">Run 1000 task &raquo;</a></p>
</div>
<div class="col-md-4">
<h2>Recurring tasks</h2>
<p><a class="btn btn-success" href="@Url.Action("Recurring")">Run &raquo;</a></p>
</div>
</div>
147 changes: 141 additions & 6 deletions src/Hangfire.Mongo.Tests/MongoWriteOnlyTransactionFacts.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -541,40 +541,125 @@ public void RemoveFromList_DoesNotRemoveRecords_WithSameValue_ButDifferentKey()
[Fact]
public void TrimList_TrimsAList_ToASpecifiedRange()
{
// ARRANGE
Commit(x =>
{
x.InsertToList("my-key", "0");
x.InsertToList("my-key", "1");
});

// ACT
Commit(x =>
{
x.InsertToList("my-key", "2");
x.InsertToList("my-key", "3");
x.TrimList("my-key", 1, 2);
});

// ASSERT
var records = _database
.JobGraph
.Find(new BsonDocument("_t", nameof(ListDto)))
.ToList()
.Select(b => new ListDto(b))
.ToList();

var records = _database.JobGraph.Find(new BsonDocument("_t", nameof(ListDto))).ToList().Select(b => new ListDto(b)).ToList();
Assert.Equal(2, records.Count);
Assert.Equal("1", records[0].Value);
Assert.Equal("2", records[1].Value);
}

[Fact]
public void TrimList_TrimsAListInSameTransaction_ToASpecifiedRange()
{
// ARRANGE
Commit(x =>
{
x.InsertToList("my-key", "0");
x.InsertToList("my-key", "1");
x.InsertToList("my-key", "2");
x.InsertToList("my-key", "3");
// ACT
x.TrimList("my-key", 1, 2);
});

// ASSERT
var records = _database
.JobGraph
.Find(new BsonDocument("_t", nameof(ListDto)))
.ToList()
.Select(b => new ListDto(b))
.ToList();

Assert.Equal(2, records.Count);
Assert.Equal("1", records[0].Value);
Assert.Equal("2", records[1].Value);
}

[Fact]
public void TrimList_RemovesRecordsToEnd_IfKeepAndingAt_GreaterThanMaxElementIndex()
public void TrimList_GreaterThanMaxElementIndex_RemovesRecordsToEndIfKeepEndingAt()
{
Commit(x =>
{
x.InsertToList("my-key", "0");
x.InsertToList("my-key", "1");
x.InsertToList("my-key", "2");
x.InsertToList("my-key1", "1");
x.InsertToList("my-key1", "2");
});

Commit(x =>
{
x.TrimList("my-key", 1, 100);
});

var recordCount = _database.JobGraph.Count(new BsonDocument("_t", nameof(ListDto)));
var filter = new BsonDocument
{
["_t"] = nameof(ListDto),
[nameof(ListDto.Item)] = "my-key"
};
var recordCount = _database.JobGraph.Count(filter);

Assert.Equal(2, recordCount);
}

[Fact]
public void TrimList_GreaterThanMaxElementIndexSameTransaction_RemovesRecordsToEndIfKeepEndingAt()
{
Commit(x =>
{
x.InsertToList("my-key", "0");
x.InsertToList("my-key", "1");
x.InsertToList("my-key1", "1");
x.InsertToList("my-key1", "2");
});

Commit(x =>
{
x.InsertToList("my-key1", "3");
x.InsertToList("my-key1", "4");
x.InsertToList("my-key", "2");
x.TrimList("my-key", 1, 100);
});

var filter = new BsonDocument
{
["_t"] = nameof(ListDto),
[nameof(ListDto.Item)] = "my-key"
};

var recordCount = _database.JobGraph.Count(filter);

Assert.Equal(2, recordCount);
}

[Fact]
public void TrimList_RemovesAllRecords_WhenStartingFromValue_GreaterThanMaxElementIndex()
public void TrimList_StartingFromValue_RemovesAllRecordsGreaterThanMaxElementIndex()
{
Commit(x =>
{
Expand All @@ -588,21 +673,71 @@ public void TrimList_RemovesAllRecords_WhenStartingFromValue_GreaterThanMaxEleme
}

[Fact]
public void TrimList_RemovesAllRecords_IfStartFromGreaterThanEndingAt()
public void TrimList_StartFromGreaterThanEndingAt_RemovesAllRecords()
{
// ARRANGE
Commit(x =>
{
x.InsertToList("my-key", "0");
});

// ACT
Commit(x =>
{
x.TrimList("my-key", 1, 0);
});

// ASSERT
var recordCount = _database.JobGraph.Count(new BsonDocument("_t", nameof(ListDto)));

Assert.Equal(0, recordCount);
}

[Fact]
public void TrimList_StartFromGreaterThanEndingAtSameTransaction_RemovesAllRecords()
{
// ARRANGE
Commit(x =>
{
x.InsertToList("my-key", "0");
// ACT
x.TrimList("my-key", 1, 0);
});

// ASSERT
var recordCount = _database.JobGraph.Count(new BsonDocument("_t", nameof(ListDto)));

Assert.Equal(0, recordCount);
}

[Fact]
public void TrimList_DifferentKeys_RemovesRecordsOnlyOfGivenKey()
{
// ARRANGE
Commit(x =>
{
x.InsertToList("my-key", "0");
});

// ACT
Commit(x =>
{
x.TrimList("another-key", 1, 0);
});

// ASSERT
var recordCount = _database.JobGraph.Count(new BsonDocument("_t", nameof(ListDto)));

Assert.Equal(1, recordCount);
}

[Fact]
public void TrimList_RemovesRecords_OnlyOfAGivenKey()
public void TrimList_DifferentKeysSameTransaction_RemovesRecordsOnlyOfGivenKey()
{
Commit(x =>
{
Expand Down
20 changes: 13 additions & 7 deletions src/Hangfire.Mongo/MongoJobUpdates.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.Linq;
using System.Collections.Generic;
using System.Linq;
using MongoDB.Bson;
using MongoDB.Driver;

Expand All @@ -13,11 +14,11 @@ public class MongoJobUpdates
/// Set updates
/// </summary>
public BsonDocument Set { get; } = new();

/// <summary>
/// Push updates
/// </summary>
public BsonDocument Push { get; } = new();
public List<BsonDocument> Pushes { get; } = new();

/// <summary>
/// Creates a UpdateOneModel with a filter for the given job id
Expand All @@ -33,12 +34,17 @@ public UpdateOneModel<BsonDocument> CreateUpdateModel(string jobId)
update["$set"] = Set;
}

if (Push.Any())
if (Pushes.Any())
{
update["$push"] = Push;
var pushByElement = Pushes
.SelectMany(p => p)
.GroupBy(elem => elem.Name)
.Select(g => new BsonElement(g.Key, new BsonDocument("$each", new BsonArray(g.Select(e => e.Value)))));

update["$push"] = new BsonDocument(pushByElement);
}


var updateModel = new UpdateOneModel<BsonDocument>(filter, update);
return updateModel;
}
Expand Down
36 changes: 25 additions & 11 deletions src/Hangfire.Mongo/MongoWriteOnlyTransaction.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,10 @@ public override void SetJobState(string jobId, IState state)

var updates = GetOrAddJobUpdates(jobId);
updates.Set[nameof(JobDto.StateName)] = state.Name;
updates.Push[nameof(JobDto.StateHistory)] = stateDto;
updates.Pushes.Add(new BsonDocument
{
[nameof(JobDto.StateHistory)] = stateDto
});
}

public override void AddJobState(string jobId, IState state)
Expand All @@ -178,7 +181,10 @@ public override void AddJobState(string jobId, IState state)
}.Serialize();

var updates = GetOrAddJobUpdates(jobId);
updates.Push[nameof(JobDto.StateHistory)] = stateDto;
updates.Pushes.Add(new BsonDocument
{
[nameof(JobDto.StateHistory)] = stateDto
});
}

public override void SetJobParameter(string id, string name, string value)
Expand Down Expand Up @@ -334,19 +340,26 @@ public override void TrimList(string key, int keepStartingFrom, int keepEndingAt
var start = keepStartingFrom + 1;
var end = keepEndingAt + 1;

// get all ids
// get all ids for given key
var filter = new BsonDocument
{
["_t"] = nameof(ListDto),
[nameof(ListDto.Item)] = key
};
var allIds = DbContext.JobGraph
.Find(new BsonDocument("_t", nameof(ListDto)))
.Find(filter)
.Project(new BsonDocument("_id", 1))
.ToList()
.Select(b => b["_id"].AsObjectId)
.ToList();

// Add LisDto's scheduled for insertion writemodels collection, add it here.
allIds
.AddRange(_writeModels.OfType<InsertOneModel<BsonDocument>>()
.Where(model => ListDtoHasItem(key, model))
.Select(model => model.Document["_id"].AsObjectId));
var existing = _writeModels.OfType<InsertOneModel<BsonDocument>>()
.Where(model => ListDtoHasItem(key, model))
.Select(model => model.Document["_id"].AsObjectId)
.ToList();

allIds.AddRange(existing);

var toTrim = allIds
.OrderByDescending(id => id.Timestamp)
Expand All @@ -355,10 +368,11 @@ public override void TrimList(string key, int keepStartingFrom, int keepEndingAt
.Select(x => x.Id)
.ToList();

var filter = new BsonDocument
// toTrim1.AddRange(existing);

filter = new BsonDocument
{
["_id"] = new BsonDocument("$in", new BsonArray(toTrim)),
[nameof(ListDto.Item)] = key
["_id"] = new BsonDocument("$in", new BsonArray(toTrim))
};

var writeModel = new DeleteManyModel<BsonDocument>(filter);
Expand Down

0 comments on commit ff7b3df

Please sign in to comment.