optimize trimming list and show retry info in dashboard

ChangeLog.md

@@ -1,6 +1,10 @@
 
 ## Change log
 
+### v1.9.15
+- Fix Slow operations reported by Atlas #374
+- Fix Dashboard does not show exception for retries #375
+
 ### v1.9.14
 - Add index needed for improved performance of MongoMonitoringApi.GetQueues
 

src/Hangfire.Mongo.Sample.ASPNetCore/Controllers/HomeController.cs

@@ -32,6 +32,23 @@ public void Execute(int index)
             }
         }
 
+        public class RandomExceptionJobHandler
+        {
+            public void Execute(int index)
+            {
+                Thread.Sleep(200);
+                var rand = new Random();
+                var next = rand.Next(1, 100);
+                var shouldThrow = next > 50;
+                if (shouldThrow)
+                {
+                    throw new InvalidOperationException($"Throwing exception as {next} > 50 = true");
+                }
+                Debug.WriteLine(
+                    $@"Hangfire random exception task started ({index}) - [{Thread.CurrentThread.ManagedThreadId}]");
+            }
+        }
+
         public ActionResult Index()
         {
             return View();
@@ -42,6 +59,12 @@ public ActionResult LongRunning(int id, int iterations)
             BackgroundJob.Enqueue<LongRunningJobHandler>(j => j.Execute(id, iterations)); 
             return RedirectToAction("Index");
         }
+
+        public ActionResult RandomException(int id)
+        {
+            BackgroundJob.Enqueue<RandomExceptionJobHandler>(j => j.Execute(id)); 
+            return RedirectToAction("Index");
+        }
 
         public ActionResult FireAndForget(int id)
         {

src/Hangfire.Mongo.Sample.ASPNetCore/Hangfire.Mongo.Sample.ASPNetCore.csproj

@@ -2,7 +2,7 @@
   <PropertyGroup>
     <TargetFramework>net6.0</TargetFramework>
     <Description />
-    <Version>1.9.14</Version>
+    <Version>1.9.15</Version>
     <Description>MongoDB storage implementation for Hangfire (background job system for ASP.NET applications).</Description>
     <Copyright>Copyright © 2014-2019 Sergey Zwezdin, Martin Lobger, Jonas Gottschau</Copyright>
     <Authors>Sergey Zwezdin, Martin Lobger, Jonas Gottschau</Authors>

src/Hangfire.Mongo.Sample.ASPNetCore/Views/Home/Index.cshtml

@@ -9,11 +9,11 @@
 </div>
 
 <div class="row">
-<div class="col-md-4">
-        <h2>Long running tasks</h2>
-    <p><a class="btn btn-success" href="@Url.Action("LongRunning", new {id = 1, iterations = 100})">
-        Run 1 long running task &raquo;
-    </a></p>
+    <div class="col-md-4">
+        <h2>Misc tasks</h2>
+        <p><a class="btn btn-success" href="@Url.Action("LongRunning", new {id = 1, iterations = 100})">Run 1 long running task &raquo;</a></p>
+        <p><a class="btn btn-success" href="@Url.Action("RandomException", new {id = 1})">Run 1 random failing task &raquo;</a></p>
+        <p><a class="btn btn-success" href="@Url.Action("Recurring")">Run 1 recurring task &raquo;</a></p>
     </div>
     <div class="col-md-4">
         <h2>Fire-and-forget tasks</h2>
@@ -31,8 +31,4 @@
         <p><a class="btn btn-success" href="@Url.Action("Delayed", new {id = 250})">Run 250 task &raquo;</a></p>
         <p><a class="btn btn-success" href="@Url.Action("Delayed", new {id = 1000})">Run 1000 task &raquo;</a></p>
     </div>
-    <div class="col-md-4">
-        <h2>Recurring tasks</h2>
-        <p><a class="btn btn-success" href="@Url.Action("Recurring")">Run &raquo;</a></p>
-    </div>
 </div>

src/Hangfire.Mongo.Sample.CosmosDB/Hangfire.Mongo.Sample.CosmosDB.csproj

@@ -2,7 +2,7 @@
   <PropertyGroup>
     <TargetFramework>net6.0</TargetFramework>
     <Description />
-    <Version>1.9.14</Version>
+    <Version>1.9.15</Version>
     <Description>MongoDB storage implementation for Hangfire (background job system for ASP.NET applications).</Description>
     <Copyright>Copyright © 2014-2019 Sergey Zwezdin, Martin Lobger, Jonas Gottschau</Copyright>
     <Authors>Sergey Zwezdin, Martin Lobger, Jonas Gottschau</Authors>

src/Hangfire.Mongo.Sample.NETCore/Hangfire.Mongo.Sample.NETCore.csproj

@@ -4,7 +4,7 @@
     <AssemblyName>Hangfire.Mongo.Sample.NETCore</AssemblyName>
     <OutputType>Exe</OutputType>
     <PackageId>Hangfire.Mongo.Sample.NETCore</PackageId>
-    <Version>1.9.14</Version>
+    <Version>1.9.15</Version>
     <Description>MongoDB storage implementation for Hangfire (background job system for ASP.NET applications).</Description>
     <Copyright>Copyright © 2014-2019 Sergey Zwezdin, Martin Lobger, Jonas Gottschau</Copyright>
     <Authors>Sergey Zwezdin, Martin Lobger, Jonas Gottschau</Authors>

src/Hangfire.Mongo.Tests/MongoWriteOnlyTransactionFacts.cs

@@ -541,40 +541,125 @@ public void RemoveFromList_DoesNotRemoveRecords_WithSameValue_ButDifferentKey()
         [Fact]
         public void TrimList_TrimsAList_ToASpecifiedRange()
         {
+            // ARRANGE
             Commit(x =>
             {
                 x.InsertToList("my-key", "0");
                 x.InsertToList("my-key", "1");
+
+            });
+
+            // ACT
+            Commit(x =>
+            {
                 x.InsertToList("my-key", "2");
                 x.InsertToList("my-key", "3");
                 x.TrimList("my-key", 1, 2);
             });
+
+            // ASSERT
+            var records = _database
+                .JobGraph
+                .Find(new BsonDocument("_t", nameof(ListDto)))
+                .ToList()
+                .Select(b => new ListDto(b))
+                .ToList();
 
-            var records = _database.JobGraph.Find(new BsonDocument("_t", nameof(ListDto))).ToList().Select(b => new ListDto(b)).ToList();
+            Assert.Equal(2, records.Count);
+            Assert.Equal("1", records[0].Value);
+            Assert.Equal("2", records[1].Value);
+        }
+
+        [Fact]
+        public void TrimList_TrimsAListInSameTransaction_ToASpecifiedRange()
+        {
+            // ARRANGE
+            Commit(x =>
+            {
+                x.InsertToList("my-key", "0");
+                x.InsertToList("my-key", "1");
+                x.InsertToList("my-key", "2");
+                x.InsertToList("my-key", "3");
+
+
+
+
+                // ACT
+                x.TrimList("my-key", 1, 2);
+            });
+
+            // ASSERT
+            var records = _database
+                .JobGraph
+                .Find(new BsonDocument("_t", nameof(ListDto)))
+                .ToList()
+                .Select(b => new ListDto(b))
+                .ToList();
 
             Assert.Equal(2, records.Count);
             Assert.Equal("1", records[0].Value);
             Assert.Equal("2", records[1].Value);
         }
 
         [Fact]
-        public void TrimList_RemovesRecordsToEnd_IfKeepAndingAt_GreaterThanMaxElementIndex()
+        public void TrimList_GreaterThanMaxElementIndex_RemovesRecordsToEndIfKeepEndingAt()
         {
             Commit(x =>
             {
                 x.InsertToList("my-key", "0");
                 x.InsertToList("my-key", "1");
                 x.InsertToList("my-key", "2");
+                x.InsertToList("my-key1", "1");
+                x.InsertToList("my-key1", "2");
+            });
+
+            Commit(x =>
+            {
                 x.TrimList("my-key", 1, 100);
             });
 
-            var recordCount = _database.JobGraph.Count(new BsonDocument("_t", nameof(ListDto)));
+            var filter = new BsonDocument
+            {
+                ["_t"] = nameof(ListDto),
+                [nameof(ListDto.Item)] = "my-key"
+            };
+            var recordCount = _database.JobGraph.Count(filter);
+
+            Assert.Equal(2, recordCount);
+        }
+
+        [Fact]
+        public void TrimList_GreaterThanMaxElementIndexSameTransaction_RemovesRecordsToEndIfKeepEndingAt()
+        {
+            Commit(x =>
+            {
+                x.InsertToList("my-key", "0");
+                x.InsertToList("my-key", "1");
+                x.InsertToList("my-key1", "1");
+                x.InsertToList("my-key1", "2");
+            });
+
+            Commit(x =>
+            {
+                x.InsertToList("my-key1", "3");
+                x.InsertToList("my-key1", "4");
+                x.InsertToList("my-key", "2");
+                x.TrimList("my-key", 1, 100);
+            });
+
+            var filter = new BsonDocument
+            {
+                ["_t"] = nameof(ListDto),
+                [nameof(ListDto.Item)] = "my-key"
+            };
+
+            var recordCount = _database.JobGraph.Count(filter);
 
             Assert.Equal(2, recordCount);
         }
 
         [Fact]
-        public void TrimList_RemovesAllRecords_WhenStartingFromValue_GreaterThanMaxElementIndex()
+        public void TrimList_StartingFromValue_RemovesAllRecordsGreaterThanMaxElementIndex()
         {
             Commit(x =>
             {
@@ -588,21 +673,71 @@ public void TrimList_RemovesAllRecords_WhenStartingFromValue_GreaterThanMaxEleme
         }
 
         [Fact]
-        public void TrimList_RemovesAllRecords_IfStartFromGreaterThanEndingAt()
+        public void TrimList_StartFromGreaterThanEndingAt_RemovesAllRecords()
         {
+            // ARRANGE
             Commit(x =>
             {
                 x.InsertToList("my-key", "0");
+
+            });
+
+            // ACT
+            Commit(x =>
+            {
                 x.TrimList("my-key", 1, 0);
             });
 
+            // ASSERT
             var recordCount = _database.JobGraph.Count(new BsonDocument("_t", nameof(ListDto)));
 
             Assert.Equal(0, recordCount);
         }
+
+        [Fact]
+        public void TrimList_StartFromGreaterThanEndingAtSameTransaction_RemovesAllRecords()
+        {
+            // ARRANGE
+            Commit(x =>
+            {
+                x.InsertToList("my-key", "0");
+
+
+
+
+                // ACT
+                x.TrimList("my-key", 1, 0);
+            });
 
+            // ASSERT
+            var recordCount = _database.JobGraph.Count(new BsonDocument("_t", nameof(ListDto)));
+
+            Assert.Equal(0, recordCount);
+        }
+
+        [Fact]
+        public void TrimList_DifferentKeys_RemovesRecordsOnlyOfGivenKey()
+        {
+            // ARRANGE
+            Commit(x =>
+            {
+                x.InsertToList("my-key", "0");
+            });
+
+            // ACT
+            Commit(x =>
+            {
+                x.TrimList("another-key", 1, 0);
+            });
+
+            // ASSERT
+            var recordCount = _database.JobGraph.Count(new BsonDocument("_t", nameof(ListDto)));
+
+            Assert.Equal(1, recordCount);
+        }
+
         [Fact]
-        public void TrimList_RemovesRecords_OnlyOfAGivenKey()
+        public void TrimList_DifferentKeysSameTransaction_RemovesRecordsOnlyOfGivenKey()
         {
             Commit(x =>
             {

src/Hangfire.Mongo/Hangfire.Mongo.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
     <PropertyGroup>
-        <VersionPrefix>1.9.14</VersionPrefix>
+        <VersionPrefix>1.9.15</VersionPrefix>
         <TargetFramework>netstandard2.0</TargetFramework>
         <NoWarn>$(NoWarn);CS0618</NoWarn>
         <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
@@ -21,8 +21,9 @@
         <owners>Sergey Zwezdin, Jonas Gottschau</owners>
         <Description>MongoDB storage implementation for Hangfire (background job system for ASP.NET applications).</Description>
         <PackageTags>Hangfire AspNet OWIN MongoDB CosmosDB Long-Running Background Fire-And-Forget Delayed Recurring Tasks Jobs Scheduler Threading Queues</PackageTags>
-        <PackageReleaseNotes>1.9.13
-            - Add index needed for improved performance of MongoMonitoringApi.GetQueues
+        <PackageReleaseNotes>1.9.15
+            - Fix Slow operations reported by Atlas #374
+            - Fix Dashboard does not show exception for retries #375
         </PackageReleaseNotes>
         <PackageReadmeFile>README.md</PackageReadmeFile>
         <!--<PackageLicenseUrl>https://raw.githubusercontent.com/sergun/Hangfire.Mongo/master/LICENSE</PackageLicenseUrl>-->

src/Hangfire.Mongo/MongoJobUpdates.cs

@@ -1,4 +1,5 @@
-using System.Linq;
+using System.Collections.Generic;
+using System.Linq;
 using MongoDB.Bson;
 using MongoDB.Driver;
 
@@ -13,11 +14,11 @@ public class MongoJobUpdates
     /// Set updates
     /// </summary>
     public BsonDocument Set { get; } = new();
-    
+
     /// <summary>
     /// Push updates
     /// </summary>
-    public BsonDocument Push { get; } = new();
+    public List<BsonDocument> Pushes { get; } = new();
 
     /// <summary>
     /// Creates a UpdateOneModel with a filter for the given job id
@@ -33,12 +34,17 @@ public UpdateOneModel<BsonDocument> CreateUpdateModel(string jobId)
             update["$set"] = Set;
         }
 
-        if (Push.Any())
+        if (Pushes.Any())
         {
-            update["$push"] = Push;
+            var pushByElement = Pushes
+                .SelectMany(p => p)
+                .GroupBy(elem => elem.Name)
+                .Select(g => new BsonElement(g.Key, new BsonDocument("$each", new BsonArray(g.Select(e => e.Value)))));
+
+            update["$push"] = new BsonDocument(pushByElement);
         }
-        
-        
+
+
         var updateModel = new UpdateOneModel<BsonDocument>(filter, update);
         return updateModel;
     }