add storage exception injection logic into fault injection test runner

src/Orleans.Transactions/State/TransactionQueue.cs

@@ -44,7 +44,6 @@ private class PMessages
 
         private TState stableState;
         private long stableSequenceNumber;
-
         public ReadWriteLock<TState> RWLock { get; }
         public CausalClock Clock { get; }
 

test/Transactions/Orleans.Transactions.Azure.Test/FaultInjection/FaultInjectionAzureTableTransactionStateStorage.cs

@@ -0,0 +1,85 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Orleans.Runtime;
+using Orleans.Transactions.Abstractions;
+using Orleans.Transactions.AzureStorage;
+using Orleans.Transactions.Tests.FaultInjection;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Options;
+using Orleans.Configuration;
+
+namespace Orleans.Transactions.Azure.Tests.FaultInjection
+{
+    public class FaultInjectionAzureTableTransactionStateStorage<TState> : ITransactionalStateStorage<TState>
+        where TState : class, new()
+    {
+        private readonly AzureTableTransactionalStateStorage<TState> stateStorage;
+        private readonly IControlledTransactionFaultInjector faultInjector;
+        public FaultInjectionAzureTableTransactionStateStorage(IControlledTransactionFaultInjector faultInjector,
+            AzureTableTransactionalStateStorage<TState> azureStateStorage)
+        {
+            this.faultInjector = faultInjector;
+            this.stateStorage = azureStateStorage;
+        }
+
+        public Task<TransactionalStorageLoadResponse<TState>> Load()
+        {
+            return this.stateStorage.Load();
+        }
+
+        public async Task<string> Store(
+
+            string expectedETag,
+            string metadata,
+
+            // a list of transactions to prepare.
+            List<PendingTransactionState<TState>> statesToPrepare,
+
+            // if non-null, commit all pending transaction up to and including this sequence number.
+            long? commitUpTo,
+
+            // if non-null, abort all pending transactions with sequence numbers strictly larger than this one.
+            long? abortAfter
+        )
+        {
+            faultInjector.BeforeStore();
+            var result = await this.stateStorage.Store(expectedETag, metadata, statesToPrepare, commitUpTo, abortAfter);
+            faultInjector.AfterStore();
+            return result;
+        }
+    }
+
+    public class FaultInjectionAzureTableTransactionStateStorageFactory : ITransactionalStateStorageFactory,
+        ILifecycleParticipant<ISiloLifecycle>
+    {
+        private readonly AzureTableTransactionalStateStorageFactory factory;
+
+        public static ITransactionalStateStorageFactory Create(IServiceProvider services, string name)
+        {
+            IOptionsSnapshot<AzureTableTransactionalStateOptions> optionsSnapshot = services.GetRequiredService<IOptionsSnapshot<AzureTableTransactionalStateOptions>>();
+            var azureFactory = ActivatorUtilities.CreateInstance<AzureTableTransactionalStateStorageFactory>(services, name, optionsSnapshot.Get(name));
+            return new FaultInjectionAzureTableTransactionStateStorageFactory(azureFactory);
+        }
+
+        public FaultInjectionAzureTableTransactionStateStorageFactory(
+            AzureTableTransactionalStateStorageFactory factory)
+        {
+            this.factory = factory;
+        }
+
+        public ITransactionalStateStorage<TState> Create<TState>(string stateName, IGrainActivationContext context) where TState : class, new()
+        {
+            var azureStateStorage = this.factory.Create<TState>(stateName, context);
+            return ActivatorUtilities.CreateInstance<FaultInjectionAzureTableTransactionStateStorage<TState>>(
+                context.ActivationServices, azureStateStorage);
+        }
+
+        public void Participate(ISiloLifecycle lifecycle)
+        {
+            this.factory.Participate(lifecycle);
+        }
+    }
+}

test/Transactions/Orleans.Transactions.Azure.Test/FaultInjection/HostingExtensions.cs

@@ -0,0 +1,42 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
+using Orleans.Configuration;
+using Orleans.Hosting;
+using Orleans.Providers;
+using Orleans.Runtime;
+using Orleans.Transactions.Abstractions;
+using Orleans.Transactions.AzureStorage;
+
+namespace Orleans.Transactions.Azure.Tests.FaultInjection
+{
+    public static class HostingExtensions
+    {
+
+        public static ISiloHostBuilder AddFaultInjectionAzureTableTransactionalStateStorage(this ISiloHostBuilder builder, Action<AzureTableTransactionalStateOptions> configureOptions)
+        {
+            return builder.AddFaultInjectionAzureTableTransactionalStateStorage(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME, configureOptions);
+        }
+
+        public static ISiloHostBuilder AddFaultInjectionAzureTableTransactionalStateStorage(this ISiloHostBuilder builder, string name, Action<AzureTableTransactionalStateOptions> configureOptions)
+        {
+            return builder.ConfigureServices(services => services.AddFaultInjectionAzureTableTransactionalStateStorage(name, ob => ob.Configure(configureOptions)));
+        }
+
+        private static IServiceCollection AddFaultInjectionAzureTableTransactionalStateStorage(this IServiceCollection services, string name,
+            Action<OptionsBuilder<AzureTableTransactionalStateOptions>> configureOptions = null)
+        {
+            configureOptions?.Invoke(services.AddOptions<AzureTableTransactionalStateOptions>(name));
+
+            services.TryAddSingleton<ITransactionalStateStorageFactory>(sp => sp.GetServiceByName<ITransactionalStateStorageFactory>(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME));
+            services.AddSingletonNamedService<ITransactionalStateStorageFactory>(name, FaultInjectionAzureTableTransactionStateStorageFactory.Create);
+            services.AddSingletonNamedService<ILifecycleParticipant<ISiloLifecycle>>(name, (s, n) => (ILifecycleParticipant<ISiloLifecycle>)s.GetRequiredServiceByName<ITransactionalStateStorageFactory>(n));
+
+            return services;
+        }
+    }
+}

test/Transactions/Orleans.Transactions.Azure.Test/FaultInjection/SimpleAzureStorageExceptionInjector.cs

@@ -0,0 +1,53 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.WindowsAzure.Storage;
+using Orleans.Transactions.Abstractions;
+using Orleans.Transactions.Tests.DeactivatingInjection;
+using Orleans.Transactions.Tests.FaultInjection;
+
+namespace Orleans.Transactions.Azure.Tests
+{
+    public class SimpleAzureStorageExceptionInjector : IControlledTransactionFaultInjector
+    {
+        public bool InjectBeforeStore { get; set; }
+        public bool InjectAfterStore { get; set; }
+        private int injectionBeforeStoreCounter = 0;
+        private int injectionAfterStoreCounter = 0;
+        private ILogger logger;
+        public SimpleAzureStorageExceptionInjector(ILogger<SimpleAzureStorageExceptionInjector> logger)
+        {
+            this.logger = logger;
+        }
+
+        public void AfterStore()
+        {
+            if (InjectAfterStore)
+            {
+                InjectAfterStore = false;
+                this.injectionAfterStoreCounter++;
+                this.logger.LogInformation($"Storage exception thrown after store, thrown total {injectionAfterStoreCounter}");
+                throw new SimpleAzureStorageException();
+            }
+
+        }
+
+        public void BeforeStore()
+        {
+            if (InjectBeforeStore)
+            {
+                InjectBeforeStore = false;
+                this.injectionBeforeStoreCounter++;
+                this.logger.LogInformation($"Storage exception thrown before store. Thrown total {injectionBeforeStoreCounter}");
+                throw new SimpleAzureStorageException();
+            }
+        }
+    }
+
+    public class SimpleAzureStorageException : StorageException
+    {
+    }
+}

test/Transactions/Orleans.Transactions.Azure.Test/TestFixture.cs

@@ -1,8 +1,12 @@
 using Orleans.Runtime;
+using Microsoft.Extensions.DependencyInjection;
 using Orleans.Hosting;
 using Orleans.TestingHost;
+using Orleans.Transactions.Azure.Tests;
+using Orleans.Transactions.Azure.Tests.FaultInjection;
 using Orleans.Transactions.Tests;
 using Orleans.Transactions.Tests.DeactivationTransaction;
+using Orleans.Transactions.Tests.FaultInjection;
 using TestExtensions;
 using Tester;
 
@@ -37,7 +41,7 @@ public void Configure(ISiloHostBuilder hostBuilder)
         }
     }
 
-    public class DeactivationTestFixture : BaseTestClusterFixture
+    public class FaultInjectionTestFixture : BaseTestClusterFixture
     {
         protected override void CheckPreconditionsOrThrow()
         {
@@ -56,12 +60,16 @@ public void Configure(ISiloHostBuilder hostBuilder)
             {
                 hostBuilder
                     .ConfigureTracingForTransactionTests()
-                    .AddAzureTableTransactionalStateStorage(TransactionTestConstants.TransactionStore, options =>
+                    .AddFaultInjectionAzureTableTransactionalStateStorage(TransactionTestConstants.TransactionStore, options =>
                     {
                         options.ConnectionString = TestDefaultConfiguration.DataConnectionString;
                     })
-                    .UseDeactivationTransactionState()
-                    .UseTransactions();
+                    .UseFaultInjectionTransactionState()
+                    .UseTransactions()
+                    .ConfigureServices(svc =>
+                    {
+                        svc.AddScoped<IControlledTransactionFaultInjector, SimpleAzureStorageExceptionInjector>();
+                    });
             }
         }
     }

test/Transactions/Orleans.Transactions.Azure.Test/TransactionDeactivationTests.cs → test/Transactions/Orleans.Transactions.Azure.Test/TransactionFaultInjectionTests.cs

@@ -10,10 +10,10 @@
 
 namespace Orleans.Transactions.Azure.Tests
 {
-    [TestCategory("Azure"), TestCategory("Transactions"), TestCategory("Functional")]
-    public class TransactionDeactivationTests : GrainDeactivationTransactionTestRunner, IClassFixture<DeactivationTestFixture>
+    [TestCategory("Azure"), TestCategory("Transactions")]
+    public class TransactionFaultInjectionTests : FaultInjectionTransactionTestRunner, IClassFixture<FaultInjectionTestFixture>
     {
-        public TransactionDeactivationTests(DeactivationTestFixture fixture, ITestOutputHelper output)
+        public TransactionFaultInjectionTests(FaultInjectionTestFixture fixture, ITestOutputHelper output)
             : base(fixture.GrainFactory, output)
         {
             fixture.EnsurePreconditionsMet();