diff --git a/src/Orleans.Core/Configuration/OptionLogger/DefaultOptionsFormatter.cs b/src/Orleans.Core/Configuration/OptionLogger/DefaultOptionsFormatter.cs index f7fbe4d074..456360a173 100644 --- a/src/Orleans.Core/Configuration/OptionLogger/DefaultOptionsFormatter.cs +++ b/src/Orleans.Core/Configuration/OptionLogger/DefaultOptionsFormatter.cs @@ -87,7 +87,7 @@ private IEnumerable FormatProperty(PropertyInfo property) } } - internal class DefaultOptionsFormatterResolver : IOptionFormatterResolver + internal class DefaultOptionsFormatterResolver : IOptionFormatterResolver where T: class, new() { private IOptionsSnapshot optionsSnapshot; diff --git a/src/Orleans.Core/Messaging/RequestInvocationHistory.cs b/src/Orleans.Core/Messaging/RequestInvocationHistory.cs index d7f6cdf05b..90ad9b8245 100644 --- a/src/Orleans.Core/Messaging/RequestInvocationHistory.cs +++ b/src/Orleans.Core/Messaging/RequestInvocationHistory.cs @@ -2,7 +2,7 @@ namespace Orleans.Runtime { - // used for tracking request invocation history for deadlock detection and call chain reentrancy + // used for tracking request invocation history for deadlock detection. [Serializable] internal sealed class RequestInvocationHistory { @@ -14,12 +14,12 @@ public RequestInvocationHistory(GrainId grainId, ActivationId activationId, stri { this.GrainId = grainId; this.ActivationId = activationId; - this.DebugContext = debugContext; + DebugContext = debugContext; } public override string ToString() { - return $"RequestInvocationHistory {GrainId}:{ActivationId}:{DebugContext}"; + return String.Format("RequestInvocationHistory {0}:{1}:{2}", GrainId, ActivationId, DebugContext); } } } diff --git a/src/Orleans.Runtime.Legacy/Configuration/LegacyClusterConfigurationExtensions.cs b/src/Orleans.Runtime.Legacy/Configuration/LegacyClusterConfigurationExtensions.cs index eb50cdbe16..05f9824226 100644 --- a/src/Orleans.Runtime.Legacy/Configuration/LegacyClusterConfigurationExtensions.cs +++ b/src/Orleans.Runtime.Legacy/Configuration/LegacyClusterConfigurationExtensions.cs @@ -192,7 +192,7 @@ private static void AddLegacyClusterConfigurationSupport(IServiceCollection serv var nodeConfig = configuration.GetOrCreateNodeConfigurationForSilo(siloOptions.Value.SiloName); options.ExcludedGrainTypes.AddRange(nodeConfig.ExcludedGrainTypes); }); - + services.AddOptions() .Configure((options, config) => { @@ -280,7 +280,7 @@ private static void AddLegacyClusterConfigurationSupport(IServiceCollection serv { options.IsRunningAsUnitTest = config.IsRunningAsUnitTest; }); - + services.AddOptions() .Configure((options, config) => { diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index d4de86b018..a5bc74b4c6 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -24,7 +24,7 @@ namespace Orleans.Runtime /// internal class ActivationData : IGrainActivationContext, IActivationData, IInvokable, IDisposable { - // This class is used for activations that have extension invokers. It keeps a dictionary of + // This class is used for activations that have extension invokers. It keeps a dictionary of // invoker objects to use with the activation, and extend the default invoker // defined for the grain class. // Note that in all cases we never have more than one copy of an actual invoker; @@ -34,7 +34,7 @@ private class ExtensionInvoker : IGrainMethodInvoker, IGrainExtensionMap // Because calls to ExtensionInvoker are allways made within the activation context, // we rely on the single-threading guarantee of the runtime and do not protect the map with a lock. private Dictionary> extensionMap; // key is the extension interface ID - + /// /// Try to add an extension for the specific interface ID. /// Fail and return false if there is already an extension for that interface ID. @@ -171,7 +171,7 @@ internal class GrainActivationContextFactory private IGrainMethodInvoker lastInvoker; private IServiceScope serviceScope; private HashSet timers; - + public ActivationData( ActivationAddress addr, string genericArguments, @@ -264,6 +264,8 @@ internal bool TryGetExtensionHandler(Type extensionType, out IGrainExtension res return extensionInvoker != null && extensionInvoker.TryGetExtensionHandler(extensionType, out result); } + public HashSet RunningRequestsSenders { get; } = new HashSet(); + public ISchedulingContext SchedulingContext { get; } public string GrainTypeName @@ -311,14 +313,14 @@ private static void SetGrainActivationContextInScopedServices(IServiceProvider s var contextFactory = sp.GetRequiredService(); contextFactory.Context = context; } - + private Streams.StreamDirectory streamDirectory; internal Streams.StreamDirectory GetStreamDirectory() { return streamDirectory ?? (streamDirectory = new Streams.StreamDirectory()); } - internal bool IsUsingStreams + internal bool IsUsingStreams { get { return streamDirectory != null; } } @@ -341,7 +343,7 @@ GrainReference IActivationData.GrainReference { get { return GrainReference; } } - + public GrainId Identity { get { return Grain; } @@ -433,7 +435,7 @@ public void ResetCollectionTicket() public void SetCollectionTicket(DateTime ticket) { if (ticket == default(DateTime)) throw new ArgumentException("default(DateTime) is disallowed", "ticket"); - if (CollectionTicket != default(DateTime)) + if (CollectionTicket != default(DateTime)) { throw new InvalidOperationException("call ResetCollectionTicket before calling SetCollectionTicket."); } @@ -447,7 +449,7 @@ public void SetCollectionTicket(DateTime ticket) // Currently, the only supported multi-activation grain is one using the StatelessWorkerPlacement strategy. internal bool IsStatelessWorker => this.PlacedUsing is StatelessWorkerPlacement; - + /// /// Returns a value indicating whether or not this placement strategy requires activations to be registered in /// the grain directory. @@ -469,6 +471,12 @@ public void RecordRunning(Message message) // Note: This method is always called while holding lock on this activation, so no need for additional locks here numRunning++; + if (message.Direction != Message.Directions.OneWay + && message.SendingActivation != null + && !message.SendingGrain?.IsClient == true) + { + RunningRequestsSenders.Add(message.SendingActivation); + } if (Running != null) return; @@ -482,7 +490,7 @@ public void ResetRunning(Message message) { // Note: This method is always called while holding lock on this activation, so no need for additional locks here numRunning--; - + RunningRequestsSenders.Remove(message.SendingActivation); if (numRunning == 0) { becameIdle = DateTime.UtcNow; @@ -515,7 +523,7 @@ public void ResetRunning(Message message) /// Increment the number of in-flight messages currently being processed. public void IncrementInFlightCount() { Interlocked.Increment(ref inFlightCount); } - + /// Decrement the number of in-flight messages currently being processed. public void DecrementInFlightCount() { Interlocked.Decrement(ref inFlightCount); } @@ -524,14 +532,14 @@ public void ResetRunning(Message message) /// Decrement the number of messages currently in the process of being received. public void DecrementEnqueuedOnDispatcherCount() { Interlocked.Decrement(ref enqueuedOnDispatcherCount); } - + /// /// grouped by sending activation: responses first, then sorted by id /// private List waiting; - public int WaitingCount - { + public int WaitingCount + { get { return waiting == null ? 0 : waiting.Count; @@ -594,7 +602,7 @@ public EnqueueMessageResult EnqueueMessage(Message message) } /// - /// Check whether this activation is overloaded. + /// Check whether this activation is overloaded. /// Returns LimitExceededException if overloaded, otherwise nullc> /// /// Logger to use for reporting any overflow condition @@ -617,7 +625,7 @@ public LimitExceededException CheckOverloaded(ILogger log) if (maxRequestsHardLimit > 0 && count > maxRequestsHardLimit) // Hard limit { - log.Warn(ErrorCode.Catalog_Reject_ActivationTooManyRequests, + log.Warn(ErrorCode.Catalog_Reject_ActivationTooManyRequests, String.Format("Overload - {0} enqueued requests for activation {1}, exceeding hard limit rejection threshold of {2}", count, this, maxRequestsHardLimit)); @@ -691,7 +699,7 @@ public TimeSpan GetIdleness(DateTime now) { if (now == default(DateTime)) throw new ArgumentException("default(DateTime) is not allowed; Use DateTime.UtcNow instead.", "now"); - + return now - becameIdle; } @@ -795,7 +803,7 @@ public void OnTimerDisposed(IGrainTimer orleansTimerInsideGrain) internal Task WaitForAllTimersToFinish() { lock(this) - { + { if (timers == null) { return Task.CompletedTask; diff --git a/src/Orleans.Runtime/Catalog/Catalog.cs b/src/Orleans.Runtime/Catalog/Catalog.cs index 6af84b183a..824b4d9ad3 100644 --- a/src/Orleans.Runtime/Catalog/Catalog.cs +++ b/src/Orleans.Runtime/Catalog/Catalog.cs @@ -35,7 +35,7 @@ internal class NonExistentActivationException : Exception public NonExistentActivationException() : base("NonExistentActivationException") { } public NonExistentActivationException(string msg) : base(msg) { } - public NonExistentActivationException(string message, Exception innerException) + public NonExistentActivationException(string message, Exception innerException) : base(message, innerException) { } public NonExistentActivationException(string msg, ActivationAddress nonExistentActivation, bool isStatelessWorker) @@ -274,7 +274,7 @@ public List> GetGrainStatistics() // TODO: generic type expansion var grainTypeName = TypeUtils.GetFullName(data.GrainInstanceType); - + Dictionary grains; int n; if (!counts.TryGetValue(grainTypeName, out grains)) @@ -347,8 +347,8 @@ public DetailedGrainReport GetDetailedGrainReport(GrainId grain) } List acts = activations.FindTargets(grain); - report.LocalActivations = acts != null ? - acts.Select(activationData => activationData.ToDetailedString()).ToList() : + report.LocalActivations = acts != null ? + acts.Select(activationData => activationData.ToDetailedString()).ToList() : new List(); return report; } @@ -397,18 +397,18 @@ internal int UnregisterGrainForTesting(GrainId grain) int numActsBefore = acts.Count; foreach (var act in acts) UnregisterMessageTarget(act); - + return numActsBefore; } internal bool CanInterleave(ActivationId running, Message message) { - return this.TryGetActivationData(running, out ActivationData target) - && target.GrainInstance != null - && this.GrainTypeManager.TryGetData(TypeUtils.GetFullName(target.GrainInstanceType), out GrainTypeData data) - && (data.IsReentrant - || (message.GetDeserializedBody(this.serializationManager) is InvokeMethodRequest invokeMethodRequest - && data.MayInterleave((invokeMethodRequest)))); + ActivationData target; + GrainTypeData data; + return TryGetActivationData(running, out target) && + target.GrainInstance != null && + GrainTypeManager.TryGetData(TypeUtils.GetFullName(target.GrainInstanceType), out data) && + (data.IsReentrant || data.MayInterleave((InvokeMethodRequest)message.GetDeserializedBody(this.serializationManager))); } public void GetGrainTypeInfo(int typeCode, out string grainClass, out PlacementStrategy placement, out MultiClusterRegistrationStrategy activationStrategy, string genericArguments = null) @@ -449,7 +449,7 @@ public ActivationData GetOrCreateActivation( { return result; } - + int typeCode = address.Grain.TypeCode; string actualGrainType = null; MultiClusterRegistrationStrategy activationStrategy; @@ -502,7 +502,7 @@ public ActivationData GetOrCreateActivation( CounterStatistic.FindOrCreate(StatisticNames.CATALOG_ACTIVATION_NON_EXISTENT_ACTIVATIONS).Increment(); throw new NonExistentActivationException(msg, address, placement is StatelessWorkerPlacement); } - + SetupActivationInstance(result, grainType, genericArguments); activatedPromise = InitActivation(result, grainType, genericArguments, requestContextData); return result; @@ -714,7 +714,7 @@ private void CreateGrainInstance(string grainTypeName, ActivationData data, stri data.SetupContext(grainTypeData, this.serviceProvider); Grain grain = grainCreator.CreateGrainInstance(data); - + //if grain implements IStreamSubscriptionObserver, then install stream consumer extension on it if(grain is IStreamSubscriptionObserver) InstallStreamConsumerExtension(data, grain as IStreamSubscriptionObserver); @@ -722,7 +722,7 @@ private void CreateGrainInstance(string grainTypeName, ActivationData data, stri grain.Data = data; data.SetGrainInstance(grain); } - + activations.IncrementGrainCounter(grainClassName); if (logger.IsEnabled(LogLevel.Debug)) logger.Debug("CreateGrainInstance {0}{1}", data.Grain, data.ActivationId); @@ -932,11 +932,11 @@ private void DestroyActivationAsync(ActivationData activation, MultiTaskCompleti /// // Overall code flow: // Deactivating state was already set before, in the correct context under lock. - // that means no more new requests will be accepted into this activation and all timer were stopped (no new ticks will be delivered or enqueued) + // that means no more new requests will be accepted into this activation and all timer were stopped (no new ticks will be delivered or enqueued) // Wait for all already scheduled ticks to finish // CallGrainDeactivate // when AsyncDeactivate promise is resolved (NOT when all Deactivate turns are done, which may be orphan tasks): - // Unregister in the directory + // Unregister in the directory // when all AsyncDeactivate turns are done (Dispatcher.OnActivationCompletedRequest): // Set Invalid state // UnregisterMessageTarget -> no new tasks will be enqueue (if an orphan task get enqueud, it is ignored and dropped on the floor). @@ -1003,7 +1003,7 @@ private async void FinishDestroyActivations(List list, int numbe //logger.Info(ErrorCode.Catalog_DestroyActivations_Done, "Starting FinishDestroyActivations #{0} - with {1} Activations.", number, list.Count); // step 3 - UnregisterManyAsync try - { + { List activationsToDeactivate = list. Where((ActivationData d) => d.IsUsingGrainDirectory). Select((ActivationData d) => ActivationAddress.GetAddress(LocalSilo, d.Grain, d.ActivationId)).ToList(); @@ -1039,7 +1039,7 @@ await scheduler.RunOrQueueTask(() => // IMPORTANT: no more awaits and .Ignore after that point. - // Just use this opportunity to invalidate local Cache Entry as well. + // Just use this opportunity to invalidate local Cache Entry as well. // If this silo is not the grain directory partition for this grain, it may have it in its cache. try { @@ -1188,7 +1188,7 @@ private async Task CallGrainDeactivateAndCleanupStreams(Activati if (TryGetActivationData(activation.ActivationId, out ignore) && activation.State == ActivationState.Deactivating) { - RequestContext.Clear(); // Clear any previous RC, so it does not leak into this call by mistake. + RequestContext.Clear(); // Clear any previous RC, so it does not leak into this call by mistake. await activation.Lifecycle.OnStop().WithCancellation(ct); } if (logger.IsEnabled(LogLevel.Debug)) logger.Debug(ErrorCode.Catalog_AfterCallingDeactivate, "Returned from calling {1} grain's OnDeactivateAsync() method {0}", activation, grainTypeName); @@ -1233,7 +1233,7 @@ private struct ActivationRegistrationResult /// public static readonly ActivationRegistrationResult Success = new ActivationRegistrationResult { - IsSuccess = true + IsSuccess = true }; public ActivationRegistrationResult(ActivationAddress existingActivationAddress) @@ -1242,7 +1242,7 @@ public ActivationRegistrationResult(ActivationAddress existingActivationAddress) ExistingActivationAddress = existingActivationAddress; IsSuccess = false; } - + /// /// Returns true if this instance represents a successful registration, false otherwise. /// @@ -1264,13 +1264,13 @@ private async Task RegisterActivationInGrainDirect { ActivationAddress address = activation.Address; - // Currently, the only grain type that is not registered in the Grain Directory is StatelessWorker. + // Currently, the only grain type that is not registered in the Grain Directory is StatelessWorker. // Among those that are registered in the directory, we currently do not have any multi activations. if (activation.IsUsingGrainDirectory) { var result = await scheduler.RunOrQueueTask(() => directory.RegisterAsync(address, singleActivation:true), this.SchedulingContext); if (address.Equals(result.Address)) return ActivationRegistrationResult.Success; - + return new ActivationRegistrationResult(existingActivationAddress: result.Address); } else if (activation.PlacedUsing is StatelessWorkerPlacement stPlacement) @@ -1302,7 +1302,7 @@ private async Task RegisterActivationInGrainDirect } } - // We currently don't have any other case for multiple activations except for StatelessWorker. + // We currently don't have any other case for multiple activations except for StatelessWorker. } /// @@ -1388,7 +1388,7 @@ private List TryGetActivationDatas(List addre } private void OnSiloStatusChange(SiloAddress updatedSilo, SiloStatus status) - { + { // ignore joining events and also events on myself. if (updatedSilo.Equals(LocalSilo)) return; diff --git a/src/Orleans.Runtime/Configuration/Options/SchedulingOptions.cs b/src/Orleans.Runtime/Configuration/Options/SchedulingOptions.cs index 01ec590980..d84de246af 100644 --- a/src/Orleans.Runtime/Configuration/Options/SchedulingOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/SchedulingOptions.cs @@ -36,7 +36,7 @@ public class SchedulingOptions public static readonly TimeSpan DEFAULT_DELAY_WARNING_THRESHOLD = TimeSpan.FromMilliseconds(10000); // 10 seconds /// - /// ActivationSchedulingQuantum is a soft time limit on the duration of activation macro-turn (a number of micro-turns). + /// ActivationSchedulingQuantum is a soft time limit on the duration of activation macro-turn (a number of micro-turns). /// If an activation was running its micro-turns longer than this, we will give up the thread. /// If this is set to zero or a negative number, then the full work queue is drained (MaxWorkItemsPerTurn allowing). /// @@ -44,7 +44,7 @@ public class SchedulingOptions public static readonly TimeSpan DEFAULT_ACTIVATION_SCHEDULING_QUANTUM = TimeSpan.FromMilliseconds(100); /// - /// TurnWarningLengthThreshold is a soft time limit to generate trace warning when the micro-turn executes longer then this period in CPU. + /// TurnWarningLengthThreshold is a soft time limit to generate trace warning when the micro-turn executes longer then this period in CPU. /// public TimeSpan TurnWarningLengthThreshold { get; set; } = DEFAULT_TURN_WARNING_THRESHOLD; public static readonly TimeSpan DEFAULT_TURN_WARNING_THRESHOLD = TimeSpan.FromMilliseconds(200); diff --git a/src/Orleans.Runtime/Core/Dispatcher.cs b/src/Orleans.Runtime/Core/Dispatcher.cs index e676501f0b..9633ad9f41 100644 --- a/src/Orleans.Runtime/Core/Dispatcher.cs +++ b/src/Orleans.Runtime/Core/Dispatcher.cs @@ -33,8 +33,8 @@ internal class Dispatcher private readonly SchedulingOptions schedulingOptions; private readonly ILogger invokeWorkItemLogger; internal Dispatcher( - OrleansTaskScheduler scheduler, - ISiloMessageCenter transport, + OrleansTaskScheduler scheduler, + ISiloMessageCenter transport, Catalog catalog, IOptions messagingOptions, PlacementDirectorsManager placementDirectorsManager, @@ -93,10 +93,10 @@ public void ReceiveMessage(Message message) { Task ignore; ActivationData target = catalog.GetOrCreateActivation( - message.TargetAddress, + message.TargetAddress, message.IsNewPlacement, message.NewGrainType, - String.IsNullOrEmpty(message.GenericGrainType) ? null : message.GenericGrainType, + String.IsNullOrEmpty(message.GenericGrainType) ? null : message.GenericGrainType, message.RequestContextData, out ignore); @@ -125,7 +125,7 @@ public void ReceiveMessage(Message message) try { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation"); - + var nea = ex as Catalog.NonExistentActivationException; if (nea == null) { @@ -199,9 +199,9 @@ await this.localGrainDirectory.UnregisterAfterNonexistingActivation( } public void RejectMessage( - Message message, - Message.RejectionTypes rejectType, - Exception exc, + Message message, + Message.RejectionTypes rejectType, + Exception exc, string rejectInfo = null) { if (message.Direction == Message.Directions.Request) @@ -276,13 +276,13 @@ private void ReceiveRequest(Message message, ActivationData targetActivation) { try { - CheckDeadlock(targetActivation, message); + CheckDeadlock(message); } catch (DeadlockException exc) { // Record that this message is no longer flowing through the system MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Deadlock"); - logger.Warn(ErrorCode.Dispatcher_DetectedDeadlock, + logger.Warn(ErrorCode.Dispatcher_DetectedDeadlock, "Detected Application Deadlock: {0}", exc.Message); // We want to send DeadlockException back as an application exception, rather than as a system rejection. SendResponse(message, Response.ExceptionResponse(exc)); @@ -316,19 +316,18 @@ private bool ActivationMayAcceptRequest(ActivationData targetActivation, Message } /// - /// Whether an incoming message can interleave + /// Whether an incoming message can interleave /// /// /// /// public bool CanInterleave(ActivationData targetActivation, Message incoming) { - bool canInterleave = + bool canInterleave = incoming.IsAlwaysInterleave || targetActivation.Running == null - || targetActivation.Running.IsReadOnly && incoming.IsReadOnly - || schedulingOptions.AllowCallChainReentrancy - && IsMessageACallChainLoop(incoming) + || (targetActivation.Running.IsReadOnly && incoming.IsReadOnly) + || (schedulingOptions.AllowCallChainReentrancy && targetActivation.ActivationId.Equals(incoming.SendingActivation)) || catalog.CanInterleave(targetActivation.ActivationId, incoming); return canInterleave; @@ -337,78 +336,56 @@ public bool CanInterleave(ActivationData targetActivation, Message incoming) /// /// https://github.com/dotnet/orleans/issues/3184 /// Checks whether reentrancy is allowed for calls to grains that are already part of the call chain. - /// Covers following case: grain A calls grain B, and while executing the invoked method B calls back to A. + /// Covers following case: grain A calls grain B, and while executing the invoked method B calls back to A. /// Design: Senders collection `RunningRequestsSenders` contains sending grains references - /// during duration of request processing. If target of outgoing request is found in that collection - + /// during duration of request processing. If target of outgoing request is found in that collection - /// such request will be marked as interleaving in order to prevent deadlocks. /// private void MarkSameCallChainMessageAsInterleaving(ActivationData sendingActivation, Message outgoing) { - if (schedulingOptions.AllowCallChainReentrancy && IsMessageACallChainLoop(outgoing)) + if (!schedulingOptions.AllowCallChainReentrancy) + { + return; + } + + if (sendingActivation?.RunningRequestsSenders.Contains(outgoing.TargetActivation) == true) { outgoing.IsAlwaysInterleave = true; } } /// - /// Check if the current message will cause a loop in the existing call chain + /// Check if the current message will cause deadlock. + /// Throw DeadlockException if yes. /// /// Message to analyze - /// - private bool IsMessageACallChainLoop(Message message) + private void CheckDeadlock(Message message) { var requestContext = message.RequestContextData; - + object obj; if (requestContext == null || - !requestContext.TryGetValue(RequestContext.CALL_CHAIN_REQUEST_CONTEXT_HEADER, out object obj) || - obj == null) return false; // first call in a chain + !requestContext.TryGetValue(RequestContext.CALL_CHAIN_REQUEST_CONTEXT_HEADER, out obj) || + obj == null) return; // first call in a chain var prevChain = ((IList)obj); ActivationId nextActivationId = message.TargetActivation; - // check if the target activation already appears in the call chain. foreach (object invocationObj in prevChain) { var prevId = ((RequestInvocationHistory)invocationObj).ActivationId; - if (prevId.Equals(nextActivationId)) - { - return true; - } - } - - return false; - } + if (!prevId.Equals(nextActivationId) || catalog.CanInterleave(nextActivationId, message)) continue; - /// - /// Check if the current message will cause deadlock. - /// Throw DeadlockException if yes. - /// - /// - /// Message to analyze - private void CheckDeadlock(ActivationData targetActivation, Message message) - { - if (IsMessageACallChainLoop(message) && !catalog.CanInterleave(targetActivation.ActivationId, message)) - { - IEnumerable> callChain = Enumerable.Empty>(); - string exceptionMessage = string.Empty; - var requestContext = message.RequestContextData; - - if (requestContext != null - && requestContext.TryGetValue(RequestContext.CALL_CHAIN_REQUEST_CONTEXT_HEADER, out object obj) - && obj is IList prevChain) - { - var newChain = new List(); - newChain.AddRange(prevChain.Cast()); - newChain.Add(new RequestInvocationHistory(message.TargetGrain, message.TargetActivation, - message.DebugContext)); - - exceptionMessage = Utils.EnumerableToString(newChain, - elem => $"{elem.GrainId}.{elem.DebugContext}"); - callChain = newChain.Select(req => new Tuple(req.GrainId, req.DebugContext)); - } + var newChain = new List(); + newChain.AddRange(prevChain.Cast()); + newChain.Add(new RequestInvocationHistory(message.TargetGrain, message.TargetActivation, message.DebugContext)); throw new DeadlockException( - $"Deadlock Exception for grain call chain {exceptionMessage}.", callChain.ToList()); + String.Format( + "Deadlock Exception for grain call chain {0}.", + Utils.EnumerableToString( + newChain, + elem => String.Format("{0}.{1}", elem.GrainId, elem.DebugContext))), + newChain.Select(req => new Tuple(req.GrainId, req.DebugContext)).ToList()); } } @@ -487,13 +464,13 @@ private void EnqueueRequest(Message message, ActivationData targetActivation) } internal void ProcessRequestToInvalidActivation( - Message message, - ActivationAddress oldAddress, - ActivationAddress forwardingAddress, - string failedOperation, + Message message, + ActivationAddress oldAddress, + ActivationAddress forwardingAddress, + string failedOperation, Exception exc = null) { - // Just use this opportunity to invalidate local Cache Entry as well. + // Just use this opportunity to invalidate local Cache Entry as well. if (oldAddress != null) { this.localGrainDirectory.InvalidateCacheEntry(oldAddress); @@ -507,12 +484,12 @@ internal void ProcessRequestToInvalidActivation( internal void ProcessRequestsToInvalidActivation( List messages, ActivationAddress oldAddress, - ActivationAddress forwardingAddress, + ActivationAddress forwardingAddress, string failedOperation, Exception exc = null, bool rejectMessages = false) { - // Just use this opportunity to invalidate local Cache Entry as well. + // Just use this opportunity to invalidate local Cache Entry as well. if (oldAddress != null) { this.localGrainDirectory.InvalidateCacheEntry(oldAddress); @@ -538,7 +515,7 @@ internal void ProcessRequestsToInvalidActivation( { TryForwardRequest(message, oldAddress, forwardingAddress, failedOperation, exc); } - + } } ), catalog.SchedulingContext); @@ -668,7 +645,7 @@ private static bool MayForward(Message message, SiloMessagingOptions messagingOp /// - may buffer for transaction completion / commit if it ends a transaction /// - choose target placement address, maintaining send order /// - add ordering info and maintain send order - /// + /// /// /// /// @@ -814,7 +791,7 @@ internal void SendResponse(Message request, Response response) internal void SendSystemTargetMessage(Message message) { - message.Category = message.TargetGrain.Equals(Constants.MembershipOracleId) ? + message.Category = message.TargetGrain.Equals(Constants.MembershipOracleId) ? Message.Categories.Ping : Message.Categories.System; if (message.TargetSilo == null) @@ -845,7 +822,7 @@ public void TransportMessage(Message message, ActivationData sendingActivation = /// Invoked when an activation has finished a transaction and may be ready for additional transactions /// /// The activation that has just completed processing this message - /// The message that has just completed processing. + /// The message that has just completed processing. /// This will be null for the case of completion of Activate/Deactivate calls. internal void OnActivationCompletedRequest(ActivationData activation, Message message) { @@ -892,7 +869,7 @@ internal void RunMessagePump(ActivationData activation) var nextMessage = activation.PeekNextWaitingMessage(); if (nextMessage == null) continue; if (!ActivationMayAcceptRequest(activation, nextMessage)) continue; - + activation.DequeueNextWaitingMessage(); // we might be over-writing an already running read only request. HandleIncomingRequest(nextMessage, activation); diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index 1ce0aa5491..7f94c24ba8 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -99,7 +99,7 @@ public InsideRuntimeClient( private SiloAddress MySilo { get; } public GrainFactory ConcreteGrainFactory { get; } - + private Catalog Catalog => this.catalog ?? (this.catalog = this.ServiceProvider.GetRequiredService()); private ILocalGrainDirectory Directory @@ -252,12 +252,12 @@ public void SniffIncomingMessage(Message message) //// 1: //// Also record sending activation address for responses only in the cache. //// We don't record sending addresses for requests, since it is not clear that this silo ever wants to send messages to the grain sending this request. - //// However, it is sure that this silo does send messages to the sender of a reply. + //// However, it is sure that this silo does send messages to the sender of a reply. //// In most cases it will already have its address cached, unless it had a wrong outdated address cached and now this is a fresher address. //// It is anyway always safe to cache the replier address. - //// 2: + //// 2: //// after further thought decided not to do it. - //// It seems to better not bother caching the sender of a response at all, + //// It seems to better not bother caching the sender of a response at all, //// and instead to take a very occasional hit of a full remote look-up instead of this small but non-zero hit on every response. //if (message.Direction.Equals(Message.Directions.Response) && message.Result.Equals(Message.ResponseTypes.Success)) //{ @@ -289,14 +289,9 @@ public async Task Invoke(IAddressable target, IInvokable invokable, Message mess } RequestContextExtensions.Import(message.RequestContextData); - - if (!message.TargetGrain.IsSystemTarget) + if (schedulingOptions.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget) { - if (schedulingOptions.PerformDeadlockDetection || schedulingOptions.AllowCallChainReentrancy) - { - UpdateInvocationHistoryInRequestContext( - new RequestInvocationHistory(message.TargetGrain, message.TargetActivation, message.DebugContext)); - } + UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message.TargetGrain, message.TargetActivation, message.DebugContext)); // RequestContext is automatically saved in the msg upon send and propagated to the next hop // in RuntimeClient.CreateMessage -> RequestContextExtensions.ExportToMessage(message); } @@ -334,7 +329,7 @@ public async Task Invoke(IAddressable target, IInvokable invokable, Message mess !(target is IGrainExtension) && !TryInstallExtension(request.InterfaceId, invokable, message.GenericGrainType, ref invoker)) { - // We are trying the invoke a grain extension method on a grain + // We are trying the invoke a grain extension method on a grain // -- most likely reason is that the dynamic extension is not installed for this grain // So throw a specific exception here rather than a general InvalidCastException var error = String.Format( @@ -350,7 +345,7 @@ public async Task Invoke(IAddressable target, IInvokable invokable, Message mess throw exc; } - + var requestInvoker = new GrainMethodInvoker(target, request, invoker, GrainCallFilters, interfaceToImplementationMapping); await requestInvoker.Invoke(); resultObject = requestInvoker.Result; @@ -371,7 +366,7 @@ public async Task Invoke(IAddressable target, IInvokable invokable, Message mess if (transactionInfo != null) { transactionInfo.ReconcilePending(); - + // Record reason for abort, if not alread set transactionInfo.RecordException(exc1, serializationManager); @@ -513,7 +508,7 @@ private void SafeSendResponse(Message message, object resultObject) private static readonly Lazy> prepForRemotingLazy = new Lazy>(CreateExceptionPrepForRemotingMethod); - + private static Func CreateExceptionPrepForRemotingMethod() { var methodInfo = typeof(Exception).GetMethod( @@ -573,20 +568,19 @@ private void SafeSendExceptionResponse(Message message, Exception ex) } // assumes deadlock information was already loaded into RequestContext from the message - private static void UpdateInvocationHistoryInRequestContext(RequestInvocationHistory thisInvocation) + private static void UpdateDeadlockInfoInRequestContext(RequestInvocationHistory thisInvocation) { - IList prevChain; - - if (RequestContext.Get(RequestContext.CALL_CHAIN_REQUEST_CONTEXT_HEADER) is IList obj) + IList prevChain; + object obj = RequestContext.Get(RequestContext.CALL_CHAIN_REQUEST_CONTEXT_HEADER); + if (obj != null) { - prevChain = obj; + prevChain = ((IList)obj); } else { prevChain = new List(); RequestContext.Set(RequestContext.CALL_CHAIN_REQUEST_CONTEXT_HEADER, prevChain); } - // append this call to the end of the call chain. Update in place. prevChain.Add(thisInvocation); } @@ -618,7 +612,7 @@ public void ReceiveResponse(Message message) if (message.CacheInvalidationHeader == null) { // Remove from local directory cache. Note that SendingGrain is the original target, since message is the rejection response. - // If CacheMgmtHeader is present, we already did this. Otherwise, we left this code for backward compatability. + // If CacheMgmtHeader is present, we already did this. Otherwise, we left this code for backward compatability. // It should be retired as we move to use CacheMgmtHeader in all relevant places. this.Directory.InvalidateCacheEntry(message.SendingAddress); } @@ -641,7 +635,7 @@ public void ReceiveResponse(Message message) callbackData.TransactionInfo.Join(message.TransactionInfo); } // IMPORTANT: we do not schedule the response callback via the scheduler, since the only thing it does - // is to resolve/break the resolver. The continuations/waits that are based on this resolution will be scheduled as work items. + // is to resolve/break the resolver. The continuations/waits that are based on this resolution will be scheduled as work items. callbackData.DoCallback(message); } else diff --git a/test/Extensions/TesterAzureUtils/Streaming/HaloStreamSubscribeTests.cs b/test/Extensions/TesterAzureUtils/Streaming/HaloStreamSubscribeTests.cs index 1791fb4679..7d359cd5d3 100644 --- a/test/Extensions/TesterAzureUtils/Streaming/HaloStreamSubscribeTests.cs +++ b/test/Extensions/TesterAzureUtils/Streaming/HaloStreamSubscribeTests.cs @@ -75,16 +75,12 @@ public void Configure(ISiloHostBuilder hostBuilder) public override void Dispose() { base.Dispose(); - - if (HostedCluster != null) - { - AzureQueueStreamProviderUtils.DeleteAllUsedAzureQueues(NullLoggerFactory.Instance, - AzureQueueUtilities.GenerateQueueNames(this.HostedCluster.Options.ClusterId, queueCount), - TestDefaultConfiguration.DataConnectionString).Wait(); - AzureQueueStreamProviderUtils.DeleteAllUsedAzureQueues(NullLoggerFactory.Instance, - AzureQueueUtilities.GenerateQueueNames($"{this.HostedCluster.Options.ClusterId}2", queueCount), - TestDefaultConfiguration.DataConnectionString).Wait(); - } + AzureQueueStreamProviderUtils.DeleteAllUsedAzureQueues(NullLoggerFactory.Instance, + AzureQueueUtilities.GenerateQueueNames(this.HostedCluster.Options.ClusterId, queueCount), + TestDefaultConfiguration.DataConnectionString).Wait(); + AzureQueueStreamProviderUtils.DeleteAllUsedAzureQueues(NullLoggerFactory.Instance, + AzureQueueUtilities.GenerateQueueNames($"{this.HostedCluster.Options.ClusterId}2", queueCount), + TestDefaultConfiguration.DataConnectionString).Wait(); } } diff --git a/test/Grains/TestGrains/DeadlockGrain.cs b/test/Grains/TestGrains/DeadlockGrain.cs index 4b5fae3564..432b954e81 100644 --- a/test/Grains/TestGrains/DeadlockGrain.cs +++ b/test/Grains/TestGrains/DeadlockGrain.cs @@ -38,18 +38,16 @@ public class DeadlockNonReentrantGrain : Grain, IDeadlockNonReentrantGrain { private string Id { get { return String.Format("DeadlockNonReentrantGrain {0}", base.IdentityString); } } - public async Task CallNext_1(List> callChain, int currCallIndex) + public Task CallNext_1(List> callChain, int currCallIndex) { - this.GetLogger(Id).Info("Inside grain {0}.{1} CallNext_1().", Id, currCallIndex); - await DeadlockGrain.CallNext(GrainFactory, callChain, currCallIndex); - this.GetLogger(Id).Info("Inside grain {0}.{1} CallNext_1() Finished.", Id, currCallIndex); + this.GetLogger(Id).Info("Inside grain {0} CallNext_1().", Id); + return DeadlockGrain.CallNext(GrainFactory, callChain, currCallIndex); } - public async Task CallNext_2(List> callChain, int currCallIndex) + public Task CallNext_2(List> callChain, int currCallIndex) { - this.GetLogger(Id).Info("Inside grain {0}.{1} CallNext_2().", Id, currCallIndex); - await DeadlockGrain.CallNext(GrainFactory, callChain, currCallIndex); - this.GetLogger(Id).Info("Inside grain {0}.{1} CallNext_2() Finished.", Id, currCallIndex); + this.GetLogger(Id).Info("Inside grain {0} CallNext_2().", Id); + return DeadlockGrain.CallNext(GrainFactory, callChain, currCallIndex); } } diff --git a/test/TesterInternal/AllowCallChainReentrancyTests.cs b/test/TesterInternal/AllowCallChainReentrancyTests.cs deleted file mode 100644 index 63d42c4fc6..0000000000 --- a/test/TesterInternal/AllowCallChainReentrancyTests.cs +++ /dev/null @@ -1,97 +0,0 @@ -using System; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Orleans.Configuration; -using Orleans.Hosting; -using Orleans.TestingHost; -using TestExtensions; -using Xunit; -using Xunit.Abstractions; - -namespace UnitTests.General -{ - public class AllowCallChainReentrancyTests : OrleansTestingBase - { - private readonly Fixture fixture; - - public class Fixture : BaseTestClusterFixture - { - protected override void ConfigureTestCluster(TestClusterBuilder builder) - { - builder.AddSiloBuilderConfigurator(); - } - - private class SiloConfigurator : ISiloBuilderConfigurator - { - public void Configure(ISiloHostBuilder hostBuilder) - { - hostBuilder.Configure(options => - { - options.PerformDeadlockDetection = false; - options.AllowCallChainReentrancy = true; - }); - - } - } - } - - private const int numIterations = 30; - - private readonly CallChainReentrancyTestHelper testHelper; - - public AllowCallChainReentrancyTests(ITestOutputHelper output) - { - if(output == null) throw new ArgumentNullException(nameof(output)); - - this.fixture = new Fixture(); - testHelper = new CallChainReentrancyTestHelper - { - Random = random, - Fixture = fixture, - NumIterations = numIterations - }; - } - - // 1) Allowed reentrancy A, A - [Fact, TestCategory("Functional"), TestCategory("Deadlock")] - public async Task DeadlockDetection_1() - { - await testHelper.DeadlockDetection_1(); - } - - // 2) Allowed reentrancy on non-reentrant grains A, B, A - [Fact, TestCategory("Functional"), TestCategory("Deadlock")] - public async Task DeadlockDetection_2() - { - await testHelper.DeadlockDetection_2(); - } - - // 3) Allowed reentrancy X, A, X, A - [Fact, TestCategory("Functional"), TestCategory("Deadlock")] - public async Task DeadlockDetection_3() - { - await testHelper.DeadlockDetection_3(); - } - - // 4) No Deadlock X, X - [Fact, TestCategory("Functional"), TestCategory("Deadlock")] - public async Task DeadlockDetection_4() - { - await testHelper.DeadlockDetection_4(); - } - - // 5) No Deadlock X, A, X - [Fact, TestCategory("Functional"), TestCategory("Deadlock")] - public async Task DeadlockDetection_5() - { - await testHelper.DeadlockDetection_5(); - } - - // 6) Allowed reentrancy on non-reentrant grains A, B, C, A - [Fact, TestCategory("Functional"), TestCategory("Deadlock")] - public async Task DeadlockDetection_6() - { - await testHelper.DeadlockDetection_6(); - } - } -} \ No newline at end of file diff --git a/test/TesterInternal/CallChainReentrancyTestHelper.cs b/test/TesterInternal/DeadlockDetectionWithAllowCallChainReentrancyTests.cs similarity index 50% rename from test/TesterInternal/CallChainReentrancyTestHelper.cs rename to test/TesterInternal/DeadlockDetectionWithAllowCallChainReentrancyTests.cs index 46a1090533..0b348d0eb9 100644 --- a/test/TesterInternal/CallChainReentrancyTestHelper.cs +++ b/test/TesterInternal/DeadlockDetectionWithAllowCallChainReentrancyTests.cs @@ -1,35 +1,66 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; -using TestExtensions; +using Orleans; +using Orleans.TestingHost; using UnitTests.GrainInterfaces; +using Xunit; +using TestExtensions; +using Orleans.Hosting; +using Orleans.Configuration; namespace UnitTests.General { - public class CallChainReentrancyTestHelper + public class DeadlockDetectionWithAllowCallChainReentrancyTests : OrleansTestingBase, IClassFixture { - public Random Random { get; set; } - public BaseTestClusterFixture Fixture { get; set; } - public int NumIterations { get; set; } + private readonly Fixture fixture; + + public class Fixture : BaseTestClusterFixture + { + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.AddSiloBuilderConfigurator(); + } + + private class SiloConfigurator : ISiloBuilderConfigurator + { + public void Configure(ISiloHostBuilder hostBuilder) + { + hostBuilder.Configure(options => + { + options.PerformDeadlockDetection = true; + options.AllowCallChainReentrancy = true; + }); + } + } + + } + + private const int numIterations = 30; + + public DeadlockDetectionWithAllowCallChainReentrancyTests(Fixture fixture) + { + this.fixture = fixture; + } // 2 silos, loop across all cases (to force all grains to be local and remote): - // Non Reentrant A, B, C - // Reentrant X + // Non Reentrant A, B + // Reentrant C // 1) No Deadlock A, A // 2) No Deadlock A, B, A - // 3) No Deadlock X, A, X, A - // 4) No Deadlock X, X - // 5) No Deadlock X, A, X - // 6) No Deadlock A, B, C, A + // 3) No Deadlock C, A, C, A + // 4) No Deadlock C, C + // 5) No Deadlock C, A, C // 1) Allowed reentrancy A, A + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_1() { - long baseGrainId = this.Random.Next(); - for (int i = 0; i < this.NumIterations; i++) + long baseGrainId = random.Next(); + for (int i = 0; i < numIterations; i++) { long grainId = baseGrainId + i; - IDeadlockNonReentrantGrain firstGrain = this.Fixture.GrainFactory.GetGrain(grainId); + IDeadlockNonReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); List> callChain = new List>(); callChain.Add(new Tuple(grainId, true)); callChain.Add(new Tuple(grainId, true)); @@ -38,14 +69,15 @@ public async Task DeadlockDetection_1() } // 2) Allowed reentrancy on non-reentrant grains A, B, A + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_2() { - long baseGrainId = this.Random.Next(); + long baseGrainId = random.Next(); long bBase = 100; - for (int i = 0; i < this.NumIterations; i++) + for (int i = 0; i < numIterations; i++) { long grainId = baseGrainId + i; - IDeadlockNonReentrantGrain firstGrain = this.Fixture.GrainFactory.GetGrain(grainId); + IDeadlockNonReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); List> callChain = new List>(); callChain.Add(new Tuple(grainId, true)); callChain.Add(new Tuple(bBase + grainId, true)); @@ -54,76 +86,60 @@ public async Task DeadlockDetection_2() } } - // 3) Allowed reentrancy X, A, X, A + // 3) Allowed reentrancy C, A, C, A + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_3() { - long baseGrainId = this.Random.Next(); - long xBase = 1000; - for (int i = 0; i < this.NumIterations; i++) + long baseGrainId = random.Next(); + long cBase = 200; + for (int i = 0; i < numIterations; i++) { long grainId = baseGrainId + i; - IDeadlockReentrantGrain firstGrain = this.Fixture.GrainFactory.GetGrain(grainId); + IDeadlockReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); List> callChain = new List>(); - callChain.Add(new Tuple(xBase + grainId, false)); + callChain.Add(new Tuple(cBase + grainId, false)); callChain.Add(new Tuple(grainId, true)); - callChain.Add(new Tuple(xBase + grainId, false)); + callChain.Add(new Tuple(cBase + grainId, false)); callChain.Add(new Tuple(grainId, true)); await firstGrain.CallNext_1(callChain, 1); } } - // 4) No Deadlock X, X + // 4) No Deadlock C, C + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_4() { - long baseGrainId = this.Random.Next(); - long xBase = 1000; - for (int i = 0; i < this.NumIterations; i++) + long baseGrainId = random.Next(); + long cBase = 200; + for (int i = 0; i < numIterations; i++) { long grainId = baseGrainId + i; - IDeadlockReentrantGrain firstGrain = this.Fixture.GrainFactory.GetGrain(grainId); + IDeadlockReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); List> callChain = new List>(); - callChain.Add(new Tuple(xBase + grainId, false)); - callChain.Add(new Tuple(xBase + grainId, false)); + callChain.Add(new Tuple(cBase + grainId, false)); + callChain.Add(new Tuple(cBase + grainId, false)); await firstGrain.CallNext_1(callChain, 1); } } - // 5) No Deadlock X, A, X + // 5) No Deadlock C, A, C + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_5() { - long baseGrainId = this.Random.Next(); - long xBase = 1000; - for (int i = 0; i < this.NumIterations; i++) - { - long grainId = baseGrainId + i; - IDeadlockReentrantGrain firstGrain = this.Fixture.GrainFactory.GetGrain(grainId); - List> callChain = new List>(); - callChain.Add(new Tuple(xBase + grainId, false)); - callChain.Add(new Tuple(grainId, true)); - callChain.Add(new Tuple(xBase + grainId, false)); - - await firstGrain.CallNext_1(callChain, 1); - } - } - - // 6) Allowed reentrancy on non-reentrant grains only when using full chain reentrancy A, B, C, A - public async Task DeadlockDetection_6() - { - long baseGrainId = this.Random.Next(); - long bBase = 100; + long baseGrainId = random.Next(); long cBase = 200; - for (int i = 0; i < this.NumIterations; i++) + for (int i = 0; i < numIterations; i++) { long grainId = baseGrainId + i; - IDeadlockNonReentrantGrain firstGrain = this.Fixture.GrainFactory.GetGrain(grainId); + IDeadlockReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); List> callChain = new List>(); + callChain.Add(new Tuple(cBase + grainId, false)); callChain.Add(new Tuple(grainId, true)); - callChain.Add(new Tuple(bBase + grainId, true)); - callChain.Add(new Tuple(cBase + grainId, true)); - callChain.Add(new Tuple(grainId, true)); + callChain.Add(new Tuple(cBase + grainId, false)); + await firstGrain.CallNext_1(callChain, 1); } } } -} \ No newline at end of file +} diff --git a/test/TesterInternal/DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests.cs b/test/TesterInternal/DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests.cs deleted file mode 100644 index 3242967e81..0000000000 --- a/test/TesterInternal/DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests.cs +++ /dev/null @@ -1,97 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Orleans; -using Orleans.TestingHost; -using UnitTests.GrainInterfaces; -using Xunit; -using TestExtensions; -using Orleans.Hosting; -using Orleans.Configuration; - -namespace UnitTests.General -{ - public class DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests : OrleansTestingBase, IClassFixture - { - private readonly Fixture fixture; - - public class Fixture : BaseTestClusterFixture - { - protected override void ConfigureTestCluster(TestClusterBuilder builder) - { - builder.AddSiloBuilderConfigurator(); - } - - private class SiloConfigurator : ISiloBuilderConfigurator - { - public void Configure(ISiloHostBuilder hostBuilder) - { - hostBuilder.Configure(options => - { - options.PerformDeadlockDetection = true; - options.AllowCallChainReentrancy = true; - }); - } - } - - } - - private const int numIterations = 30; - - private readonly CallChainReentrancyTestHelper testHelper; - - public DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests(Fixture fixture) - { - this.fixture = fixture; - testHelper = new CallChainReentrancyTestHelper() - { - Random = random, - Fixture = fixture, - NumIterations = numIterations - }; - } - - - // 1) Allowed reentrancy A, A - [Fact, TestCategory("Functional"), TestCategory("Deadlock")] - public async Task DeadlockDetection_1() - { - await testHelper.DeadlockDetection_1(); - } - - // 2) Allowed reentrancy on non-reentrant grains A, B, A - [Fact, TestCategory("Functional"), TestCategory("Deadlock")] - public async Task DeadlockDetection_2() - { - await testHelper.DeadlockDetection_2(); - } - - // 3) Allowed reentrancy X, A, X, A - [Fact, TestCategory("Functional"), TestCategory("Deadlock")] - public async Task DeadlockDetection_3() - { - await testHelper.DeadlockDetection_3(); - } - - // 4) No Deadlock X, X - [Fact, TestCategory("Functional"), TestCategory("Deadlock")] - public async Task DeadlockDetection_4() - { - await testHelper.DeadlockDetection_4(); - } - - // 5) No Deadlock X, A, X - [Fact, TestCategory("Functional"), TestCategory("Deadlock")] - public async Task DeadlockDetection_5() - { - await testHelper.DeadlockDetection_5(); - } - - // 6) Allowed reentrancy on non-reentrant grains A, B, C, A - [Fact, TestCategory("Functional"), TestCategory("Deadlock")] - public async Task DeadlockDetection_6() - { - await testHelper.DeadlockDetection_6(); - } - } -} diff --git a/test/TesterInternal/DeadlockDetectionWithoutAllowCallChainReentrancyTests.cs b/test/TesterInternal/DeadlockDetectionWithoutAllowCallChainReentrancyTests.cs index 3a92bbd403..ca4a9744ee 100644 --- a/test/TesterInternal/DeadlockDetectionWithoutAllowCallChainReentrancyTests.cs +++ b/test/TesterInternal/DeadlockDetectionWithoutAllowCallChainReentrancyTests.cs @@ -44,8 +44,6 @@ public DeadlockDetectionWithoutAllowCallChainReentrancyTests(Fixture fixture) this.fixture = fixture; } - // TODO add test 6 - // 2 silos, loop across all cases (to force all grains to be local and remote): // Non Reentrant A, B // Reentrant C diff --git a/test/TesterInternal/TestLoggingProvider.cs b/test/TesterInternal/TestLoggingProvider.cs deleted file mode 100644 index 1410410da0..0000000000 --- a/test/TesterInternal/TestLoggingProvider.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions.Internal; - -namespace UnitTests.General -{ - public class TestLoggingProvider : ILoggerProvider - { - private readonly TestLogger logger; - - public TestLoggingProvider(Action logMethod) - { - this.logger = new TestLogger(logMethod); - } - - private class TestLogger : ILogger - { - private readonly Action logMethod; - - public TestLogger(Action logMethod) - { - this.logMethod = logMethod; - } - - void ILogger.Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, - Func formatter) - { - this.logMethod($"{logLevel}: ({eventId}-{eventId})-{formatter(state, exception)}"); - } - - bool ILogger.IsEnabled(LogLevel logLevel) => true; - - IDisposable ILogger.BeginScope(TState state) => NullScope.Instance; - - } - - public void Dispose() - { - } - - public ILogger CreateLogger(string categoryName) - { - return this.logger; - } - } -} \ No newline at end of file