Activity function triggered multiple times

[praveena-m]

KMA-Product & KMA-AdvertisingReports activity functions called multiple times. Can someone help me to troubleshoot the issue. Note that the activity functions are long running tasks and may take anywhere between 20 mins to 6 hrs.

Azure runtime version : 1.0
FUNCTIONS_EXTENSION_VERSION : ~1
Region : Central US
Hosting Plan : Standard

Original trigger time : 2018-07-06 08:00:30,989
2nd time trigger : 2018-07-06 08:15:41,814
3rd time trigger : 2018-07-06 08:43:27,074

[FunctionName("KMA-Orch-DataRefreshOrchestration")]
        public static async Task DataRefreshOrchestration([OrchestrationTrigger]DurableOrchestrationContext ctx)
        {
            try
            {
                var parallelTasks = new List<Task>();

                // Get list of active clients.
                logger.Info(ctx, "Get list of active clients.");
                var clients = await ctx.CallActivityAsync<List<int>>("KMA-GetAllClients", null);
                logger.Info(ctx, $"Number of clients found: {clients.Count()}.");

                foreach (var clientId in clients)
                {
                    var orchTask = ctx.CallSubOrchestratorAsync("KMA-Orch-DataRefreshSubOrchestration", clientId);
                    parallelTasks.Add(orchTask);
                    logger.Info(ctx, $"Started orchestration for clientId: {clientId}.");
                }

                // Wait till the data refresh completes for all clients.
                await Task.WhenAll(parallelTasks);
                logger.Info(ctx, $"Orchestration completed for all clients.");
              
            }
            catch (Exception ex)
            {
                logger.LogAllExceptions(ex);
            }
            finally
            {
                logger.Info("DataAsync OrchestrationTrigger completed.");
            }
        }

 [FunctionName("KMA-Orch-DataRefreshSubOrchestration")]
        public static async Task DataRefreshSubOrchestration([OrchestrationTrigger]DurableOrchestrationContext ctx)
        {
            var clientId = ctx.GetInput<int>();
            var today = ctx.CurrentUtcDateTime.Date;
            var productTasks = new List<Task>();
            var orderTasks = new List<Task>();
            var advertisingTasks = new List<Task>();
            var parallelTasks = new List<Task>();

            try
            {
                logger.Info(ctx, $"Started data refresh for ClientID: {clientId} with InstanceId: {ctx.InstanceId}.");

                // Get list of active clients.
                logger.Info(ctx, $"Get list Seller MarketPlaces for ClientID: {clientId}.");
                var sellers = await ctx.CallActivityAsync<List<QueueMessage>>("KMA-GetAllSellerMarketPlaces", clientId);

                foreach (var seller in sellers.GroupBy(x => x.SellerId))
                {
                    // Get seller info.
                    var sellerInfo = seller.FirstOrDefault();
                    var reportTasks = new List<Task>();

                    // Import finance data.
                    logger.Info(ctx, $"Calling KMA-Finance activity trigger for ClientID: {sellerInfo.ClientId}, SellerId: {sellerInfo.SellerId}.");
                    var financeTask = ctx.CallActivityAsync<Task>("KMA-Finance", new QueueMessage(sellerInfo.ClientId, sellerId: sellerInfo.SellerId, mwsAuthToken: sellerInfo.MWSAuthToken, postedBefore: today, postedAfter: today.AddDays(-1)));

                    foreach (var sellerMarketPlace in seller)
                    {
                        if (!string.IsNullOrEmpty(sellerMarketPlace.ProfileId) && !string.IsNullOrEmpty(sellerMarketPlace.RefreshToken))
                        {

                            logger.Info(ctx, $"Calling KMA-AdvertisingReports activity trigger for ClientID: {sellerMarketPlace.ClientId}, SellerId: {sellerMarketPlace.SellerId}, MarketPlaceId: {sellerMarketPlace.MarketplaceId}.");
                            var advertisingTask = ctx.CallActivityAsync<Task>("KMA-AdvertisingReports", new QueueMessage(sellerMarketPlace.ClientId, sellerId: sellerMarketPlace.SellerId, marketplaceId: sellerMarketPlace.MarketplaceId, postedBefore: today, postedAfter: today.AddDays(-31), profileId: sellerMarketPlace.ProfileId, refreshToken: sellerMarketPlace.RefreshToken));
                            advertisingTasks.Add(advertisingTask);
                        }

                        // Import report data.
                        logger.Info(ctx, $"Calling KMA-RequestReport activity trigger for ClientID: {sellerMarketPlace.ClientId}, SellerId: {sellerMarketPlace.SellerId}, MarketPlaceId: {sellerMarketPlace.MarketplaceId}.");
                        var reportTask = ctx.CallActivityAsync<Task>("KMA-RequestReport", new QueueMessage(sellerMarketPlace.ClientId, sellerId: sellerMarketPlace.SellerId, mwsAuthToken: sellerMarketPlace.MWSAuthToken, marketplaceId: sellerMarketPlace.MarketplaceId, reportType: ReportType._GET_MERCHANT_LISTINGS_ALL_DATA_));
                        reportTasks.Add(reportTask);

                        // Import order data.
                        logger.Info(ctx, $"Calling KMA-Order activity trigger for ClientID: {sellerMarketPlace.ClientId}, SellerId: {sellerMarketPlace.SellerId}, MarketPlaceId: {sellerMarketPlace.MarketplaceId}.");
                        var orderTask = ctx.CallActivityAsync<Task>("KMA-Order", new QueueMessage(sellerMarketPlace.ClientId, sellerId: sellerMarketPlace.SellerId, mwsAuthToken: sellerMarketPlace.MWSAuthToken, marketplaceId: sellerMarketPlace.MarketplaceId, postedBefore: today, postedAfter: today.AddDays(-1)));
                        orderTasks.Add(orderTask);
                    }

                    // Add to parallel tasks.
                    parallelTasks.AddRange(advertisingTasks);
                    parallelTasks.Add(financeTask);

                    // Wait till report tasks complete.
                    await Task.WhenAll(reportTasks);
                    logger.Info(ctx, $"Report completed for ClientID: {sellerInfo.ClientId}.");

                    // Trigger product api for once after report api completes for all marketplaces.
                    logger.Info(ctx, $"Calling KMA-Product activity trigger for ClientID: {sellerInfo.ClientId}, SellerId: {sellerInfo.SellerId}.");
                    var productTask = ctx.CallActivityAsync<Task>("KMA-Product", new QueueMessage(sellerInfo.ClientId, sellerId: sellerInfo.SellerId, mwsAuthToken: sellerInfo.MWSAuthToken));
                    productTasks.Add(productTask);
                }

                // Wait till parallel tasks complete.
                await Task.WhenAll(parallelTasks);
                logger.Info(ctx, $"Finance and Advertising completed for ClientID: {clientId}.");

                               // Wait for completion of order data import tasks.
                await Task.WhenAll(orderTasks);
                logger.Info(ctx, $"Order completed for ClientID: {clientId}.");

                await Task.WhenAll(productTasks);
                logger.Info(ctx, $"Product completed for ClientID: {clientId}.");
            }
            catch (Exception ex)
            {
                logger.LogAllExceptions(ex);
            }
            finally
            {
                logger.Info($"Completed data refresh for ClientID: {clientId} with InstanceId: {ctx.InstanceId}.");
            }
        }
  [FunctionName("KMA-Product")]
        public static async Task Run([ActivityTrigger]DurableActivityContext ctx)
        {
            try
            {
            }
            catch
            {

            }
            
         }

[SimonLuckenuik]

Please raise this issue in the Durable Functions repo: https://github.com/Azure/azure-functions-durable-extension/issues

[SimonLuckenuik]

Assuming you are using Consumption Plan, some comments:

• The 20 min to 6hours is too long for the consumption plan, it will timeout at up to 10 minutes:
  https://docs.microsoft.com/en-us/azure/azure-functions/functions-scale#consumption-plan
  • You should consider breaking that function in multiple smaller functions that you would chain, or create processing batches and execute them in a fan-out scenario.
• One of the best practice is to avoid long running functions (also applicable to Durable Functions):
  https://docs.microsoft.com/en-us/azure/azure-functions/functions-best-practices#avoid-long-running-functions
• You can also check the following issue for information about how to handle timeouts gracefully using C#:
  Allow a function to handle execution timeout gracefully and prevent process restart azure-functions-host#2153.

If you reach the timeout, the function will stop executing and it will try to execute again.

[praveena-m]

I have opened issue in durable functions repo. https://github.com/Azure/azure-functions-durable-extension/issues/386
Hence closing this issue.