Replace the impl of Syscall.rename and remove Mono.Posix.NETStandard …

…(follow-up to #1641) (#1643)

Co-authored-by: Tatsuro Shibamura <me@shibayan.jp>

src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs

@@ -59,6 +59,8 @@ public class DurableTaskExtension :
         private TaskHubWorker taskHubWorker;
         private IConnectionStringResolver connectionStringResolver;
         private bool isTaskHubWorkerStarted;
+        private HttpClient durableHttpClient;
+        private EventSourceListener eventSourceListener;
 
 #if FUNCTIONS_V1
         /// <summary>
@@ -144,6 +146,9 @@ public string HubName
         /// <param name="context">Extension context provided by WebJobs.</param>
         void IExtensionConfigProvider.Initialize(ExtensionConfigContext context)
         {
+            // We initialize linux logging early on in case any initialization steps below were to trigger a log event.
+            this.InitializeLinuxLogging();
+
             ConfigureLoaderHooks();
 
             // Functions V1 has it's configuration initialized at startup time (now).
@@ -201,10 +206,53 @@ void IExtensionConfigProvider.Initialize(ExtensionConfigContext context)
 #endif
         }
 
+        /// <summary>
+        /// Initializes the logging service for App Service if it detects that we are running in
+        /// the linux platform.
+        /// </summary>
+        private void InitializeLinuxLogging()
+        {
+            // Read enviroment variables to determine host platform
+            string containerName = this.nameResolver.Resolve("CONTAINER_NAME");
+            string azureWebsiteInstanceId = this.nameResolver.Resolve("WEBSITE_INSTANCE_ID");
+            string functionsLogsMountPath = this.nameResolver.Resolve("FUNCTIONS_LOGS_MOUNT_PATH");
+
+            // Determine host platform
+            bool inAppService = !string.IsNullOrEmpty(azureWebsiteInstanceId);
+            bool inLinuxDedicated = inAppService && !string.IsNullOrEmpty(functionsLogsMountPath);
+            bool inLinuxConsumption = !inAppService && !string.IsNullOrEmpty(containerName);
+
+            // Reading other enviroment variables for intializing the logger
+            string tenant = this.nameResolver.Resolve("WEBSITE_STAMP_DEPLOYMENT_ID");
+            string stampName = this.nameResolver.Resolve("WEBSITE_HOME_STAMPNAME");
+
+            // If running in linux, initialize the EventSource listener with the appropiate logger.
+            LinuxAppServiceLogger linuxLogger = null;
+            if (inLinuxDedicated)
+            {
+                linuxLogger = new LinuxAppServiceLogger(writeToConsole: false, containerName, tenant, stampName);
+            }
+            else if (inLinuxConsumption)
+            {
+                linuxLogger = new LinuxAppServiceLogger(writeToConsole: true, containerName, tenant, stampName);
+            }
+
+            if (linuxLogger != null)
+            {
+                // The logging service for linux works by capturing EventSource messages,
+                // which our linux platform does not recognize, and logging them via a
+                // different strategy such as writing to console or to a file.
+                this.eventSourceListener = new EventSourceListener(linuxLogger);
+            }
+        }
+
         /// <inheritdoc />
         public void Dispose()
         {
+            // Not flushing the linux logger may lead to lost logs
+            Serilog.Log.CloseAndFlush();
             this.HttpApiHandler?.Dispose();
+            this.eventSourceListener?.Dispose();
         }
 
 #if !FUNCTIONS_V1

src/WebJobs.Extensions.DurableTask/EventSourceListener.cs

@@ -0,0 +1,58 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
+using System;
+using System.Diagnostics.Tracing;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.Azure.WebJobs.Extensions.DurableTask
+{
+    /// <summary>
+    /// An EventSource Listener. It sets up a callback for internal EventSource events
+    /// that we use to capture their data and log it in Linux App Service plans.
+    /// </summary>
+    internal class EventSourceListener : EventListener
+    {
+        private readonly LinuxAppServiceLogger logger;
+
+        /// <summary>
+        /// Create an EventSourceListener to capture and log Durable EventSource
+        /// data in Linux.
+        /// </summary>
+        /// <param name="logger">A LinuxAppService logger configured for the current linux host.</param>
+        public EventSourceListener(LinuxAppServiceLogger logger)
+        {
+            this.logger = logger;
+        }
+
+        /// <summary>
+        /// Gets called for every EventSource in the process, this method allows us to determine
+        /// if the listener will subscribe to a particular EventSource provider.
+        /// We only listen to DurableTask and DurableTask-Extension EventSource providers.
+        /// </summary>
+        /// <param name="eventSource">An instance of EventSource.</param>
+        protected override void OnEventSourceCreated(EventSource eventSource)
+        {
+            // "7DA4779A-152E-44A2-A6F2-F80D991A5BEE" is the old DurableTask-Core event source,
+            // so we provide extra logic to ignore it.
+            if ((eventSource.Name == "DurableTask-Core"
+                  && eventSource.Guid != new Guid("7DA4779A-152E-44A2-A6F2-F80D991A5BEE")) ||
+                eventSource.Name == "DurableTask-AzureStorage" ||
+                eventSource.Name == "WebJobs-Extensions-DurableTask" ||
+                eventSource.Name == "DurableTask-SqlServer")
+            {
+                this.EnableEvents(eventSource, EventLevel.LogAlways, EventKeywords.All);
+            }
+        }
+
+        /// <summary>
+        /// Gets called after every EventSource event. We capture that event's data and log it
+        /// using the appropiate strategy for the current linux host.
+        /// </summary>
+        /// <param name="eventData">The EventSource event data, for logging.</param>
+        protected override void OnEventWritten(EventWrittenEventArgs eventData)
+        {
+            this.logger.Log(eventData);
+        }
+    }
+}

src/WebJobs.Extensions.DurableTask/LinuxAppServiceLogger.cs

@@ -0,0 +1,144 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
+using System;
+using System.Diagnostics;
+using System.Diagnostics.Tracing;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json.Linq;
+using Serilog;
+
+namespace Microsoft.Azure.WebJobs.Extensions.DurableTask
+{
+    /// <summary>
+    /// In charge of logging services for our linux App Service offerings: Consumption and Dedicated.
+    /// In Consumption, we log to the console and identify our log by a prefix.
+    /// In Dedicated, we log asynchronously to a pre-defined logging path using Serilog.
+    /// This class is utilized by <c>EventSourceListener</c> to write logs corresponding to
+    /// specific EventSource providers.
+    /// </summary>
+    internal class LinuxAppServiceLogger
+    {
+        private const string ConsolePrefix = "MS_DURABLE_FUNCTION_EVENTS_LOGS";
+        internal const int MaxArchives = 5;
+
+        // variable below is internal static for testing purposes
+        // we need to be able to change the logging path for a windows-based CI
+#pragma warning disable SA1401 // Fields should be private
+        internal static string LoggingPath = "/var/log/functionsLogs/durableevents.log";
+#pragma warning restore SA1401 // Fields should be private
+
+        // logging metadata
+        private readonly JToken roleInstance;
+        private readonly JToken tenant;
+        private readonly JToken sourceMoniker;
+        private readonly JToken procID;
+
+        // if true, we write to console (linux consumption), else to a file (linux dedicated).
+        private readonly bool writeToConsole;
+
+        /// <summary>
+        /// Create a LinuxAppServiceLogger instance.
+        /// </summary>
+        /// <param name="writeToConsole">If true, write to console (linux consumption) else to a file (dedicated).</param>
+        /// <param name="containerName">The app's container name.</param>
+        /// <param name="tenant">The app's tenant.</param>
+        /// <param name="stampName">The app's stamp.</param>
+        public LinuxAppServiceLogger(
+            bool writeToConsole,
+            string containerName,
+            string tenant,
+            string stampName)
+        {
+            // Initializing fixed logging metadata
+            this.writeToConsole = writeToConsole;
+            this.roleInstance = JToken.FromObject("App-" + containerName);
+            this.tenant = JToken.FromObject(tenant);
+
+            this.sourceMoniker = JToken.FromObject(
+                string.IsNullOrEmpty(stampName) ? string.Empty : "L" + stampName.Replace("-", "").ToUpperInvariant());
+            using (var process = Process.GetCurrentProcess())
+            {
+                this.procID = process.Id;
+            }
+
+            // Initialize file logger, if in Linux Dedicated
+            if (!writeToConsole)
+            {
+                var tenMbInBytes = 10000000;
+                Serilog.Log.Logger = new LoggerConfiguration()
+                    .WriteTo.Async(a =>
+                    {
+                        a.File(
+                            LinuxAppServiceLogger.LoggingPath,
+                            outputTemplate: "{Message}{NewLine}",
+                            fileSizeLimitBytes: tenMbInBytes,
+                            rollOnFileSizeLimit: true,
+                            retainedFileCountLimit: 10);
+                    })
+                    .CreateLogger();
+            }
+        }
+
+        /// <summary>
+        /// Given EventSource message data, we generate a JSON-string that we can log.
+        /// </summary>
+        /// <param name="eventData">An EventSource message, usually generated by an EventListener.</param>
+        /// <returns>A JSON-formatted string representing the input.</returns>
+        private string GenerateJsonStr(EventWrittenEventArgs eventData)
+        {
+            var values = eventData.Payload;
+            var keys = eventData.PayloadNames;
+
+            // We pack them into a JSON
+            JObject json = new JObject
+            {
+                { "EventId", eventData.EventId },
+                { "TimeStamp", DateTime.UtcNow },
+                { "RoleInstance", this.roleInstance },
+                { "Tenant", this.tenant },
+                { "Pid", this.procID },
+                { "Tid", Thread.CurrentThread.ManagedThreadId },
+            };
+
+            for (int i = 0; i < values.Count; i++)
+            {
+                json.Add(keys[i], JToken.FromObject(values[i]));
+            }
+
+            // Generate string-representation of JSON. Also remove newlines.
+            string jsonString = json.ToString(Newtonsoft.Json.Formatting.None);
+            jsonString = jsonString.Replace("\n", "\\n").Replace("\r", "\\r");
+            return jsonString;
+        }
+
+        /// <summary>
+        /// Log EventSource message data in Linux AppService.
+        /// </summary>
+        /// <param name="eventData">An EventSource message, usually generated by an EventListener.</param>
+        public void Log(EventWrittenEventArgs eventData)
+        {
+            // Generate JSON string to log based on the EventSource message
+            string jsonString = this.GenerateJsonStr(eventData);
+
+            // We write to console in Linux Consumption
+            if (this.writeToConsole)
+            {
+                // We're ignoring exceptions in the unobserved Task
+                string consoleLine = ConsolePrefix + " " + jsonString;
+                _ = Console.Out.WriteLineAsync(consoleLine);
+            }
+            else
+            {
+                // We write to a file in Linux Dedicated
+                // Serilog handles file rolling (archiving) and deletion of old logs
+                // Log-level should also be irrelevant as no minimal level has been configured
+                Serilog.Log.Information(jsonString);
+            }
+        }
+    }
+}