[Linux Consumption] Support persistent storage using Azure file share (…

…Azure-App-Service#121) Adds support for creating & mounting Azure file shares for Linux consumption scm sites. Function apps can opt-in using the app setting ENABLE_KUDU_PERSISTENT_STORAGE=1. Currently the persistent storage is used for deplpyment logs only. This change also moves artifacts location from /deployments path to its own path /artifacts.
Hazhzeng · Jul 4, 2020 · 453fa52 · 453fa52
1 parent 6b5f575
commit 453fa52
Show file tree

Hide file tree

Showing 45 changed files with 1,132 additions and 230 deletions.
diff --git a/Common/Constants.cs b/Common/Constants.cs
@@ -34,6 +34,7 @@ public static class Constants
         public const string NpmDebugLogFile = "npm-debug.log";
 
         public const string DeploymentCachePath = "deployments";
+        public const string ArtifactsPath = "artifacts";
         public const string SiteExtensionsCachePath = "siteextensions";
         public const string DeploymentToolsPath = "tools";
         public const string SiteFolder = @"site";
@@ -156,5 +157,10 @@ public static TimeSpan MaxAllowedExecutionTime
         public const string LinuxLogEventStreamName = "MS_KUDU_LOGS";
         public const string WebSiteHomeStampName = "WEBSITE_HOME_STAMPNAME";
         public const string WebSiteStampDeploymentId = "WEBSITE_STAMP_DEPLOYMENT_ID";
+        public const string MeshInitURI = "MESH_INIT_URI";
+        public const string AzureWebJobsStorage = "AzureWebJobsStorage";
+        public const string KuduFileShareMountPath = "/kudu-mnt";
+        public const string KuduFileSharePrefix = "kudu-mnt";
+        public const string EnablePersistentStorage = "ENABLE_KUDU_PERSISTENT_STORAGE";
     }
 }
diff --git a/Kudu.Console/Program.cs b/Kudu.Console/Program.cs
@@ -21,6 +21,7 @@
 using Kudu.Core.SourceControl.Git;
 using Kudu.Core.Tracing;
 using System.Reflection;
+using Kudu.Core.LinuxConsumption;
 using XmlSettings;
 using log4net;
 using log4net.Config;
@@ -283,7 +284,8 @@ private static IEnvironment GetEnvironment(string siteRoot, string requestId)
                 repositoryPath,
                 requestId,
                 Path.Combine(AppContext.BaseDirectory, "KuduConsole", "kudu.dll"),
-                null);
+                null,
+                new FileSystemPathProvider(new NullMeshPersistentFileSystem()));
         }
     }
 }
diff --git a/Kudu.Contracts/IEnvironment.cs b/Kudu.Contracts/IEnvironment.cs
@@ -7,6 +7,7 @@ public interface IEnvironment
         string RepositoryPath { get; set; }     // e.g. /site/repository
         string WebRootPath { get; }             // e.g. /site/wwwroot
         string DeploymentsPath { get; }         // e.g. /site/deployments
+        string ArtifactsPath { get; }          // e.g /site/artifacts
         string DeploymentToolsPath { get; }     // e.g. /site/deployments/tools
         string SiteExtensionSettingsPath { get; }     // e.g. /site/siteextensions
         string DiagnosticsPath { get; }         // e.g. /site/diagnostics

diff --git a/Kudu.Core/Environment.cs b/Kudu.Core/Environment.cs
@@ -11,15 +11,18 @@
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Extensions;
 using Kudu.Core.Settings;
+using Kudu.Core.LinuxConsumption;
 
 namespace Kudu.Core
 {
     public class Environment : IEnvironment
     {
         private readonly IHttpContextAccessor _httpContextAccessor;
+        private readonly IFileSystemPathProvider _fileSystemPathsProvider;
 
         private readonly string _webRootPath;
         private readonly string _deploymentsPath;
+        private readonly string _artifactsPath;
         private readonly string _deploymentToolsPath;
         private readonly string _siteExtensionSettingsPath;
         private readonly string _diagnosticsPath;
@@ -106,7 +109,8 @@ public Environment(
                 string repositoryPath,
                 string requestId,
                 string kuduConsoleFullPath,
-                IHttpContextAccessor httpContextAccessor)
+                IHttpContextAccessor httpContextAccessor,
+                IFileSystemPathProvider fileSystemPathsProvider)
         {
             RootPath = rootPath;
 
@@ -117,6 +121,7 @@ public Environment(
             _zipTempPath = Path.Combine(_tempPath, Constants.ZipTempPath);
             _webRootPath = Path.Combine(SiteRootPath, Constants.WebRoot);
             _deploymentsPath = Path.Combine(SiteRootPath, Constants.DeploymentCachePath);
+            _artifactsPath = Path.Combine(SiteRootPath, Constants.ArtifactsPath);
             _deploymentToolsPath = Path.Combine(_deploymentsPath, Constants.DeploymentToolsPath);
             _siteExtensionSettingsPath = Path.Combine(SiteRootPath, Constants.SiteExtensionsCachePath);
             _diagnosticsPath = Path.Combine(SiteRootPath, Constants.DiagnosticsPath);
@@ -162,6 +167,7 @@ public Environment(
             RequestId = !string.IsNullOrEmpty(requestId) ? requestId : Guid.Empty.ToString();
 
             _httpContextAccessor = httpContextAccessor;
+            _fileSystemPathsProvider = fileSystemPathsProvider ?? throw new ArgumentNullException(nameof(fileSystemPathsProvider));
 
             KuduConsoleFullPath = kuduConsoleFullPath;
         }
@@ -191,10 +197,23 @@ public string DeploymentsPath
         {
             get
             {
+                if (_fileSystemPathsProvider.TryGetDeploymentsPath(out var path))
+                {
+                    return FileSystemHelpers.EnsureDirectory(path); ;
+                }
+
                 return FileSystemHelpers.EnsureDirectory(_deploymentsPath);
             }
         }
 
+        public string ArtifactsPath
+        {
+            get
+            {
+                return FileSystemHelpers.EnsureDirectory(_artifactsPath);
+            }
+        }
+
         public string DeploymentToolsPath
         {
             get

diff --git a/Kudu.Core/Helpers/LinuxConsumptionDeploymentHelper.cs b/Kudu.Core/Helpers/LinuxConsumptionDeploymentHelper.cs
@@ -31,7 +31,7 @@ public static async Task SetupLinuxConsumptionFunctionAppDeployment(
             string sas = settings.GetValue(Constants.ScmRunFromPackage) ?? System.Environment.GetEnvironmentVariable(Constants.ScmRunFromPackage);
 
             string builtFolder = context.OutputPath;
-            string packageFolder = env.DeploymentsPath;
+            string packageFolder = env.ArtifactsPath;
             string packageFileName = OryxBuildConstants.FunctionAppBuildSettings.LinuxConsumptionArtifactName;
 
             // Package built content from oryx build artifact

diff --git a/Kudu.Core/Infrastructure/OperationManager.cs b/Kudu.Core/Infrastructure/OperationManager.cs
@@ -104,5 +104,21 @@ public static T SafeExecute<T>(Func<T> action)
                 return default(T);
             }
         }
+
+        public static async Task<TResult> ExecuteWithTimeout<TResult>(Task<TResult> task, TimeSpan timeout)
+        {
+            var timeoutCancellationTokenSource = new CancellationTokenSource();
+
+            var completedTask = await Task.WhenAny(task, Task.Delay(timeout, timeoutCancellationTokenSource.Token));
+            if (completedTask == task)
+            {
+                timeoutCancellationTokenSource.Cancel();
+                return await task;
+            }
+            else
+            {
+                throw new TimeoutException("The operation has timed out.");
+            }
+        }
     }
 }
diff --git a/Kudu.Core/LinuxConsumption/FileSystemPathProvider.cs b/Kudu.Core/LinuxConsumption/FileSystemPathProvider.cs
@@ -0,0 +1,39 @@
+using System;
+using System.Diagnostics.Tracing;
+using Kudu.Core.Infrastructure;
+using Kudu.Core.Tracing;
+
+namespace Kudu.Core.LinuxConsumption
+{
+    public class FileSystemPathProvider : IFileSystemPathProvider
+    {
+        private readonly IMeshPersistentFileSystem _persistentFileSystem;
+
+        public FileSystemPathProvider(IMeshPersistentFileSystem persistentFileSystem)
+        {
+            _persistentFileSystem =
+                persistentFileSystem ?? throw new ArgumentNullException(nameof(persistentFileSystem));
+        }
+
+        public bool TryGetDeploymentsPath(out string path)
+        {
+            path = _persistentFileSystem.GetDeploymentsPath();
+            return !string.IsNullOrEmpty(path) && EnsureMountedDeploymentsPath(path);
+        }
+
+        private bool EnsureMountedDeploymentsPath(string path)
+        {
+            try
+            {
+                FileSystemHelpers.EnsureDirectory(path);
+                return true;
+            }
+            catch (Exception e)
+            {
+                KuduEventGenerator.Log().LogMessage(EventLevel.Informational, ServerConfiguration.GetApplicationName(),
+                    $"{nameof(EnsureMountedDeploymentsPath)} Failed. Path = {path}", e.ToString());
+                return false;
+            }
+        }
+    }
+}
diff --git a/Kudu.Core/LinuxConsumption/IFileSystemPathProvider.cs b/Kudu.Core/LinuxConsumption/IFileSystemPathProvider.cs
@@ -0,0 +1,7 @@
+namespace Kudu.Core.LinuxConsumption
+{
+    public interface IFileSystemPathProvider
+    {
+        bool TryGetDeploymentsPath(out string path);
+    }
+}
diff --git a/Kudu.Core/LinuxConsumption/IMeshPersistentFileSystem.cs b/Kudu.Core/LinuxConsumption/IMeshPersistentFileSystem.cs
@@ -0,0 +1,13 @@
+using System.Threading.Tasks;
+
+namespace Kudu.Core.LinuxConsumption
+{
+    public interface IMeshPersistentFileSystem
+    {
+        Task<bool> MountFileShare();
+
+        bool GetStatus(out string message);
+
+        string GetDeploymentsPath();
+    }
+}
diff --git a/Kudu.Core/LinuxConsumption/IMeshServiceClient.cs b/Kudu.Core/LinuxConsumption/IMeshServiceClient.cs
@@ -0,0 +1,9 @@
+using System.Threading.Tasks;
+
+namespace Kudu.Core.LinuxConsumption
+{
+    public interface IMeshServiceClient
+    {
+        Task MountCifs(string connectionString, string contentShare, string targetPath);
+    }
+}
diff --git a/Kudu.Core/LinuxConsumption/IStorageClient.cs b/Kudu.Core/LinuxConsumption/IStorageClient.cs
@@ -0,0 +1,9 @@
+using System.Threading.Tasks;
+
+namespace Kudu.Core.LinuxConsumption
+{
+    public interface IStorageClient
+    {
+        Task CreateFileShare(string siteName, string connectionString, string fileShareName);
+    }
+}
diff --git a/Kudu.Core/LinuxConsumption/ISystemEnvironment.cs b/Kudu.Core/LinuxConsumption/ISystemEnvironment.cs
@@ -0,0 +1,9 @@
+namespace Kudu.Core.LinuxConsumption
+{
+    public interface ISystemEnvironment
+    {
+        string GetEnvironmentVariable(string name);
+
+        void SetEnvironmentVariable(string name, string value);
+    }
+}
diff --git a/Kudu.Core/LinuxConsumptionEnvironment.cs → ...onsumption/LinuxConsumptionEnvironment.cs b/Kudu.Core/LinuxConsumptionEnvironment.cs → ...onsumption/LinuxConsumptionEnvironment.cs
@@ -1,12 +1,10 @@
 using System;
-using System.Collections.Generic;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Kudu.Contracts;
 using Kudu.Contracts.Settings;
 
-namespace Kudu.Core
+namespace Kudu.Core.LinuxConsumption
 {
     public class LinuxConsumptionEnvironment : ILinuxConsumptionEnvironment
     {
@@ -59,8 +57,6 @@ public bool InStandbyMode
             }
         }
 
-        Task ILinuxConsumptionEnvironment.DelayCompletionTask => throw new NotImplementedException();
-
         public void DelayRequests()
         {
             _delayLock.EnterUpgradeableReadLock();