[functionapp] Enable server side build in OryxBuilder

Common/Constants.cs

@@ -118,6 +118,7 @@ public static TimeSpan MaxAllowedExecutionTime
         public const string FunctionsPortal = "FunctionsPortal";
         public const string FunctionKeyNewFormat = "~0.7";
         public const string FunctionRunTimeVersion = "FUNCTIONS_EXTENSION_VERSION";
+        public const string ScmRunFromPackage = "SCM_RUN_FROM_PACKAGE";
         public const string WebSiteSku = "WEBSITE_SKU";
         public const string WebSiteElasticScaleEnabled = "WEBSITE_ELASTIC_SCALING_ENABLED";
         public const string DynamicSku = "Dynamic";

Kudu.Contracts/Settings/SettingsKeys.cs

@@ -49,6 +49,7 @@ public static class SettingsKeys
         // Antares container specific settings
         public const string PlaceholderMode = "WEBSITE_PLACEHOLDER_MODE";
         public const string ContainerReady = "WEBSITE_CONTAINER_READY";
+        public const string WebsiteHostname = "WEBSITE_HOSTNAME";
         public const string AuthEncryptionKey = "WEBSITE_AUTH_ENCRYPTION_KEY";
         public const string ContainerEncryptionKey = "CONTAINER_ENCRYPTION_KEY";
     }

Kudu.Core/Deployment/Generator/ExternalCommandBuilder.cs

@@ -17,6 +17,7 @@ namespace Kudu.Core.Deployment.Generator
     //
     //  ExternalCommandBuilder
     //      CustomBuilder
+    //      OryxBuilder
     //      GeneratorSiteBuilder
     //          BaseBasicBuilder
     //              BasicBuilder

Kudu.Core/Deployment/Generator/OryxBuilder.cs

@@ -1,11 +1,13 @@
-using System.Threading.Tasks;
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using System.Net.Http;
+using Microsoft.WindowsAzure.Storage.Blob;
+using Microsoft.WindowsAzure.Storage;
+using Kudu.Core.Infrastructure;
 using Kudu.Core.Helpers;
 using Kudu.Contracts.Settings;
 using Kudu.Core.Deployment.Oryx;
-using Kudu.Core.Infrastructure;
-using System.IO;
-using System;
-using Kudu.Core.Commands;
 
 namespace Kudu.Core.Deployment.Generator
 {
@@ -42,7 +44,7 @@ public override Task Build(DeploymentContext context)
 
                 RunCommand(context, kuduSyncCommand, false, "Oryx-Build: Running kudu sync...");
             }
-           
+
             if (args.RunOryxBuild)
             {
                 PreOryxBuild(context);
@@ -65,6 +67,12 @@ public override Task Build(DeploymentContext context)
                 }
             }
 
+            // Detect if package upload is necessary for server side build
+            if (FunctionAppHelper.HasScmRunFromPackage() && FunctionAppHelper.LooksLikeFunctionApp())
+            {
+                SetupLinuxConsumptionFunctionAppDeployment(context).Wait();
+            }
+
             return Task.CompletedTask;
         }
 
@@ -91,10 +99,17 @@ private void SetupFunctionAppExpressArtifacts(DeploymentContext context)
             FileSystemHelpers.EnsureDirectory(sitePackages);
 
             string zipAppName = $"{DateTime.UtcNow.ToString("yyyyMMddHHmmss")}.zip";
-            string zipFile = Path.Combine(sitePackages, zipAppName);
+            PackageArtifactFromFolder(context, OryxBuildConstants.FunctionAppBuildSettings.ExpressBuildSetup, sitePackages, zipAppName, zipQuota:3);
+
+            File.WriteAllText(packageNameFile, zipAppName);
+            File.WriteAllText(packagePathFile, sitePackages);
+        }
 
+        private string PackageArtifactFromFolder(DeploymentContext context, string srcDirectory, string destDirectory, string destFilename, int zipQuota = 0)
+        {
             context.Logger.Log("Writing the artifacts to a zip file");
-            var exe = ExternalCommandFactory.BuildExternalCommandExecutable(OryxBuildConstants.FunctionAppBuildSettings.ExpressBuildSetup, sitePackages, context.Logger);
+            string zipFile = Path.Combine(destDirectory, destFilename);
+            var exe = ExternalCommandFactory.BuildExternalCommandExecutable(srcDirectory, destDirectory, context.Logger);
             try
             {
                 exe.ExecuteWithProgressWriter(context.Logger, context.Tracer, $"zip -r {zipFile} .", String.Empty);
@@ -106,10 +121,33 @@ private void SetupFunctionAppExpressArtifacts(DeploymentContext context)
             }
 
             // Just to be sure that we don't keep adding zip files here
-            DeploymentHelper.PurgeZipsIfNecessary(sitePackages, context.Tracer, totalAllowedZips: 3);
+            if (zipQuota > 0)
+            {
+                DeploymentHelper.PurgeZipsIfNecessary(destDirectory, context.Tracer, totalAllowedZips: zipQuota);
+            }
 
-            File.WriteAllText(packageNameFile, zipAppName);
-            File.WriteAllText(packagePathFile, sitePackages);
+            return zipFile;
+        }
+
+        /// <summary>
+        /// Specifically used for Linux Consumption to support Server Side build scenario
+        /// </summary>
+        /// <param name="context"></param>
+        private async Task SetupLinuxConsumptionFunctionAppDeployment(DeploymentContext context)
+        {
+            string sas = System.Environment.GetEnvironmentVariable(Constants.ScmRunFromPackage);
+            string builtFolder = context.RepositoryPath;
+            string packageFolder = Environment.DeploymentsPath;
+            string packageFileName = $"{DateTime.UtcNow.ToString("yyyyMMddHHmmss")}.zip";
+
+            // Package built content from oryx build artifact
+            string filePath = PackageArtifactFromFolder(context, builtFolder, packageFolder, packageFileName);
+
+            // Upload from DeploymentsPath
+            await UploadLinuxConsumptionFunctionAppBuiltContent(context, sas, filePath);
+
+            // Remove Linux consumption plan functionapp workers for the site
+            await RemoveLinuxConsumptionFunctionAppWorkers(context);
         }
 
         //public override void PostBuild(DeploymentContext context)
@@ -118,5 +156,63 @@ private void SetupFunctionAppExpressArtifacts(DeploymentContext context)
         //    context.Logger.Log($"Skipping post build. Project type: {ProjectType}");
         //    FileLogHelper.Log("Completed PostBuild oryx....");
         //}
+
+        private async Task UploadLinuxConsumptionFunctionAppBuiltContent(DeploymentContext context, string sas, string filePath)
+        {
+            context.Logger.Log($"Uploading built content {filePath} -> {sas}");
+
+            // Check if SCM_RUN_FROM_PACKAGE does exist
+            if (string.IsNullOrEmpty(sas))
+            {
+                context.Logger.Log($"Failed to upload because SCM_RUN_FROM_PACKAGE is not provided.");
+                throw new DeploymentFailedException(new ArgumentException("Failed to upload because SAS is empty."));
+            }
+
+            // Parse SAS
+            Uri sasUri = null;
+            if (!Uri.TryCreate(sas, UriKind.Absolute, out sasUri))
+            {
+                context.Logger.Log($"Malformed SAS when uploading built content.");
+                throw new DeploymentFailedException(new ArgumentException("Failed to upload because SAS is malformed."));
+            }
+
+            // Upload blob to Azure Storage
+            CloudBlockBlob blob = new CloudBlockBlob(sasUri);
+            try
+            {
+                await blob.UploadFromFileAsync(filePath);
+            } catch (StorageException se)
+            {
+                context.Logger.Log($"Failed to upload because Azure Storage responds {se.RequestInformation.HttpStatusCode}.");
+                context.Logger.Log(se.Message);
+                throw new DeploymentFailedException(se);
+            }
+        }
+
+        private async Task RemoveLinuxConsumptionFunctionAppWorkers(DeploymentContext context)
+        {
+            string webSiteHostName = System.Environment.GetEnvironmentVariable(SettingsKeys.WebsiteHostname);
+            string sitename = ServerConfiguration.GetApplicationName();
+
+            context.Logger.Log($"Reseting all workers for {webSiteHostName}");
+
+            try
+            {
+                await OperationManager.AttemptAsync(async () =>
+                {
+                    await PostDeploymentHelper.RemoveAllWorkersAsync(webSiteHostName, sitename);
+                }, retries: 3, delayBeforeRetry: 2000);
+            }
+            catch (ArgumentException ae)
+            {
+                context.Logger.Log($"Reset all workers has malformed webSiteHostName or sitename {ae.Message}");
+                throw new DeploymentFailedException(ae);
+            }
+            catch (HttpRequestException hre)
+            {
+                context.Logger.Log($"Reset all workers endpoint responded with {hre.Message}");
+                throw new DeploymentFailedException(hre);
+            }
+        }
     }
 }

Kudu.Core/Deployment/Oryx/FunctionAppOryxArguments.cs

@@ -4,13 +4,13 @@
 
 namespace Kudu.Core.Deployment.Oryx
 {
-    class FunctionAppOryxArguments : IOryxArguments
+    public class FunctionAppOryxArguments : IOryxArguments
     {
         public bool RunOryxBuild { get; set; }
 
         public BuildOptimizationsFlags Flags { get; set; }
 
-        private readonly WorkerRuntime FunctionsWorkerRuntime;
+        protected readonly WorkerRuntime FunctionsWorkerRuntime;
         public bool SkipKuduSync { get; set; }
 
         public FunctionAppOryxArguments()
@@ -23,7 +23,7 @@ public FunctionAppOryxArguments()
             SkipKuduSync = Flags == BuildOptimizationsFlags.UseExpressBuild;
         }
 
-        public string GenerateOryxBuildCommand(DeploymentContext context)
+        public virtual string GenerateOryxBuildCommand(DeploymentContext context)
         {
             StringBuilder args = new StringBuilder();
 
@@ -36,7 +36,7 @@ public string GenerateOryxBuildCommand(DeploymentContext context)
             return args.ToString();
         }
 
-        private void AddOryxBuildCommand(StringBuilder args, DeploymentContext context, string source, string destination)
+        protected void AddOryxBuildCommand(StringBuilder args, DeploymentContext context, string source, string destination)
         {
             // If it is express build, we don't directly need to write to /home/site/wwwroot
             // So, we build into a different directory to avoid overlap
@@ -55,7 +55,7 @@ private void AddOryxBuildCommand(StringBuilder args, DeploymentContext context,
             OryxArgumentsHelper.AddOryxBuildCommand(args, source, destination);
         }
 
-        private void AddLanguage(StringBuilder args, WorkerRuntime workerRuntime)
+        protected void AddLanguage(StringBuilder args, WorkerRuntime workerRuntime)
         {
             switch (workerRuntime)
             {
@@ -73,7 +73,7 @@ private void AddLanguage(StringBuilder args, WorkerRuntime workerRuntime)
             }
         }
 
-        private void AddLanguageVersion(StringBuilder args, WorkerRuntime workerRuntime)
+        protected void AddLanguageVersion(StringBuilder args, WorkerRuntime workerRuntime)
         {
             var workerVersion = ResolveWorkerRuntimeVersion(FunctionsWorkerRuntime);
             if (!string.IsNullOrEmpty(workerVersion))
@@ -82,7 +82,7 @@ private void AddLanguageVersion(StringBuilder args, WorkerRuntime workerRuntime)
             }
         }
 
-        private void AddBuildOptimizationFlags(StringBuilder args, DeploymentContext context, BuildOptimizationsFlags optimizationFlags)
+        protected void AddBuildOptimizationFlags(StringBuilder args, DeploymentContext context, BuildOptimizationsFlags optimizationFlags)
         {
             switch (Flags)
             {
@@ -99,7 +99,7 @@ private void AddBuildOptimizationFlags(StringBuilder args, DeploymentContext con
             }
         }
 
-        private void AddWorkerRuntimeArgs(StringBuilder args, WorkerRuntime workerRuntime)
+        protected void AddWorkerRuntimeArgs(StringBuilder args, WorkerRuntime workerRuntime)
         {
             switch (workerRuntime)
             {

Kudu.Core/Deployment/Oryx/FunctionAppSupportedWorkerRuntime.cs

@@ -14,7 +14,11 @@ public class FunctionAppSupportedWorkerRuntime
     {
         public static WorkerRuntime ParseWorkerRuntime(string value)
         {
-            if (value.StartsWith("NODE", StringComparison.OrdinalIgnoreCase))
+            if (string.IsNullOrEmpty(value))
+            {
+                return WorkerRuntime.None;
+            }
+            else if (value.StartsWith("NODE", StringComparison.OrdinalIgnoreCase))
             {
                 return WorkerRuntime.Node;
             }

Kudu.Core/Deployment/Oryx/IOryxArguments.cs

@@ -4,7 +4,7 @@
 
 namespace Kudu.Core.Deployment.Oryx
 {
-    interface IOryxArguments
+    public interface IOryxArguments
     {
         bool RunOryxBuild { get; set; }
 

Kudu.Core/Deployment/Oryx/LinuxConsumptionFunctionAppOryxArguments.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Kudu.Core.Deployment.Oryx
+{
+    public class LinuxConsumptionFunctionAppOryxArguments : FunctionAppOryxArguments
+    {
+        public LinuxConsumptionFunctionAppOryxArguments() : base()
+        {
+            SkipKuduSync = true;
+            Flags = BuildOptimizationsFlags.Off;
+        }
+
+        public override string GenerateOryxBuildCommand(DeploymentContext context)
+        {
+            StringBuilder args = new StringBuilder();
+
+            base.AddOryxBuildCommand(args, context, source: context.RepositoryPath, destination: context.RepositoryPath);
+            base.AddLanguage(args, base.FunctionsWorkerRuntime);
+            base.AddLanguageVersion(args, base.FunctionsWorkerRuntime);
+            base.AddBuildOptimizationFlags(args, context, Flags);
+            base.AddWorkerRuntimeArgs(args, base.FunctionsWorkerRuntime);
+
+            return args.ToString();
+        }
+    }
+}

Kudu.Core/Deployment/Oryx/OryxArgumentsFactory.cs

@@ -2,13 +2,18 @@
 
 namespace Kudu.Core.Deployment.Oryx
 {
-    class OryxArgumentsFactory
+    public class OryxArgumentsFactory
     {
         public static IOryxArguments CreateOryxArguments()
         {
             if (FunctionAppHelper.LooksLikeFunctionApp())
             {
-                return new FunctionAppOryxArguments();
+                if (FunctionAppHelper.HasScmRunFromPackage())
+                {
+                    return new LinuxConsumptionFunctionAppOryxArguments();
+                } else {
+                    return new FunctionAppOryxArguments();
+                }
             }
             return new AppServiceOryxArguments();
         }

Kudu.Core/Helpers/PostDeploymentHelper.cs

@@ -11,6 +11,7 @@
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using System.Web;
 using Kudu.Contracts.Settings;
 using Kudu.Core.Deployment;
 using Kudu.Core.Infrastructure;
@@ -348,6 +349,41 @@ public static async Task PerformAutoSwap(string requestId, TraceListener tracer)
             }
         }
 
+        /// <summary>
+        /// Remove all site workers after cloudbuilt content is uploaded
+        /// </summary>
+        /// <param name="websiteHostname">WEBSITE_HOSTNAME</param>
+        /// <param name="sitename">WEBSITE_SITE_NAME</param>
+        /// <exception cref="ArgumentException">Thrown when RemoveAllWorkers url is malformed.</exception>
+        /// <exception cref="HttpRequestException">Thrown when request to RemoveAllWorkers is not OK.</exception>
+        public static async Task RemoveAllWorkersAsync(string websiteHostname, string sitename)
+        {
+            // Generate URL encoded auth token
+            string websiteAuthEncryptionKey = System.Environment.GetEnvironmentVariable(SettingsKeys.AuthEncryptionKey);
+            DateTime expiry = DateTime.UtcNow.AddMinutes(5);
+            string authToken = SimpleWebTokenHelper.CreateToken(expiry, websiteAuthEncryptionKey.ToKeyBytes());
+            string authTokenEncoded = HttpUtility.UrlEncode(authToken);
+
+            // Generate RemoveAllWorker request URI
+            string baseUrl = $"http://{websiteHostname}/operations/removeworker/{sitename}/allStandard?token={authTokenEncoded}";
+            Uri baseUri = null;
+            if (!Uri.TryCreate(baseUrl, UriKind.Absolute, out baseUri))
+            {
+                throw new ArgumentException($"Malformed URI is used in RemoveAllWorkers");
+            }
+            Trace(TraceEventType.Information, "Calling RemoveAllWorkers to refresh the function app");
+
+            // Initiate GET request
+            using (var client = HttpClientFactory())
+            using (var response = await client.GetAsync(baseUri))
+            {
+                response.EnsureSuccessStatusCode();
+                Trace(TraceEventType.Information, "RemoveAllWorkers, statusCode = {0}", response.StatusCode);
+            }
+
+            return;
+        }
+
         private static void VerifyEnvironments()
         {
             if (string.IsNullOrEmpty(HttpHost))

Kudu.Core/Infrastructure/FunctionAppHelper.cs

@@ -1,4 +1,5 @@
-using System;
+using Kudu.Contracts.Settings;
+using System;
 using System.Linq;
 
 namespace Kudu.Core.Infrastructure
@@ -10,6 +11,11 @@ public static bool LooksLikeFunctionApp()
             return !string.IsNullOrEmpty(System.Environment.GetEnvironmentVariable(Constants.FunctionRunTimeVersion));
         }
 
+        public static bool HasScmRunFromPackage()
+        {
+            return !string.IsNullOrEmpty(System.Environment.GetEnvironmentVariable(Constants.ScmRunFromPackage));
+        }
+
         public static bool IsCSharpFunctionFromProjectFile(string projectPath)
         {
             return VsHelper.IncludesAnyReferencePackage(projectPath, "Microsoft.NET.Sdk.Functions");