[functionapp] Enable server side build in OryxBuilder (#45)

* Enable scm build in oryx builder
Fix using directives
Add TODO to remove all standard workers
Address issues in comment
Rename Linux consumption functionapp related function
Internal call does not require https
Update remove all worker URL
Catch error when SCM_RUN_FROM_PACKAGE is not provided

* Add unit tests for Oryx Factory

* Make SetupLinuxConsumptionFunctionAppDeployment(context).Wait() synchronous

* Remove assignment auth

* Move ScmRunFromPackage from AppSettings to Environment Variables

* Null check for pushdeploymentcontroller.cs

* Reenable Authorize

* Address unit tests

Common/Constants.cs

@@ -118,6 +118,7 @@ public static TimeSpan MaxAllowedExecutionTime
         public const string FunctionsPortal = "FunctionsPortal";
         public const string FunctionKeyNewFormat = "~0.7";
         public const string FunctionRunTimeVersion = "FUNCTIONS_EXTENSION_VERSION";
+        public const string ScmRunFromPackage = "SCM_RUN_FROM_PACKAGE";
         public const string WebSiteSku = "WEBSITE_SKU";
         public const string WebSiteElasticScaleEnabled = "WEBSITE_ELASTIC_SCALING_ENABLED";
         public const string DynamicSku = "Dynamic";

Kudu.Contracts/Settings/SettingsKeys.cs

@@ -49,6 +49,7 @@ public static class SettingsKeys
         // Antares container specific settings
         public const string PlaceholderMode = "WEBSITE_PLACEHOLDER_MODE";
         public const string ContainerReady = "WEBSITE_CONTAINER_READY";
+        public const string WebsiteHostname = "WEBSITE_HOSTNAME";
         public const string AuthEncryptionKey = "WEBSITE_AUTH_ENCRYPTION_KEY";
         public const string ContainerEncryptionKey = "CONTAINER_ENCRYPTION_KEY";
     }

Kudu.Core/Deployment/Generator/ExternalCommandBuilder.cs

@@ -17,6 +17,7 @@ namespace Kudu.Core.Deployment.Generator
     //
     //  ExternalCommandBuilder
     //      CustomBuilder
+    //      OryxBuilder
     //      GeneratorSiteBuilder
     //          BaseBasicBuilder
     //              BasicBuilder

Kudu.Core/Deployment/Generator/OryxBuilder.cs

@@ -1,11 +1,13 @@
-using System.Threading.Tasks;
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using System.Net.Http;
+using Microsoft.WindowsAzure.Storage.Blob;
+using Microsoft.WindowsAzure.Storage;
+using Kudu.Core.Infrastructure;
 using Kudu.Core.Helpers;
 using Kudu.Contracts.Settings;
 using Kudu.Core.Deployment.Oryx;
-using Kudu.Core.Infrastructure;
-using System.IO;
-using System;
-using Kudu.Core.Commands;
 
 namespace Kudu.Core.Deployment.Generator
 {
@@ -42,7 +44,7 @@ public override Task Build(DeploymentContext context)
 
                 RunCommand(context, kuduSyncCommand, false, "Oryx-Build: Running kudu sync...");
             }
-           
+
             if (args.RunOryxBuild)
             {
                 PreOryxBuild(context);
@@ -65,6 +67,12 @@ public override Task Build(DeploymentContext context)
                 }
             }
 
+            // Detect if package upload is necessary for server side build
+            if (FunctionAppHelper.HasScmRunFromPackage() && FunctionAppHelper.LooksLikeFunctionApp())
+            {
+                SetupLinuxConsumptionFunctionAppDeployment(context).Wait();
+            }
+
             return Task.CompletedTask;
         }
 
@@ -91,10 +99,17 @@ private void SetupFunctionAppExpressArtifacts(DeploymentContext context)
             FileSystemHelpers.EnsureDirectory(sitePackages);
 
             string zipAppName = $"{DateTime.UtcNow.ToString("yyyyMMddHHmmss")}.zip";
-            string zipFile = Path.Combine(sitePackages, zipAppName);
+            PackageArtifactFromFolder(context, OryxBuildConstants.FunctionAppBuildSettings.ExpressBuildSetup, sitePackages, zipAppName, zipQuota:3);
+
+            File.WriteAllText(packageNameFile, zipAppName);
+            File.WriteAllText(packagePathFile, sitePackages);
+        }
 
+        private string PackageArtifactFromFolder(DeploymentContext context, string srcDirectory, string destDirectory, string destFilename, int zipQuota = 0)
+        {
             context.Logger.Log("Writing the artifacts to a zip file");
-            var exe = ExternalCommandFactory.BuildExternalCommandExecutable(OryxBuildConstants.FunctionAppBuildSettings.ExpressBuildSetup, sitePackages, context.Logger);
+            string zipFile = Path.Combine(destDirectory, destFilename);
+            var exe = ExternalCommandFactory.BuildExternalCommandExecutable(srcDirectory, destDirectory, context.Logger);
             try
             {
                 exe.ExecuteWithProgressWriter(context.Logger, context.Tracer, $"zip -r {zipFile} .", String.Empty);
@@ -106,10 +121,33 @@ private void SetupFunctionAppExpressArtifacts(DeploymentContext context)
             }
 
             // Just to be sure that we don't keep adding zip files here
-            DeploymentHelper.PurgeZipsIfNecessary(sitePackages, context.Tracer, totalAllowedZips: 3);
+            if (zipQuota > 0)
+            {
+                DeploymentHelper.PurgeZipsIfNecessary(destDirectory, context.Tracer, totalAllowedZips: zipQuota);
+            }
 
-            File.WriteAllText(packageNameFile, zipAppName);
-            File.WriteAllText(packagePathFile, sitePackages);
+            return zipFile;
+        }
+
+        /// <summary>
+        /// Specifically used for Linux Consumption to support Server Side build scenario
+        /// </summary>
+        /// <param name="context"></param>
+        private async Task SetupLinuxConsumptionFunctionAppDeployment(DeploymentContext context)
+        {
+            string sas = System.Environment.GetEnvironmentVariable(Constants.ScmRunFromPackage);
+            string builtFolder = context.RepositoryPath;
+            string packageFolder = Environment.DeploymentsPath;
+            string packageFileName = $"{DateTime.UtcNow.ToString("yyyyMMddHHmmss")}.zip";
+
+            // Package built content from oryx build artifact
+            string filePath = PackageArtifactFromFolder(context, builtFolder, packageFolder, packageFileName);
+
+            // Upload from DeploymentsPath
+            await UploadLinuxConsumptionFunctionAppBuiltContent(context, sas, filePath);
+
+            // Remove Linux consumption plan functionapp workers for the site
+            await RemoveLinuxConsumptionFunctionAppWorkers(context);
         }
 
         //public override void PostBuild(DeploymentContext context)
@@ -118,5 +156,63 @@ private void SetupFunctionAppExpressArtifacts(DeploymentContext context)
         //    context.Logger.Log($"Skipping post build. Project type: {ProjectType}");
         //    FileLogHelper.Log("Completed PostBuild oryx....");
         //}
+
+        private async Task UploadLinuxConsumptionFunctionAppBuiltContent(DeploymentContext context, string sas, string filePath)
+        {
+            context.Logger.Log($"Uploading built content {filePath} -> {sas}");
+
+            // Check if SCM_RUN_FROM_PACKAGE does exist
+            if (string.IsNullOrEmpty(sas))
+            {
+                context.Logger.Log($"Failed to upload because SCM_RUN_FROM_PACKAGE is not provided.");
+                throw new DeploymentFailedException(new ArgumentException("Failed to upload because SAS is empty."));
+            }
+
+            // Parse SAS
+            Uri sasUri = null;
+            if (!Uri.TryCreate(sas, UriKind.Absolute, out sasUri))
+            {
+                context.Logger.Log($"Malformed SAS when uploading built content.");
+                throw new DeploymentFailedException(new ArgumentException("Failed to upload because SAS is malformed."));
+            }
+
+            // Upload blob to Azure Storage
+            CloudBlockBlob blob = new CloudBlockBlob(sasUri);
+            try
+            {
+                await blob.UploadFromFileAsync(filePath);
+            } catch (StorageException se)
+            {
+                context.Logger.Log($"Failed to upload because Azure Storage responds {se.RequestInformation.HttpStatusCode}.");
+                context.Logger.Log(se.Message);
+                throw new DeploymentFailedException(se);
+            }
+        }
+
+        private async Task RemoveLinuxConsumptionFunctionAppWorkers(DeploymentContext context)
+        {
+            string webSiteHostName = System.Environment.GetEnvironmentVariable(SettingsKeys.WebsiteHostname);
+            string sitename = ServerConfiguration.GetApplicationName();
+
+            context.Logger.Log($"Reseting all workers for {webSiteHostName}");
+
+            try
+            {
+                await OperationManager.AttemptAsync(async () =>
+                {
+                    await PostDeploymentHelper.RemoveAllWorkersAsync(webSiteHostName, sitename);
+                }, retries: 3, delayBeforeRetry: 2000);
+            }
+            catch (ArgumentException ae)
+            {
+                context.Logger.Log($"Reset all workers has malformed webSiteHostName or sitename {ae.Message}");
+                throw new DeploymentFailedException(ae);
+            }
+            catch (HttpRequestException hre)
+            {
+                context.Logger.Log($"Reset all workers endpoint responded with {hre.Message}");
+                throw new DeploymentFailedException(hre);
+            }
+        }
     }
 }

Kudu.Core/Deployment/Oryx/FunctionAppOryxArguments.cs

@@ -4,13 +4,13 @@
 
 namespace Kudu.Core.Deployment.Oryx
 {
-    class FunctionAppOryxArguments : IOryxArguments
+    public class FunctionAppOryxArguments : IOryxArguments
     {
         public bool RunOryxBuild { get; set; }
 
         public BuildOptimizationsFlags Flags { get; set; }
 
-        private readonly WorkerRuntime FunctionsWorkerRuntime;
+        protected readonly WorkerRuntime FunctionsWorkerRuntime;
         public bool SkipKuduSync { get; set; }
 
         public FunctionAppOryxArguments()
@@ -23,7 +23,7 @@ public FunctionAppOryxArguments()
             SkipKuduSync = Flags == BuildOptimizationsFlags.UseExpressBuild;
         }
 
-        public string GenerateOryxBuildCommand(DeploymentContext context)
+        public virtual string GenerateOryxBuildCommand(DeploymentContext context)
         {
             StringBuilder args = new StringBuilder();
 
@@ -36,7 +36,7 @@ public string GenerateOryxBuildCommand(DeploymentContext context)
             return args.ToString();
         }
 
-        private void AddOryxBuildCommand(StringBuilder args, DeploymentContext context, string source, string destination)
+        protected void AddOryxBuildCommand(StringBuilder args, DeploymentContext context, string source, string destination)
         {
             // If it is express build, we don't directly need to write to /home/site/wwwroot
             // So, we build into a different directory to avoid overlap
@@ -55,7 +55,7 @@ private void AddOryxBuildCommand(StringBuilder args, DeploymentContext context,
             OryxArgumentsHelper.AddOryxBuildCommand(args, source, destination);
         }
 
-        private void AddLanguage(StringBuilder args, WorkerRuntime workerRuntime)
+        protected void AddLanguage(StringBuilder args, WorkerRuntime workerRuntime)
         {
             switch (workerRuntime)
             {
@@ -73,7 +73,7 @@ private void AddLanguage(StringBuilder args, WorkerRuntime workerRuntime)
             }
         }
 
-        private void AddLanguageVersion(StringBuilder args, WorkerRuntime workerRuntime)
+        protected void AddLanguageVersion(StringBuilder args, WorkerRuntime workerRuntime)
         {
             var workerVersion = ResolveWorkerRuntimeVersion(FunctionsWorkerRuntime);
             if (!string.IsNullOrEmpty(workerVersion))
@@ -82,7 +82,7 @@ private void AddLanguageVersion(StringBuilder args, WorkerRuntime workerRuntime)
             }
         }
 
-        private void AddBuildOptimizationFlags(StringBuilder args, DeploymentContext context, BuildOptimizationsFlags optimizationFlags)
+        protected void AddBuildOptimizationFlags(StringBuilder args, DeploymentContext context, BuildOptimizationsFlags optimizationFlags)
         {
             switch (Flags)
             {
@@ -99,7 +99,7 @@ private void AddBuildOptimizationFlags(StringBuilder args, DeploymentContext con
             }
         }
 
-        private void AddWorkerRuntimeArgs(StringBuilder args, WorkerRuntime workerRuntime)
+        protected void AddWorkerRuntimeArgs(StringBuilder args, WorkerRuntime workerRuntime)
         {
             switch (workerRuntime)
             {

Kudu.Core/Deployment/Oryx/FunctionAppSupportedWorkerRuntime.cs

@@ -14,7 +14,11 @@ public class FunctionAppSupportedWorkerRuntime
     {
         public static WorkerRuntime ParseWorkerRuntime(string value)
         {
-            if (value.StartsWith("NODE", StringComparison.OrdinalIgnoreCase))
+            if (string.IsNullOrEmpty(value))
+            {
+                return WorkerRuntime.None;
+            }
+            else if (value.StartsWith("NODE", StringComparison.OrdinalIgnoreCase))
             {
                 return WorkerRuntime.Node;
             }

Kudu.Core/Deployment/Oryx/IOryxArguments.cs

@@ -4,7 +4,7 @@
 
 namespace Kudu.Core.Deployment.Oryx
 {
-    interface IOryxArguments
+    public interface IOryxArguments
     {
         bool RunOryxBuild { get; set; }
 

Kudu.Core/Deployment/Oryx/LinuxConsumptionFunctionAppOryxArguments.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Kudu.Core.Deployment.Oryx
+{
+    public class LinuxConsumptionFunctionAppOryxArguments : FunctionAppOryxArguments
+    {
+        public LinuxConsumptionFunctionAppOryxArguments() : base()
+        {
+            SkipKuduSync = true;
+            Flags = BuildOptimizationsFlags.Off;
+        }
+
+        public override string GenerateOryxBuildCommand(DeploymentContext context)
+        {
+            StringBuilder args = new StringBuilder();
+
+            base.AddOryxBuildCommand(args, context, source: context.RepositoryPath, destination: context.RepositoryPath);
+            base.AddLanguage(args, base.FunctionsWorkerRuntime);
+            base.AddLanguageVersion(args, base.FunctionsWorkerRuntime);
+            base.AddBuildOptimizationFlags(args, context, Flags);
+            base.AddWorkerRuntimeArgs(args, base.FunctionsWorkerRuntime);
+
+            return args.ToString();
+        }
+    }
+}

Kudu.Core/Deployment/Oryx/OryxArgumentsFactory.cs

@@ -2,13 +2,18 @@
 
 namespace Kudu.Core.Deployment.Oryx
 {
-    class OryxArgumentsFactory
+    public class OryxArgumentsFactory
     {
         public static IOryxArguments CreateOryxArguments()
         {
             if (FunctionAppHelper.LooksLikeFunctionApp())
             {
-                return new FunctionAppOryxArguments();
+                if (FunctionAppHelper.HasScmRunFromPackage())
+                {
+                    return new LinuxConsumptionFunctionAppOryxArguments();
+                } else {
+                    return new FunctionAppOryxArguments();
+                }
             }
             return new AppServiceOryxArguments();
         }

Kudu.Core/Helpers/PostDeploymentHelper.cs

@@ -11,6 +11,7 @@
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using System.Web;
 using Kudu.Contracts.Settings;
 using Kudu.Core.Deployment;
 using Kudu.Core.Infrastructure;
@@ -348,6 +349,41 @@ public static async Task PerformAutoSwap(string requestId, TraceListener tracer)
             }
         }
 
+        /// <summary>
+        /// Remove all site workers after cloudbuilt content is uploaded
+        /// </summary>
+        /// <param name="websiteHostname">WEBSITE_HOSTNAME</param>
+        /// <param name="sitename">WEBSITE_SITE_NAME</param>
+        /// <exception cref="ArgumentException">Thrown when RemoveAllWorkers url is malformed.</exception>
+        /// <exception cref="HttpRequestException">Thrown when request to RemoveAllWorkers is not OK.</exception>
+        public static async Task RemoveAllWorkersAsync(string websiteHostname, string sitename)
+        {
+            // Generate URL encoded auth token
+            string websiteAuthEncryptionKey = System.Environment.GetEnvironmentVariable(SettingsKeys.AuthEncryptionKey);
+            DateTime expiry = DateTime.UtcNow.AddMinutes(5);
+            string authToken = SimpleWebTokenHelper.CreateToken(expiry, websiteAuthEncryptionKey.ToKeyBytes());
+            string authTokenEncoded = HttpUtility.UrlEncode(authToken);
+
+            // Generate RemoveAllWorker request URI
+            string baseUrl = $"http://{websiteHostname}/operations/removeworker/{sitename}/allStandard?token={authTokenEncoded}";
+            Uri baseUri = null;
+            if (!Uri.TryCreate(baseUrl, UriKind.Absolute, out baseUri))
+            {
+                throw new ArgumentException($"Malformed URI is used in RemoveAllWorkers");
+            }
+            Trace(TraceEventType.Information, "Calling RemoveAllWorkers to refresh the function app");
+
+            // Initiate GET request
+            using (var client = HttpClientFactory())
+            using (var response = await client.GetAsync(baseUri))
+            {
+                response.EnsureSuccessStatusCode();
+                Trace(TraceEventType.Information, "RemoveAllWorkers, statusCode = {0}", response.StatusCode);
+            }
+
+            return;
+        }
+
         private static void VerifyEnvironments()
         {
             if (string.IsNullOrEmpty(HttpHost))

Kudu.Core/Infrastructure/FunctionAppHelper.cs

@@ -1,4 +1,5 @@
-using System;
+using Kudu.Contracts.Settings;
+using System;
 using System.Linq;
 
 namespace Kudu.Core.Infrastructure
@@ -10,6 +11,11 @@ public static bool LooksLikeFunctionApp()
             return !string.IsNullOrEmpty(System.Environment.GetEnvironmentVariable(Constants.FunctionRunTimeVersion));
         }
 
+        public static bool HasScmRunFromPackage()
+        {
+            return !string.IsNullOrEmpty(System.Environment.GetEnvironmentVariable(Constants.ScmRunFromPackage));
+        }
+
         public static bool IsCSharpFunctionFromProjectFile(string projectPath)
         {
             return VsHelper.IncludesAnyReferencePackage(projectPath, "Microsoft.NET.Sdk.Functions");