From 31d403405097b26915f885edad37252e37a18797 Mon Sep 17 00:00:00 2001 From: Denis Rumyantsev Date: Mon, 23 Sep 2024 16:24:18 +0200 Subject: [PATCH] Fallback from Node 20 to Node 16 (#4987) Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> --- src/Agent.Sdk/ContainerInfo.cs | 1 + src/Agent.Sdk/ExecutionTargetInfo.cs | 2 + .../Container/DockerCommandManager.cs | 9 ++ .../ContainerOperationProvider.cs | 107 +++++++++++++++--- src/Agent.Worker/Handlers/StepHost.cs | 12 +- .../containerHandlerInvoker.js.template | 79 ++++++------- 6 files changed, 137 insertions(+), 73 deletions(-) diff --git a/src/Agent.Sdk/ContainerInfo.cs b/src/Agent.Sdk/ContainerInfo.cs index 44d33c96a8..8e7433211d 100644 --- a/src/Agent.Sdk/ContainerInfo.cs +++ b/src/Agent.Sdk/ContainerInfo.cs @@ -83,6 +83,7 @@ public ContainerInfo(Pipelines.ContainerResource container, Boolean isJobContain public string ContainerName { get; set; } public string ContainerCommand { get; set; } public string CustomNodePath { get; set; } + public string ResultNodePath { get; set; } public Guid ContainerRegistryEndpoint { get; private set; } public string ContainerCreateOptions { get; set; } public bool SkipContainerImagePull { get; private set; } diff --git a/src/Agent.Sdk/ExecutionTargetInfo.cs b/src/Agent.Sdk/ExecutionTargetInfo.cs index 9b5cab221a..d7e22ca7ab 100644 --- a/src/Agent.Sdk/ExecutionTargetInfo.cs +++ b/src/Agent.Sdk/ExecutionTargetInfo.cs @@ -8,6 +8,7 @@ public interface ExecutionTargetInfo { PlatformUtil.OS ExecutionOS { get; } string CustomNodePath { get; set; } + string ResultNodePath { get; set; } string TranslateContainerPathForImageOS(PlatformUtil.OS runningOs, string path); string TranslateToContainerPath(string path); @@ -18,6 +19,7 @@ public class HostInfo : ExecutionTargetInfo { public PlatformUtil.OS ExecutionOS => PlatformUtil.HostOS; public string CustomNodePath { get; set; } + public string ResultNodePath { get; set; } public string TranslateToContainerPath(string path) { diff --git a/src/Agent.Worker/Container/DockerCommandManager.cs b/src/Agent.Worker/Container/DockerCommandManager.cs index c93435d20a..875ece569a 100644 --- a/src/Agent.Worker/Container/DockerCommandManager.cs +++ b/src/Agent.Worker/Container/DockerCommandManager.cs @@ -26,6 +26,7 @@ public interface IDockerCommandManager : IAgentService Task DockerCreate(IExecutionContext context, ContainerInfo container); Task DockerStart(IExecutionContext context, string containerId); Task DockerLogs(IExecutionContext context, string containerId); + Task> GetDockerLogs(IExecutionContext context, string containerId); Task> DockerPS(IExecutionContext context, string options); Task DockerRemove(IExecutionContext context, string containerId); Task DockerNetworkCreate(IExecutionContext context, string network); @@ -229,6 +230,14 @@ public async Task DockerLogs(IExecutionContext context, string containerId) return await ExecuteDockerCommandAsync(context, "logs", $"--details {containerId}", context.CancellationToken); } + public async Task> GetDockerLogs(IExecutionContext context, string containerId) + { + ArgUtil.NotNull(context, nameof(context)); + ArgUtil.NotNull(containerId, nameof(containerId)); + + return await ExecuteDockerCommandAsync(context, "logs", $"--details {containerId}"); + } + public async Task> DockerPS(IExecutionContext context, string options) { ArgUtil.NotNull(context, nameof(context)); diff --git a/src/Agent.Worker/ContainerOperationProvider.cs b/src/Agent.Worker/ContainerOperationProvider.cs index 68e9cecdbe..734cbf7659 100644 --- a/src/Agent.Worker/ContainerOperationProvider.cs +++ b/src/Agent.Worker/ContainerOperationProvider.cs @@ -525,6 +525,22 @@ private async Task StartContainerAsync(IExecutionContext executionContext, Conta container.MountVolumes.Add(new MountVolume(taskKeyFile, container.TranslateToContainerPath(taskKeyFile))); } + bool useNode20ToStartContainer = AgentKnobs.UseNode20ToStartContainer.GetValue(executionContext).AsBoolean(); + bool useAgentNode = false; + + string labelContainerStartupUsingNode20 = "container-startup-using-node-20"; + string labelContainerStartupUsingNode16 = "container-startup-using-node-16"; + string labelContainerStartupFailed = "container-startup-failed"; + + string containerNodePath(string nodeFolder) + { + return container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), nodeFolder, "bin", $"node{IOUtil.ExeExtension}")); + } + + string nodeContainerPath = containerNodePath(NodeHandler.NodeFolder); + string node16ContainerPath = containerNodePath(NodeHandler.Node16Folder); + string node20ContainerPath = containerNodePath(NodeHandler.Node20_1Folder); + if (container.IsJobContainer) { // See if this container brings its own Node.js @@ -532,30 +548,35 @@ private async Task StartContainerAsync(IExecutionContext executionContext, Conta dockerObject: container.ContainerImage, options: $"--format=\"{{{{index .Config.Labels \\\"{_nodeJsPathLabel}\\\"}}}}\""); - string node; + string nodeSetInterval(string node) + { + return $"'{node}' -e 'setInterval(function(){{}}, 24 * 60 * 60 * 1000);'"; + } + + string useDoubleQuotes(string value) + { + return value.Replace('\'', '"'); + } + if (!string.IsNullOrEmpty(container.CustomNodePath)) { - node = container.CustomNodePath; + container.ContainerCommand = useDoubleQuotes(nodeSetInterval(container.CustomNodePath)); + container.ResultNodePath = container.CustomNodePath; + } + else if (PlatformUtil.RunningOnMacOS || (PlatformUtil.RunningOnWindows && container.ImageOS == PlatformUtil.OS.Linux)) + { + // require container to have node if running on macOS, or if running on Windows and attempting to run Linux container + container.CustomNodePath = "node"; + container.ContainerCommand = useDoubleQuotes(nodeSetInterval(container.CustomNodePath)); + container.ResultNodePath = container.CustomNodePath; } else { - node = container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), AgentKnobs.UseNode20ToStartContainer.GetValue(executionContext).AsBoolean() ? NodeHandler.Node20_1Folder : NodeHandler.NodeFolder, "bin", $"node{IOUtil.ExeExtension}")); - - // if on Mac OS X, require container to have node - if (PlatformUtil.RunningOnMacOS) - { - container.CustomNodePath = "node"; - node = container.CustomNodePath; - } - // if running on Windows, and attempting to run linux container, require container to have node - else if (PlatformUtil.RunningOnWindows && container.ImageOS == PlatformUtil.OS.Linux) - { - container.CustomNodePath = "node"; - node = container.CustomNodePath; - } + useAgentNode = true; + string sleepCommand = useNode20ToStartContainer ? $"'{node20ContainerPath}' --version && echo '{labelContainerStartupUsingNode20}' && {nodeSetInterval(node20ContainerPath)} || '{node16ContainerPath}' --version && echo '{labelContainerStartupUsingNode16}' && {nodeSetInterval(node16ContainerPath)} || echo '{labelContainerStartupFailed}'" : nodeSetInterval(nodeContainerPath); + container.ContainerCommand = PlatformUtil.RunningOnWindows ? $"cmd.exe /c call {useDoubleQuotes(sleepCommand)}" : $"bash -c \"{sleepCommand}\""; + container.ResultNodePath = nodeContainerPath; } - string sleepCommand = $"\"{node}\" -e \"setInterval(function(){{}}, 24 * 60 * 60 * 1000);\""; - container.ContainerCommand = sleepCommand; } container.ContainerId = await _dockerManger.DockerCreate(executionContext, container); @@ -588,6 +609,56 @@ private async Task StartContainerAsync(IExecutionContext executionContext, Conta executionContext.Warning($"Docker container {container.ContainerId} is not in running state."); } + else if (useAgentNode && useNode20ToStartContainer) + { + bool containerStartupCompleted = false; + int containerStartupTimeoutInMilliseconds = 10000; + int delayInMilliseconds = 100; + int checksCount = 0; + + while (true) + { + List containerLogs = await _dockerManger.GetDockerLogs(executionContext, container.ContainerId); + + foreach (string logLine in containerLogs) + { + if (logLine.Contains(labelContainerStartupUsingNode20)) + { + executionContext.Debug("Using Node 20 for container startup."); + containerStartupCompleted = true; + container.ResultNodePath = node20ContainerPath; + break; + } + else if (logLine.Contains(labelContainerStartupUsingNode16)) + { + executionContext.Warning("Can not run Node 20 in container. Falling back to Node 16 for container startup."); + containerStartupCompleted = true; + container.ResultNodePath = node16ContainerPath; + break; + } + else if (logLine.Contains(labelContainerStartupFailed)) + { + executionContext.Error("Can not run both Node 20 and Node 16 in container. Container startup failed."); + containerStartupCompleted = true; + break; + } + } + + if (containerStartupCompleted) + { + break; + } + + checksCount++; + if (checksCount * delayInMilliseconds > containerStartupTimeoutInMilliseconds) + { + executionContext.Warning("Can not get startup status from container."); + break; + } + + await Task.Delay(delayInMilliseconds); + } + } } catch (Exception ex) { diff --git a/src/Agent.Worker/Handlers/StepHost.cs b/src/Agent.Worker/Handlers/StepHost.cs index 5ea48b0b32..9f7b0e86da 100644 --- a/src/Agent.Worker/Handlers/StepHost.cs +++ b/src/Agent.Worker/Handlers/StepHost.cs @@ -182,16 +182,6 @@ public async Task ExecuteAsync(string workingDirectory, HostContext.GetTrace(nameof(ContainerStepHost)).Info($"Copying containerHandlerInvoker.js to {tempDir}"); File.Copy(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "containerHandlerInvoker.js.template"), targetEntryScript, true); - string node; - if (!string.IsNullOrEmpty(Container.CustomNodePath)) - { - node = Container.CustomNodePath; - } - else - { - node = Container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node", "bin", $"node{IOUtil.ExeExtension}")); - } - string entryScript = Container.TranslateContainerPathForImageOS(PlatformUtil.HostOS, Container.TranslateToContainerPath(targetEntryScript)); string userArgs = ""; @@ -209,7 +199,7 @@ public async Task ExecuteAsync(string workingDirectory, } } - string containerExecutionArgs = $"exec -i {userArgs} {workingDirectoryParam} {Container.ContainerId} {node} {entryScript}"; + string containerExecutionArgs = $"exec -i {userArgs} {workingDirectoryParam} {Container.ContainerId} {Container.ResultNodePath} {entryScript}"; using (var processInvoker = HostContext.CreateService()) { diff --git a/src/Misc/layoutbin/containerHandlerInvoker.js.template b/src/Misc/layoutbin/containerHandlerInvoker.js.template index 62cb6f5cd9..a9eab71af9 100644 --- a/src/Misc/layoutbin/containerHandlerInvoker.js.template +++ b/src/Misc/layoutbin/containerHandlerInvoker.js.template @@ -1,62 +1,53 @@ const { spawn } = require('child_process'); -var stdinString = ""; -process.stdin.on('data', function (chunk) { - stdinString += chunk; -}); -process.stdin.on('end', function () { - var stdinData = JSON.parse(stdinString); - var handler = stdinData.handler; - var handlerArg = stdinData.args; - var handlerWorkDir = stdinData.workDir; - var prependPath = stdinData.prependPath; +const debug = log => console.log(`##vso[task.debug]${log}`); - console.log("##vso[task.debug]Handler: " + handler); - console.log("##vso[task.debug]HandlerArg: " + handlerArg); - console.log("##vso[task.debug]HandlerWorkDir: " + handlerWorkDir); - Object.keys(stdinData.environment).forEach(function (key) { - console.log("##vso[task.debug]Set env: " + key + "=" + stdinData.environment[key].toString().replace(/\r/g, '%0D').replace(/\n/g, '%0A')); - process.env[key] = stdinData.environment[key]; - }); +let stdinString = ''; +process.stdin.on('data', chunk => stdinString += chunk); + +process.stdin.on('end', () => { + const { handler, args: handlerArg, workDir: handlerWorkDir, prependPath, environment } = JSON.parse(stdinString); - var currentPath = process.env['PATH']; - var options = { + debug(`Handler: ${handler}`); + debug(`HandlerArg: ${handlerArg}`); + debug(`HandlerWorkDir: ${handlerWorkDir}`); + + for (const key in environment) { + const value = environment[key].toString().replace(/\r/g, '%0D').replace(/\n/g, '%0A'); + debug(`Set env: ${key}=${value}`); + process.env[key] = environment[key]; + } + + const options = { stdio: 'inherit', cwd: handlerWorkDir }; - if (process.platform == 'win32') { + + const isWindows = process.platform == 'win32'; + + if (isWindows) { options.argv0 = `"${handler}"`; options.windowsVerbatimArguments = true; - - if (prependPath && prependPath.length > 0) { - if (currentPath && currentPath.length > 0) { - process.env['PATH'] = prependPath + ';' + currentPath; - } - else { - process.env['PATH'] = prependPath; - } - } - } - else { - if (prependPath && prependPath.length > 0) { - if (currentPath && currentPath.length > 0) { - process.env['PATH'] = prependPath + ':' + currentPath; - } - else { - process.env['PATH'] = prependPath; - } - } } if (prependPath && prependPath.length > 0) { - console.log("##vso[task.debug]Prepend Path: " + process.env['PATH']); + const currentPath = process.env['PATH']; + process.env['PATH'] = prependPath; + + if (currentPath && currentPath.length > 0) { + process.env['PATH'] += `${isWindows ? ';' : ':'}${currentPath}`; + } + + debug(`Prepend Path: ${process.env['PATH']}`); } process.env['TF_BUILD'] = 'True'; - console.log("##vso[task.debug]Handler Setup Complete"); - var launch = spawn(handler, [handlerArg], options); - launch.on('exit', function (code) { - console.log("##vso[task.debug]Handler exit code: " + code); + debug(`Handler Setup Complete`); + const launch = spawn(handler, [handlerArg], options); + + launch.on('exit', code => { + debug(`Handler exit code: ${code}`); + if (code != 0) { process.exit(code); }