From 63a8cf458b32c7734533ef4d6ef0f7fb8df966a2 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 26 Sep 2023 14:39:32 -0400 Subject: [PATCH 1/3] Refactor ActionManager.cs --- src/Runner.Common/Constants.cs | 1 + src/Runner.Worker/ActionManager.cs | 201 +++++++++++++++-------------- 2 files changed, 103 insertions(+), 99 deletions(-) diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 7e630bd895d..177e3c98f3f 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -261,6 +261,7 @@ public static class Agent public static readonly string ForcedInternalNodeVersion = "ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION"; public static readonly string ForcedActionsNodeVersion = "ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION"; public static readonly string PrintLogToStdout = "ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT"; + public static readonly string ActionArchiveCacheDirectory = "ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE"; } public static class System diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 56196c6e4f9..5825f290732 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -762,6 +762,8 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont ArgUtil.NotNull(downloadInfo, nameof(downloadInfo)); ArgUtil.NotNullOrEmpty(downloadInfo.NameWithOwner, nameof(downloadInfo.NameWithOwner)); ArgUtil.NotNullOrEmpty(downloadInfo.Ref, nameof(downloadInfo.Ref)); + ArgUtil.NotNullOrEmpty(downloadInfo.Ref, nameof(downloadInfo.ResolvedNameWithOwner)); + ArgUtil.NotNullOrEmpty(downloadInfo.Ref, nameof(downloadInfo.ResolvedSha)); string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), downloadInfo.NameWithOwner.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), downloadInfo.Ref); string watermarkFile = GetWatermarkFilePath(destDirectory); @@ -778,11 +780,6 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont executionContext.Output($"Download action repository '{downloadInfo.NameWithOwner}@{downloadInfo.Ref}' (SHA:{downloadInfo.ResolvedSha})"); } - await DownloadRepositoryActionAsync(executionContext, downloadInfo, destDirectory); - } - - private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, WebApi.ActionDownloadInfo downloadInfo, string destDirectory) - { //download and extract action in a temp folder and rename it on success string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), "_temp_" + Guid.NewGuid()); Directory.CreateDirectory(tempDirectory); @@ -795,102 +792,11 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont string link = downloadInfo?.TarballUrl; #endif + // i will remove the trace.info later, it here for better diff. Trace.Info($"Save archive '{link}' into {archiveFile}."); try { - int retryCount = 0; - - // Allow up to 20 * 60s for any action to be downloaded from github graph. - int timeoutSeconds = 20 * 60; - while (retryCount < 3) - { - using (var actionDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds))) - using (var actionDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(actionDownloadTimeout.Token, executionContext.CancellationToken)) - { - try - { - //open zip stream in async mode - using (FileStream fs = new(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true)) - using (var httpClientHandler = HostContext.CreateHttpClientHandler()) - using (var httpClient = new HttpClient(httpClientHandler)) - { - httpClient.DefaultRequestHeaders.Authorization = CreateAuthHeader(downloadInfo.Authentication?.Token); - - httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); - using (var response = await httpClient.GetAsync(link)) - { - var requestId = UrlUtil.GetGitHubRequestId(response.Headers); - if (!string.IsNullOrEmpty(requestId)) - { - Trace.Info($"Request URL: {link} X-GitHub-Request-Id: {requestId} Http Status: {response.StatusCode}"); - } - - if (response.IsSuccessStatusCode) - { - using (var result = await response.Content.ReadAsStreamAsync()) - { - await result.CopyToAsync(fs, _defaultCopyBufferSize, actionDownloadCancellation.Token); - await fs.FlushAsync(actionDownloadCancellation.Token); - - // download succeed, break out the retry loop. - break; - } - } - else if (response.StatusCode == HttpStatusCode.NotFound) - { - // It doesn't make sense to retry in this case, so just stop - throw new ActionNotFoundException(new Uri(link), requestId); - } - else - { - // Something else bad happened, let's go to our retry logic - response.EnsureSuccessStatusCode(); - } - } - } - } - catch (OperationCanceledException) when (executionContext.CancellationToken.IsCancellationRequested) - { - Trace.Info("Action download has been cancelled."); - throw; - } - catch (OperationCanceledException ex) when (!executionContext.CancellationToken.IsCancellationRequested && retryCount >= 2) - { - Trace.Info($"Action download final retry timeout after {timeoutSeconds} seconds."); - throw new TimeoutException($"Action '{link}' download has timed out. Error: {ex.Message}"); - } - catch (ActionNotFoundException) - { - Trace.Info($"The action at '{link}' does not exist"); - throw; - } - catch (Exception ex) when (retryCount < 2) - { - retryCount++; - Trace.Error($"Fail to download archive '{link}' -- Attempt: {retryCount}"); - Trace.Error(ex); - if (actionDownloadTimeout.Token.IsCancellationRequested) - { - // action download didn't finish within timeout - executionContext.Warning($"Action '{link}' didn't finish download within {timeoutSeconds} seconds."); - } - else - { - executionContext.Warning($"Failed to download action '{link}'. Error: {ex.Message}"); - } - } - } - - if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_DOWNLOAD_NO_BACKOFF"))) - { - var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30)); - executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry."); - await Task.Delay(backOff); - } - } - - ArgUtil.NotNullOrEmpty(archiveFile, nameof(archiveFile)); - executionContext.Debug($"Download '{link}' to '{archiveFile}'"); + await DownloadRepositoryArchive(executionContext, link, downloadInfo.Authentication?.Token, archiveFile); var stagingDirectory = Path.Combine(tempDirectory, "_staging"); Directory.CreateDirectory(stagingDirectory); @@ -947,7 +853,6 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont } Trace.Verbose("Create watermark file indicate action download succeed."); - string watermarkFile = GetWatermarkFilePath(destDirectory); File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString()); executionContext.Debug($"Archive '{archiveFile}' has been unzipped into '{destDirectory}'."); @@ -1155,6 +1060,104 @@ private AuthenticationHeaderValue CreateAuthHeader(string token) HostContext.SecretMasker.AddValue(base64EncodingToken); return new AuthenticationHeaderValue("Basic", base64EncodingToken); } + + private async Task DownloadRepositoryArchive(IExecutionContext executionContext, string downloadUrl, string downloadAuthToken, string archiveFile) + { + Trace.Info($"Save archive '{downloadUrl}' into {archiveFile}."); + int retryCount = 0; + + // Allow up to 20 * 60s for any action to be downloaded from github graph. + int timeoutSeconds = 20 * 60; + while (retryCount < 3) + { + using (var actionDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds))) + using (var actionDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(actionDownloadTimeout.Token, executionContext.CancellationToken)) + { + try + { + //open zip stream in async mode + using (FileStream fs = new(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true)) + using (var httpClientHandler = HostContext.CreateHttpClientHandler()) + using (var httpClient = new HttpClient(httpClientHandler)) + { + httpClient.DefaultRequestHeaders.Authorization = CreateAuthHeader(downloadAuthToken); + + httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); + using (var response = await httpClient.GetAsync(downloadUrl)) + { + var requestId = UrlUtil.GetGitHubRequestId(response.Headers); + if (!string.IsNullOrEmpty(requestId)) + { + Trace.Info($"Request URL: {downloadUrl} X-GitHub-Request-Id: {requestId} Http Status: {response.StatusCode}"); + } + + if (response.IsSuccessStatusCode) + { + using (var result = await response.Content.ReadAsStreamAsync()) + { + await result.CopyToAsync(fs, _defaultCopyBufferSize, actionDownloadCancellation.Token); + await fs.FlushAsync(actionDownloadCancellation.Token); + + // download succeed, break out the retry loop. + break; + } + } + else if (response.StatusCode == HttpStatusCode.NotFound) + { + // It doesn't make sense to retry in this case, so just stop + throw new ActionNotFoundException(new Uri(downloadUrl), requestId); + } + else + { + // Something else bad happened, let's go to our retry logic + response.EnsureSuccessStatusCode(); + } + } + } + } + catch (OperationCanceledException) when (executionContext.CancellationToken.IsCancellationRequested) + { + Trace.Info("Action download has been cancelled."); + throw; + } + catch (OperationCanceledException ex) when (!executionContext.CancellationToken.IsCancellationRequested && retryCount >= 2) + { + Trace.Info($"Action download final retry timeout after {timeoutSeconds} seconds."); + throw new TimeoutException($"Action '{downloadUrl}' download has timed out. Error: {ex.Message}"); + } + catch (ActionNotFoundException) + { + Trace.Info($"The action at '{downloadUrl}' does not exist"); + throw; + } + catch (Exception ex) when (retryCount < 2) + { + retryCount++; + Trace.Error($"Fail to download archive '{downloadUrl}' -- Attempt: {retryCount}"); + Trace.Error(ex); + if (actionDownloadTimeout.Token.IsCancellationRequested) + { + // action download didn't finish within timeout + executionContext.Warning($"Action '{downloadUrl}' didn't finish download within {timeoutSeconds} seconds."); + } + else + { + executionContext.Warning($"Failed to download action '{downloadUrl}'. Error: {ex.Message}"); + } + } + } + + if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_DOWNLOAD_NO_BACKOFF"))) + { + var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30)); + executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry."); + await Task.Delay(backOff); + } + } + + ArgUtil.NotNullOrEmpty(archiveFile, nameof(archiveFile)); + executionContext.Debug($"Download '{downloadUrl}' to '{archiveFile}'"); + } } public sealed class Definition From b10e7856a52815253bc957c4c52591e672fcc36f Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 26 Sep 2023 14:50:32 -0400 Subject: [PATCH 2/3] changes. --- src/Runner.Worker/ActionManager.cs | 43 +++++++++- src/Test/L0/Worker/ActionManagerL0.cs | 117 ++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 1 deletion(-) diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 5825f290732..e6f9ecd854b 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -796,7 +796,48 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont Trace.Info($"Save archive '{link}' into {archiveFile}."); try { - await DownloadRepositoryArchive(executionContext, link, downloadInfo.Authentication?.Token, archiveFile); + var useActionArchiveCache = false; + if (executionContext.Global.Variables.GetBoolean("DistributedTask.UseActionArchiveCache") == true) + { + var hasActionArchiveCache = false; + var actionArchiveCacheDir = Environment.GetEnvironmentVariable(Constants.Variables.Agent.ActionArchiveCacheDirectory); + if (!string.IsNullOrEmpty(actionArchiveCacheDir) && + Directory.Exists(actionArchiveCacheDir)) + { + hasActionArchiveCache = true; + Trace.Info($"Check if action archive '{downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha}' already exists in cache directory '{actionArchiveCacheDir}'"); +#if OS_WINDOWS + var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.zip"); +#else + var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.tar.gz"); +#endif + if (File.Exists(cacheArchiveFile)) + { + try + { + Trace.Info($"Found action archive '{cacheArchiveFile}' in cache directory '{actionArchiveCacheDir}'"); + File.Copy(cacheArchiveFile, archiveFile); + useActionArchiveCache = true; + executionContext.Debug($"Copied action archive '{cacheArchiveFile}' to '{archiveFile}'"); + } + catch (Exception ex) + { + Trace.Error($"Failed to copy action archive '{cacheArchiveFile}' to '{archiveFile}'. Error: {ex}"); + } + } + } + + executionContext.Global.JobTelemetry.Add(new JobTelemetry() + { + Type = JobTelemetryType.General, + Message = $"Action archive cache usage: {downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha} use cache {useActionArchiveCache} has cache {hasActionArchiveCache}" + }); + } + + if (!useActionArchiveCache) + { + await DownloadRepositoryArchive(executionContext, link, downloadInfo.Authentication?.Token, archiveFile); + } var stagingDirectory = Path.Combine(tempDirectory, "_staging"); Directory.CreateDirectory(stagingDirectory); diff --git a/src/Test/L0/Worker/ActionManagerL0.cs b/src/Test/L0/Worker/ActionManagerL0.cs index dda20bdd8f1..c487ea55ef1 100644 --- a/src/Test/L0/Worker/ActionManagerL0.cs +++ b/src/Test/L0/Worker/ActionManagerL0.cs @@ -293,6 +293,118 @@ public async void PrepareActions_DownloadActionFromGraph() } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_DownloadActionFromGraph_UseCache() + { + try + { + //Arrange + Setup(); + Directory.CreateDirectory(Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "action_cache")); + Directory.CreateDirectory(Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "action_cache", "actions_download-artifact")); + Directory.CreateDirectory(Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "actions-download-artifact")); + Environment.SetEnvironmentVariable(Constants.Variables.Agent.ActionArchiveCacheDirectory, Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "action_cache")); + + const string Content = @" +# Container action +name: '1ae80bcb-c1df-4362-bdaa-54f729c60281' +description: 'Greet the world and record the time' +author: 'GitHub' +inputs: + greeting: # id of input + description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' + required: true + default: 'Hello' + entryPoint: # id of input + description: 'optional docker entrypoint overwrite.' + required: false +outputs: + time: # id of output + description: 'The time we did the greeting' +icon: 'hello.svg' # vector art to display in the GitHub Marketplace +color: 'green' # optional, decorates the entry in the GitHub Marketplace +runs: + using: 'node12' + main: 'task.js' +"; + await File.WriteAllTextAsync(Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "actions-download-artifact", "action.yml"), Content); + +#if OS_WINDOWS + ZipFile.CreateFromDirectory(Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "actions-download-artifact"), Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "action_cache", "actions_download-artifact", "master-sha.zip"), CompressionLevel.Fastest, true); +#else + string tar = WhichUtil.Which("tar", require: true, trace: _hc.GetTrace()); + + // tar -xzf + using (var processInvoker = new ProcessInvokerWrapper()) + { + processInvoker.Initialize(_hc); + processInvoker.OutputDataReceived += new EventHandler((sender, args) => + { + if (!string.IsNullOrEmpty(args.Data)) + { + _hc.GetTrace().Info(args.Data); + } + }); + + processInvoker.ErrorDataReceived += new EventHandler((sender, args) => + { + if (!string.IsNullOrEmpty(args.Data)) + { + _hc.GetTrace().Error(args.Data); + } + }); + + string cwd = Path.GetDirectoryName(Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "actions-download-artifact")); + string inputDirectory = Path.GetFileName(Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "actions-download-artifact")); + string archiveFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "action_cache", "actions_download-artifact", "master-sha.tar.gz"); + int exitCode = await processInvoker.ExecuteAsync(_hc.GetDirectory(WellKnownDirectory.Bin), tar, $"-czf \"{archiveFile}\" -C \"{cwd}\" \"{inputDirectory}\"", null, CancellationToken.None); + if (exitCode != 0) + { + throw new NotSupportedException($"Can't use 'tar -czf' to create archive file: {archiveFile}. return code: {exitCode}."); + } + } +#endif + var actionId = Guid.NewGuid(); + var actions = new List + { + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "actions/download-artifact", + Ref = "master", + RepositoryType = "GitHub" + } + } + }; + + _ec.Object.Global.Variables.Set("DistributedTask.UseActionArchiveCache", bool.TrueString); + + //Act + await _actionManager.PrepareActionsAsync(_ec.Object, actions); + + //Assert + var watermarkFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "actions/download-artifact", "master.completed"); + Assert.True(File.Exists(watermarkFile)); + + var actionYamlFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "actions/download-artifact", "master", "action.yml"); + Assert.True(File.Exists(actionYamlFile)); + + _hc.GetTrace().Info(File.ReadAllText(actionYamlFile)); + + Assert.Contains("1ae80bcb-c1df-4362-bdaa-54f729c60281", File.ReadAllText(actionYamlFile)); + } + finally + { + Environment.SetEnvironmentVariable(Constants.Variables.Agent.ActionArchiveCacheDirectory, null); + Teardown(); + } + } + [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] @@ -2272,6 +2384,7 @@ private void Setup([CallerMemberName] string name = "", bool enableComposite = t _ec.Setup(x => x.ExpressionFunctions).Returns(new List()); _ec.Object.Global.FileTable = new List(); _ec.Object.Global.Plan = new TaskOrchestrationPlanReference(); + _ec.Object.Global.JobTelemetry = new List(); _ec.Setup(x => x.Write(It.IsAny(), It.IsAny())).Callback((string tag, string message) => { _hc.GetTrace().Info($"[{tag}]{message}"); }); _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())).Callback((Issue issue, ExecutionContextLogOptions logOptions) => { _hc.GetTrace().Info($"[{issue.Type}]{logOptions.LogMessageOverride ?? issue.Message}"); }); _ec.Setup(x => x.GetGitHubContext("workspace")).Returns(Path.Combine(_workFolder, "actions", "actions")); @@ -2294,6 +2407,8 @@ private void Setup([CallerMemberName] string name = "", bool enableComposite = t { NameWithOwner = action.NameWithOwner, Ref = action.Ref, + ResolvedNameWithOwner = action.NameWithOwner, + ResolvedSha = $"{action.Ref}-sha", TarballUrl = $"https://api.github.com/repos/{action.NameWithOwner}/tarball/{action.Ref}", ZipballUrl = $"https://api.github.com/repos/{action.NameWithOwner}/zipball/{action.Ref}", }; @@ -2313,6 +2428,8 @@ private void Setup([CallerMemberName] string name = "", bool enableComposite = t { NameWithOwner = action.NameWithOwner, Ref = action.Ref, + ResolvedNameWithOwner = action.NameWithOwner, + ResolvedSha = $"{action.Ref}-sha", TarballUrl = $"https://api.github.com/repos/{action.NameWithOwner}/tarball/{action.Ref}", ZipballUrl = $"https://api.github.com/repos/{action.NameWithOwner}/zipball/{action.Ref}", }; From eee25df14e1a46fe23a654972055eb2101064c70 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 26 Sep 2023 14:51:13 -0400 Subject: [PATCH 3/3] cleanup --- src/Runner.Worker/ActionManager.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index e6f9ecd854b..9e1366dbb38 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -792,8 +792,6 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont string link = downloadInfo?.TarballUrl; #endif - // i will remove the trace.info later, it here for better diff. - Trace.Info($"Save archive '{link}' into {archiveFile}."); try { var useActionArchiveCache = false;