From 2a6260ff08feb4bdab89c7b442ce86d26954f842 Mon Sep 17 00:00:00 2001 From: Mattias Karlsson Date: Mon, 30 Sep 2024 15:03:20 +0200 Subject: [PATCH 1/2] (GH-4331/GH-4349) Migrate GHA Artifact to V4 API * fixes #4331 * fixes #4349 --- .../Build/GitHubActionsCommandsFixture.cs | 247 +++++--------- .../Build/GitHubActionsInfoFixture.cs | 10 +- .../Data/GitHubActionsRuntimeInfoTests.cs | 4 +- .../Artifact/CreateArtifactRequest.cs | 20 ++ .../Artifact/CreateArtifactResponse.cs | 16 + .../Artifact/FinalizeArtifactRequest.cs | 22 ++ .../Artifact/FinalizeArtifactResponse.cs | 16 + .../Artifact/GetSignedArtifactURLRequest.cs | 18 + .../Artifact/GetSignedArtifactURLResponse.cs | 15 + .../Artifact/GitHubActionsArtifactService.cs | 311 ++++++++++++++++++ .../Commands/ArtifactResponse.cs | 32 -- .../GitHubActions/Commands/ContainerItem.cs | 33 -- .../Commands/ContainerItemResource.cs | 60 ---- .../Commands/CreateArtifactParameters.cs | 19 -- .../Commands/GitHubActionsCommands.cs | 250 +------------- .../Commands/PatchArtifactSize.cs | 16 - .../Build/GitHubActions/Commands/Values.cs | 17 - .../Data/GitHubActionsRuntimeInfo.cs | 7 +- src/Cake.Common/Cake.Common.csproj | 1 + src/Directory.Packages.props | 1 + 20 files changed, 532 insertions(+), 583 deletions(-) create mode 100644 src/Cake.Common/Build/GitHubActions/Commands/Artifact/CreateArtifactRequest.cs create mode 100644 src/Cake.Common/Build/GitHubActions/Commands/Artifact/CreateArtifactResponse.cs create mode 100644 src/Cake.Common/Build/GitHubActions/Commands/Artifact/FinalizeArtifactRequest.cs create mode 100644 src/Cake.Common/Build/GitHubActions/Commands/Artifact/FinalizeArtifactResponse.cs create mode 100644 src/Cake.Common/Build/GitHubActions/Commands/Artifact/GetSignedArtifactURLRequest.cs create mode 100644 src/Cake.Common/Build/GitHubActions/Commands/Artifact/GetSignedArtifactURLResponse.cs create mode 100644 src/Cake.Common/Build/GitHubActions/Commands/Artifact/GitHubActionsArtifactService.cs delete mode 100644 src/Cake.Common/Build/GitHubActions/Commands/ArtifactResponse.cs delete mode 100644 src/Cake.Common/Build/GitHubActions/Commands/ContainerItem.cs delete mode 100644 src/Cake.Common/Build/GitHubActions/Commands/ContainerItemResource.cs delete mode 100644 src/Cake.Common/Build/GitHubActions/Commands/CreateArtifactParameters.cs delete mode 100644 src/Cake.Common/Build/GitHubActions/Commands/PatchArtifactSize.cs delete mode 100644 src/Cake.Common/Build/GitHubActions/Commands/Values.cs diff --git a/src/Cake.Common.Tests/Fixtures/Build/GitHubActionsCommandsFixture.cs b/src/Cake.Common.Tests/Fixtures/Build/GitHubActionsCommandsFixture.cs index e8fe917e83..58d1d92da4 100644 --- a/src/Cake.Common.Tests/Fixtures/Build/GitHubActionsCommandsFixture.cs +++ b/src/Cake.Common.Tests/Fixtures/Build/GitHubActionsCommandsFixture.cs @@ -1,10 +1,12 @@ using System; +using System.IO.Compression; using System.Linq; using System.Net; using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Cake.Common.Build.GitHubActions.Commands; +using Cake.Common.Build.GitHubActions.Commands.Artifact; using Cake.Common.Build.GitHubActions.Data; using Cake.Common.Tests.Fakes; using Cake.Core; @@ -16,96 +18,26 @@ namespace Cake.Common.Tests.Fixtures.Build { internal sealed class GitHubActionsCommandsFixture : HttpMessageHandler { - private const string ApiVersion = "6.0-preview"; - private const string AcceptHeader = "application/json; api-version=" + ApiVersion; - private const string AcceptGzip = "application/octet-stream; api-version=" + ApiVersion; - private const string AcceptEncodingGzip = "gzip"; - private const string CreateArtifactUrl = GitHubActionsInfoFixture.ActionRuntimeUrl + - "_apis/pipelines/workflows/34058136/artifacts?api-version=" + ApiVersion + "&artifactName=artifact"; - private const string CreateArtifactsUrl = GitHubActionsInfoFixture.ActionRuntimeUrl + - "_apis/pipelines/workflows/34058136/artifacts?api-version=" + ApiVersion + "&artifactName=artifacts"; - private const string PutFileUrl = GitHubActionsInfoFixture.ActionRuntimeUrl + - "_apis/resources/Containers/942031?itemPath=artifact%2Fartifact.txt"; + private const string AcceptHeaderResults = "application/json"; + private const string ArtifactUrl = GitHubActionsInfoFixture.ActionResultsUrl + "twirp/github.actions.results.api.v1.ArtifactService/"; + private const string CreateArtifactUrl = ArtifactUrl + "CreateArtifact"; + private const string FinalizeArtifactUrl = ArtifactUrl + "FinalizeArtifact"; + private const string GetSignedArtifactURLurl = ArtifactUrl + "GetSignedArtifactURL"; + private const string UploadFileUrl = "https://cake.build.net/actions-results/a9d82106-d5d5-4310-8f60-0bfac035cf02/workflow-job-run-1d849a45-2f30-5fbb-3226-b730a17a93af/artifacts/91e64594182918fa8012cdbf7d1a4f801fa0c35f485c3277268aad8e3f45377c.zip?sig=upload"; + private const string DownloadFileUrl = "https://cake.build.net/actions-results/a9d82106-d5d5-4310-8f60-0bfac035cf02/workflow-job-run-1d849a45-2f30-5fbb-3226-b730a17a93af/artifacts/91e64594182918fa8012cdbf7d1a4f801fa0c35f485c3277268aad8e3f45377c.zip?sig=download"; private const string CreateArtifactResponse = @"{ - ""containerId"": 942031, - ""size"": -1, - ""signedContent"": null, - ""fileContainerResourceUrl"": """ + GitHubActionsInfoFixture.ActionRuntimeUrl + @"_apis/resources/Containers/942031"", - ""type"": ""actions_storage"", - ""name"": ""artifact"", - ""url"": """ + GitHubActionsInfoFixture.ActionRuntimeUrl + @"_apis/pipelines/1/runs/7/artifacts?artifactName=artifact"", - ""expiresOn"": ""2021-12-14T18:43:29.7431144Z"", - ""items"": null + ""ok"": true, + ""signed_upload_url"": """ + UploadFileUrl + @""" }"; - private const string CreateArtifactsResponse = @"{ - ""containerId"": 942031, - ""size"": -1, - ""signedContent"": null, - ""fileContainerResourceUrl"": """ + GitHubActionsInfoFixture.ActionRuntimeUrl + @"_apis/resources/Containers/942031"", - ""type"": ""actions_storage"", - ""name"": ""artifact"", - ""url"": """ + GitHubActionsInfoFixture.ActionRuntimeUrl + @"_apis/pipelines/1/runs/7/artifacts?artifactName=artifacts"", - ""expiresOn"": ""2021-12-14T18:43:29.7431144Z"", - ""items"": null + private const string FinalizeArtifactResponse = @"{ + ""ok"": true, + ""artifact_id"": ""1991105334"" }"; - - private const string PutDirectoryRootUrl = GitHubActionsInfoFixture.ActionRuntimeUrl + - "_apis/resources/Containers/942031?itemPath=artifacts%2Fartifact.txt"; - private const string PutDirectoryFolderAUrl = GitHubActionsInfoFixture.ActionRuntimeUrl + - "_apis/resources/Containers/942031?itemPath=artifacts%2Ffolder_a%2Fartifact.txt"; - private const string PutDirectoryFolderBUrl = GitHubActionsInfoFixture.ActionRuntimeUrl + - "_apis/resources/Containers/942031?itemPath=artifacts%2Ffolder_b%2Fartifact.txt"; - private const string PutDirectoryFolderBFolderCUrl = GitHubActionsInfoFixture.ActionRuntimeUrl + - "_apis/resources/Containers/942031?itemPath=artifacts%2Ffolder_b%2Ffolder_c%2Fartifact.txt"; - - private const string GetArtifactResourceUrl = GitHubActionsInfoFixture.ActionRuntimeUrl + - "_apis/pipelines/workflows/34058136/artifacts?api-version=6.0-preview&artifactName=artifact"; - private const string FileContainerResourceUrl = GitHubActionsInfoFixture.ActionRuntimeUrl + @"_apis/resources/Containers/4794789"; - private const string GetArtifactResourceResponse = @"{ - ""count"": 1, - ""value"": [ - { - ""containerId"": 4794789, - ""size"": 4, - ""signedContent"": null, - ""fileContainerResourceUrl"": """ + FileContainerResourceUrl + @""", - ""type"": ""actions_storage"", - ""name"": ""artifact"", - ""url"": """ + GitHubActionsInfoFixture.ActionRuntimeUrl + @"_apis/pipelines/1/runs/7/artifacts?artifactName=artifact"", - ""expiresOn"": ""2022-03-16T08:22:01.5699067Z"", - ""items"": null - } - ] -}"; - - private const string GetContainerItemResourcesUrl = FileContainerResourceUrl + "?itemPath=artifact"; - private const string GetContainerItemResourcesResponse = @"{ - ""count"": 1, - ""value"": [ - { - ""containerId"": 4794789, - ""scopeIdentifier"": ""00000000-0000-0000-0000-000000000000"", - ""path"": ""artifact/test.txt"", - ""itemType"": ""file"", - ""status"": ""created"", - ""fileLength"": 4, - ""fileEncoding"": 1, - ""fileType"": 1, - ""dateCreated"": ""2021-12-16T09:05:18.803Z"", - ""dateLastModified"": ""2021-12-16T09:05:18.907Z"", - ""createdBy"": ""2daeb16b-86ae-4e46-ba89-92a8aa076e52"", - ""lastModifiedBy"": ""2daeb16b-86ae-4e46-ba89-92a8aa076e52"", - ""itemLocation"": """ + GetContainerItemResourcesUrl + @"%2Ftest.txt&metadata=True"", - ""contentLocation"": """ + GetContainerItemResourcesUrl + @"%2Ftest.txt"", - ""fileId"": 1407, - ""contentId"": """" - } - ] + private const string GetSignedArtifactURLResponse = @"{ + ""name"": ""artifact"", + ""signed_url"": """ + DownloadFileUrl + @""" }"; - private const string DownloadItemResourceUrl = GetContainerItemResourcesUrl + "%2Ftest.txt"; - private const string DownloadItemResourceResponse = "Cake"; - private GitHubActionsInfoFixture GitHubActionsInfoFixture { get; } private ICakeEnvironment Environment { get; } public FakeFileSystem FileSystem { get; } @@ -159,123 +91,120 @@ public GitHubActionsCommandsFixture WithNoGitHubPath() protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { - if (request.Headers.Authorization is null || request.Headers.Authorization.Scheme != "Bearer" || request.Headers.Authorization.Parameter != GitHubActionsInfoFixture.ActionRuntimeToken) + if (request.RequestUri.AbsoluteUri == DownloadFileUrl) { - return new HttpResponseMessage - { - StatusCode = HttpStatusCode.Unauthorized - }; } - - if ( - !request.Headers.TryGetValues("Accept", out var values) - || !values.Contains(AcceptHeader)) + else if (request.RequestUri.AbsoluteUri == UploadFileUrl) { - if (request.RequestUri.AbsoluteUri != DownloadItemResourceUrl - || !values.Contains(AcceptGzip) - || !request.Headers.TryGetValues("Accept-Encoding", out var encodingValues) - || !encodingValues.Contains(AcceptEncodingGzip)) + if ( + !request.Content.Headers.TryGetValues("x-ms-blob-content-type", out var contentTypes) + || !contentTypes.Contains("application/zip") + || !request.Content.Headers.TryGetValues("x-ms-blob-type", out var blobTypes) + || !blobTypes.Contains("BlockBlob")) { return new HttpResponseMessage { - StatusCode = HttpStatusCode.BadRequest + StatusCode = HttpStatusCode.Unauthorized }; } } + else if (request.Headers.Authorization is null || request.Headers.Authorization.Scheme != "Bearer" || request.Headers.Authorization.Parameter != GitHubActionsInfoFixture.ActionRuntimeToken) + { + return new HttpResponseMessage + { + StatusCode = HttpStatusCode.Unauthorized + }; + } switch (request) { #pragma warning disable SA1013 - // Create Artifact FilePath + // Get Signed Artifact Url case { - RequestUri: { AbsoluteUri: CreateArtifactUrl }, + RequestUri: { AbsoluteUri: GetSignedArtifactURLurl }, Method: { Method: "POST" }, }: { - return Ok(new StringContent(CreateArtifactResponse)); + using var getSignedArtifactURLRequestStream = await request.Content.ReadAsStreamAsync(cancellationToken); + var getSignedArtifactURLRequest = await System.Text.Json.JsonSerializer.DeserializeAsync(getSignedArtifactURLRequestStream, cancellationToken: cancellationToken); + return getSignedArtifactURLRequest switch + { + { Name: { Length: >0}, WorkflowJobRunBackendId: { Length: >0}, WorkflowRunBackendId: { Length: >0 } } => Ok(new StringContent(GetSignedArtifactURLResponse)), + _ => new HttpResponseMessage + { + StatusCode = HttpStatusCode.BadRequest + } + }; } - // Create Artifact DirectoryPath + // Create Artifact case { - RequestUri: { AbsoluteUri: CreateArtifactsUrl }, + RequestUri: { AbsoluteUri: CreateArtifactUrl }, Method: { Method: "POST" }, }: { - return Ok(new StringContent(CreateArtifactsResponse)); - } + using var createArtifactRequestStream = await request.Content.ReadAsStreamAsync(cancellationToken); + var createArtifactRequest = await System.Text.Json.JsonSerializer.DeserializeAsync(createArtifactRequestStream, cancellationToken: cancellationToken); - // Download Artifact - Get Artifact Container Resource - case - { - RequestUri: { AbsoluteUri: GetArtifactResourceUrl }, - Method: { Method: "GET" } - }: - { - return Ok(new StringContent(GetArtifactResourceResponse)); + return createArtifactRequest switch + { + { Version: 4, Name: "artifact", } => Ok(new StringContent(CreateArtifactResponse)), + { Version: 4, Name: "artifacts", } => Ok(new StringContent(CreateArtifactResponse)), + _ => new HttpResponseMessage + { + StatusCode = HttpStatusCode.BadRequest + } + }; } - - // Download Artifact - Get Artifact Container Item Resource + // Finalize Artifact case { - RequestUri: { AbsoluteUri: GetContainerItemResourcesUrl }, - Method: { Method: "GET" } + RequestUri: { AbsoluteUri: FinalizeArtifactUrl }, + Method: { Method: "POST" }, }: { - return Ok(new StringContent(GetContainerItemResourcesResponse)); - } + using var createArtifactRequestStream = await request.Content.ReadAsStreamAsync(cancellationToken); + var finalizeArtifactRequest = await System.Text.Json.JsonSerializer.DeserializeAsync(createArtifactRequestStream, cancellationToken: cancellationToken); - // Download Artifact - DownloadItemResource - case - { - RequestUri: { AbsoluteUri: DownloadItemResourceUrl }, - Method: { Method: "GET" } - }: - { - return Ok(new StringContent(DownloadItemResourceResponse)); + return finalizeArtifactRequest switch + { + { Hash: { Length: > 0}, Size: >0, Name: "artifact", } => Ok(new StringContent(FinalizeArtifactResponse)), + { Hash: { Length: > 0 }, Size: > 0, Name: "artifacts", } => Ok(new StringContent(FinalizeArtifactResponse)), + _ => new HttpResponseMessage + { + StatusCode = HttpStatusCode.BadRequest + } + }; } - // Put FilePath + // Upload File case { - RequestUri: { AbsoluteUri: PutFileUrl }, + RequestUri: { AbsoluteUri: UploadFileUrl }, Method: { Method: "PUT" } }: - case - { - RequestUri: { AbsoluteUri: CreateArtifactUrl }, - Method: { Method: "PATCH" }, - }: + { + return Ok(); + } - // Put DirectoryPath - case - { - RequestUri: { AbsoluteUri: PutDirectoryRootUrl }, - Method: { Method: "PUT" } - }: - case - { - RequestUri: { AbsoluteUri: PutDirectoryFolderAUrl }, - Method: { Method: "PUT" } - }: - case - { - RequestUri: { AbsoluteUri: PutDirectoryFolderBUrl }, - Method: { Method: "PUT" } - }: + // Download File case { - RequestUri: { AbsoluteUri: PutDirectoryFolderBFolderCUrl }, - Method: { Method: "PUT" } - }: - case - { - RequestUri: { AbsoluteUri: CreateArtifactsUrl }, - Method: { Method: "PATCH" }, + RequestUri: { AbsoluteUri: DownloadFileUrl }, + Method: { Method: "GET" } }: { - return Ok(); + await using var stream = new System.IO.MemoryStream(); + using (var zip = new ZipArchive(stream, ZipArchiveMode.Create, true)) + { + var entry = zip.CreateEntry("test.txt"); + using var entryStream = entry.Open(); + using var writer = new System.IO.StreamWriter(entryStream); + writer.Write("Cake"); + } + return Ok(new ByteArrayContent(stream.ToArray())); } #pragma warning restore SA1013 diff --git a/src/Cake.Common.Tests/Fixtures/Build/GitHubActionsInfoFixture.cs b/src/Cake.Common.Tests/Fixtures/Build/GitHubActionsInfoFixture.cs index 3a82d8469f..73a510fb16 100644 --- a/src/Cake.Common.Tests/Fixtures/Build/GitHubActionsInfoFixture.cs +++ b/src/Cake.Common.Tests/Fixtures/Build/GitHubActionsInfoFixture.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using Cake.Common.Build.GitHubActions.Data; using Cake.Core; using NSubstitute; @@ -10,8 +11,11 @@ namespace Cake.Common.Tests.Fixtures.Build { internal sealed class GitHubActionsInfoFixture { - public const string ActionRuntimeToken = "zht1j5NeW2T5ZsOxncX4CUEiWYhD4ZRwoDghkARk"; + public const string ActionRuntimeToken = "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Ikh5cTROQVRBanNucUM3bWRydEFoaHJDUjJfUSJ9.eyJuYW1laWQiOiJkZGRkZGRkZC1kZGRkLWRkZGQtZGRkZC1kZGRkZGRkZGRkZGQiLCJzY3AiOiJBY3Rpb25zLkdlbmVyaWNSZWFkOjAwMDAwMDAwLTAwMDAtMDAwMC0wMDAwLTAwMDAwMDAwMDAwMCBBY3Rpb25zLlJlc3VsdHM6YjllMjgxNTMtY2EyMC00Yjg2LTkxZGQtMDllOGY2NDRlZmRmOjFkODQ5YTQ1LTJmMzAtNWZiYi0zMjI2LWI3MzBhMTdhOTNhZiBBY3Rpb25zLlVwbG9hZEFydGlmYWN0czowMDAwMDAwMC0wMDAwLTAwMDAtMDAwMC0wMDAwMDAwMDAwMDAvMTpCdWlsZC9CdWlsZC8xNiBMb2NhdGlvblNlcnZpY2UuQ29ubmVjdCBSZWFkQW5kVXBkYXRlQnVpbGRCeVVyaTowMDAwMDAwMC0wMDAwLTAwMDAtMDAwMC0wMDAwMDAwMDAwMDAvMTpCdWlsZC9CdWlsZC8xNiIsIklkZW50aXR5VHlwZUNsYWltIjoiU3lzdGVtOlNlcnZpY2VJZGVudGl0eSIsImh0dHA6Ly9zY2hlbWFzLnhtbHNvYXAub3JnL3dzLzIwMDUvMDUvaWRlbnRpdHkvY2xhaW1zL3NpZCI6IkRERERERERELUREREQtRERERC1ERERELURERERERERERERERCIsImh0dHA6Ly9zY2hlbWFzLm1pY3Jvc29mdC5jb20vd3MvMjAwOC8wNi9pZGVudGl0eS9jbGFpbXMvcHJpbWFyeXNpZCI6ImRkZGRkZGRkLWRkZGQtZGRkZC1kZGRkLWRkZGRkZGRkZGRkZCIsImF1aSI6ImUyMTI4OTY1LThlY2EtNDgxYy1hODhkLWJmOTFlZDg3Y2RiNSIsInNpZCI6ImMwNmVjY2E0LWY3ZjUtNGY4Mi1iM2IxLTJhYjM0M2Y4Mjg3NCIsImFjIjoiW3tcIlNjb3BlXCI6XCJyZWZzL2hlYWRzL21haW5cIixcIlBlcm1pc3Npb25cIjozfV0iLCJhY3NsIjoiMTAiLCJvcmNoaWQiOiJiOWUyODE1My1jYTIwLTRiODYtOTFkZC0wOWU4ZjY0NGVmZGYuYnVpbGQudWJ1bnR1LWxhdGVzdCIsImlzcyI6InZzdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20iLCJhdWQiOiJ2c3Rva2VuLmFjdGlvbnMuZ2l0aHVidXNlcmNvbnRlbnQuY29tfHZzbzo0M2YwNTdkMC0wODAzLTRkOTEtOTRhMS1mOGViMTAzZGYxMWYiLCJuYmYiOjE3Mjc1NDQzOTIsImV4cCI6MTcyNzU2NzE5Mn0.sUTvwxD-NlbAhQJB7cIInovd9qDkFHWcwOiiQAlHCsjpRBCEUWb3tWfOmCEpn8It4FWkaSszjMd8oecBEMlyEUtk6Cm6l1AqCUnIT13B48c_2sjhjWz-UDNMt94nzYH2ulC8mBcV_kSEIHJUvOnFKrFMKEdg6axAjLCx4la9MOklVq2ehx6DC12qbUNpTELJGeWz_JvKHWexyfN1qJgUw3y4ritZDJF3HLTpb5IJS7sQmFZVB7F2P6DF-1iaCBX5hgA9KfiwWXw6oTkKd6aOEyJpcBe0b87V_-fVTivOUS-ABE5XN6TCLZSmt7X6qwTPeSoLKgQGx1h_tHwubGDjtQ"; public const string ActionRuntimeUrl = "https://pipelines.actions.githubusercontent.com/ip0FyYnZXxdEOcOwPHkRsZJd2x6G5XoT486UsAb0/"; + public const string ActionResultsUrl = "https://results-receiver.actions.githubusercontent.com/"; + private readonly Version CakeTestVersion = new Version(1, 2, 3, 4); + public ICakeEnvironment Environment { get; } public GitHubActionsInfoFixture() @@ -55,11 +59,15 @@ public GitHubActionsInfoFixture() Environment.GetEnvironmentVariable("ACTIONS_RUNTIME_TOKEN").Returns(ActionRuntimeToken); Environment.GetEnvironmentVariable("ACTIONS_RUNTIME_URL").Returns(ActionRuntimeUrl); + Environment.GetEnvironmentVariable("ACTIONS_RESULTS_URL").Returns(ActionResultsUrl); Environment.GetEnvironmentVariable("GITHUB_ENV").Returns("/opt/github.env"); Environment.GetEnvironmentVariable("GITHUB_OUTPUT").Returns("/opt/github.output"); Environment.GetEnvironmentVariable("GITHUB_STEP_SUMMARY").Returns("/opt/github.stepsummary"); Environment.GetEnvironmentVariable("GITHUB_PATH").Returns("/opt/github.path"); Environment.WorkingDirectory.Returns("/home/runner/work/cake/cake"); + + Environment.GetSpecialPath(Core.IO.SpecialPath.LocalTemp).Returns("/tmp"); + Environment.Runtime.CakeVersion.Returns(CakeTestVersion); } public GitHubActionsRunnerInfo CreateRunnerInfo(string architecture = null) diff --git a/src/Cake.Common.Tests/Unit/Build/GitHubActions/Data/GitHubActionsRuntimeInfoTests.cs b/src/Cake.Common.Tests/Unit/Build/GitHubActions/Data/GitHubActionsRuntimeInfoTests.cs index f9a966960f..758618f565 100644 --- a/src/Cake.Common.Tests/Unit/Build/GitHubActions/Data/GitHubActionsRuntimeInfoTests.cs +++ b/src/Cake.Common.Tests/Unit/Build/GitHubActions/Data/GitHubActionsRuntimeInfoTests.cs @@ -33,7 +33,9 @@ public void Should_Return_Correct_Value() var result = info.Token; // Then - Assert.Equal("zht1j5NeW2T5ZsOxncX4CUEiWYhD4ZRwoDghkARk", result); + Assert.Equal( + "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Ikh5cTROQVRBanNucUM3bWRydEFoaHJDUjJfUSJ9.eyJuYW1laWQiOiJkZGRkZGRkZC1kZGRkLWRkZGQtZGRkZC1kZGRkZGRkZGRkZGQiLCJzY3AiOiJBY3Rpb25zLkdlbmVyaWNSZWFkOjAwMDAwMDAwLTAwMDAtMDAwMC0wMDAwLTAwMDAwMDAwMDAwMCBBY3Rpb25zLlJlc3VsdHM6YjllMjgxNTMtY2EyMC00Yjg2LTkxZGQtMDllOGY2NDRlZmRmOjFkODQ5YTQ1LTJmMzAtNWZiYi0zMjI2LWI3MzBhMTdhOTNhZiBBY3Rpb25zLlVwbG9hZEFydGlmYWN0czowMDAwMDAwMC0wMDAwLTAwMDAtMDAwMC0wMDAwMDAwMDAwMDAvMTpCdWlsZC9CdWlsZC8xNiBMb2NhdGlvblNlcnZpY2UuQ29ubmVjdCBSZWFkQW5kVXBkYXRlQnVpbGRCeVVyaTowMDAwMDAwMC0wMDAwLTAwMDAtMDAwMC0wMDAwMDAwMDAwMDAvMTpCdWlsZC9CdWlsZC8xNiIsIklkZW50aXR5VHlwZUNsYWltIjoiU3lzdGVtOlNlcnZpY2VJZGVudGl0eSIsImh0dHA6Ly9zY2hlbWFzLnhtbHNvYXAub3JnL3dzLzIwMDUvMDUvaWRlbnRpdHkvY2xhaW1zL3NpZCI6IkRERERERERELUREREQtRERERC1ERERELURERERERERERERERCIsImh0dHA6Ly9zY2hlbWFzLm1pY3Jvc29mdC5jb20vd3MvMjAwOC8wNi9pZGVudGl0eS9jbGFpbXMvcHJpbWFyeXNpZCI6ImRkZGRkZGRkLWRkZGQtZGRkZC1kZGRkLWRkZGRkZGRkZGRkZCIsImF1aSI6ImUyMTI4OTY1LThlY2EtNDgxYy1hODhkLWJmOTFlZDg3Y2RiNSIsInNpZCI6ImMwNmVjY2E0LWY3ZjUtNGY4Mi1iM2IxLTJhYjM0M2Y4Mjg3NCIsImFjIjoiW3tcIlNjb3BlXCI6XCJyZWZzL2hlYWRzL21haW5cIixcIlBlcm1pc3Npb25cIjozfV0iLCJhY3NsIjoiMTAiLCJvcmNoaWQiOiJiOWUyODE1My1jYTIwLTRiODYtOTFkZC0wOWU4ZjY0NGVmZGYuYnVpbGQudWJ1bnR1LWxhdGVzdCIsImlzcyI6InZzdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20iLCJhdWQiOiJ2c3Rva2VuLmFjdGlvbnMuZ2l0aHVidXNlcmNvbnRlbnQuY29tfHZzbzo0M2YwNTdkMC0wODAzLTRkOTEtOTRhMS1mOGViMTAzZGYxMWYiLCJuYmYiOjE3Mjc1NDQzOTIsImV4cCI6MTcyNzU2NzE5Mn0.sUTvwxD-NlbAhQJB7cIInovd9qDkFHWcwOiiQAlHCsjpRBCEUWb3tWfOmCEpn8It4FWkaSszjMd8oecBEMlyEUtk6Cm6l1AqCUnIT13B48c_2sjhjWz-UDNMt94nzYH2ulC8mBcV_kSEIHJUvOnFKrFMKEdg6axAjLCx4la9MOklVq2ehx6DC12qbUNpTELJGeWz_JvKHWexyfN1qJgUw3y4ritZDJF3HLTpb5IJS7sQmFZVB7F2P6DF-1iaCBX5hgA9KfiwWXw6oTkKd6aOEyJpcBe0b87V_-fVTivOUS-ABE5XN6TCLZSmt7X6qwTPeSoLKgQGx1h_tHwubGDjtQ", + result); } } diff --git a/src/Cake.Common/Build/GitHubActions/Commands/Artifact/CreateArtifactRequest.cs b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/CreateArtifactRequest.cs new file mode 100644 index 0000000000..200c292518 --- /dev/null +++ b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/CreateArtifactRequest.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text.Json.Serialization; + +namespace Cake.Common.Build.GitHubActions.Commands.Artifact +{ +#pragma warning disable SA1313 // Parameter names should begin with lower-case letter + internal record CreateArtifactRequest( + [property: JsonPropertyName("version")] + int Version, + [property: JsonPropertyName("name")] + string Name, + [property: JsonPropertyName("workflow_run_backend_id")] + string WorkflowRunBackendId, + [property: JsonPropertyName("workflow_job_run_backend_id")] + string WorkflowJobRunBackendId); +#pragma warning restore SA1313 // Parameter names should begin with lower-case letter +} \ No newline at end of file diff --git a/src/Cake.Common/Build/GitHubActions/Commands/Artifact/CreateArtifactResponse.cs b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/CreateArtifactResponse.cs new file mode 100644 index 0000000000..aef73d7654 --- /dev/null +++ b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/CreateArtifactResponse.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text.Json.Serialization; + +namespace Cake.Common.Build.GitHubActions.Commands.Artifact +{ +#pragma warning disable SA1313 // Parameter names should begin with lower-case letter + internal record CreateArtifactResponse( + [property: JsonPropertyName("ok")] + bool Ok, + [property: JsonPropertyName("signed_upload_url")] + string SignedUploadUrl); +#pragma warning restore SA1313 // Parameter names should begin with lower-case letter +} \ No newline at end of file diff --git a/src/Cake.Common/Build/GitHubActions/Commands/Artifact/FinalizeArtifactRequest.cs b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/FinalizeArtifactRequest.cs new file mode 100644 index 0000000000..83552ad2d0 --- /dev/null +++ b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/FinalizeArtifactRequest.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text.Json.Serialization; + +namespace Cake.Common.Build.GitHubActions.Commands.Artifact +{ +#pragma warning disable SA1313 // Parameter names should begin with lower-case letter + internal record FinalizeArtifactRequest( + [property: JsonPropertyName("name")] + string Name, + [property: JsonPropertyName("hash")] + string Hash, + [property: JsonPropertyName("size")] + long Size, + [property: JsonPropertyName("workflow_run_backend_id")] + string WorkflowRunBackendId, + [property: JsonPropertyName("workflow_job_run_backend_id")] + string WorkflowJobRunBackendId); +#pragma warning restore SA1313 // Parameter names should begin with lower-case letter +} diff --git a/src/Cake.Common/Build/GitHubActions/Commands/Artifact/FinalizeArtifactResponse.cs b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/FinalizeArtifactResponse.cs new file mode 100644 index 0000000000..a42815b25d --- /dev/null +++ b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/FinalizeArtifactResponse.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text.Json.Serialization; + +namespace Cake.Common.Build.GitHubActions.Commands.Artifact +{ +#pragma warning disable SA1313 // Parameter names should begin with lower-case letter + internal record FinalizeArtifactResponse( + [property: JsonPropertyName("ok")] + bool Ok, + [property: JsonPropertyName("artifact_id")] + string ArtifactId); +#pragma warning restore SA1313 // Parameter names should begin with lower-case letter +} \ No newline at end of file diff --git a/src/Cake.Common/Build/GitHubActions/Commands/Artifact/GetSignedArtifactURLRequest.cs b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/GetSignedArtifactURLRequest.cs new file mode 100644 index 0000000000..deea3b1c10 --- /dev/null +++ b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/GetSignedArtifactURLRequest.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text.Json.Serialization; + +namespace Cake.Common.Build.GitHubActions.Commands.Artifact +{ +#pragma warning disable SA1313 // Parameter names should begin with lower-case letter + internal record GetSignedArtifactURLRequest( + [property: JsonPropertyName("workflow_run_backend_id")] + string WorkflowRunBackendId, + [property: JsonPropertyName("workflow_job_run_backend_id")] + string WorkflowJobRunBackendId, + [property: JsonPropertyName("name")] + string Name); +#pragma warning restore SA1313 // Parameter names should begin with lower-case letter +} \ No newline at end of file diff --git a/src/Cake.Common/Build/GitHubActions/Commands/Artifact/GetSignedArtifactURLResponse.cs b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/GetSignedArtifactURLResponse.cs new file mode 100644 index 0000000000..6bb1719dcc --- /dev/null +++ b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/GetSignedArtifactURLResponse.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text.Json.Serialization; +namespace Cake.Common.Build.GitHubActions.Commands.Artifact +{ +#pragma warning disable SA1313 // Parameter names should begin with lower-case letter + internal record GetSignedArtifactURLResponse( + [property: JsonPropertyName("name")] + string Name, + [property: JsonPropertyName("signed_url")] + string SignedUrl); +#pragma warning restore SA1313 // Parameter names should begin with lower-case letter +} \ No newline at end of file diff --git a/src/Cake.Common/Build/GitHubActions/Commands/Artifact/GitHubActionsArtifactService.cs b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/GitHubActionsArtifactService.cs new file mode 100644 index 0000000000..dc686e6ace --- /dev/null +++ b/src/Cake.Common/Build/GitHubActions/Commands/Artifact/GitHubActionsArtifactService.cs @@ -0,0 +1,311 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO.Compression; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Security.Cryptography; +using System.Text.Json; +using System.Threading.Tasks; +using Cake.Common.Build.GitHubActions.Data; +using Cake.Core; +using Cake.Core.IO; +using Microsoft.IdentityModel.JsonWebTokens; + +namespace Cake.Common.Build.GitHubActions.Commands.Artifact +{ + internal record GitHubActionsArtifactService( +#pragma warning disable SA1313 // Parameter names should begin with lower-case letter + ICakeEnvironment Environment, + IFileSystem FileSystem, + GitHubActionsEnvironmentInfo ActionsEnvironment, + Func CreateHttpClient) +#pragma warning restore SA1313 // Parameter names should begin with lower-case letter + { + private const string JsonContentType = "application/json"; + private const string ZipContentType = "application/zip"; + private static readonly Uri CreateArtifactUrl = new Uri("CreateArtifact", UriKind.Relative); + private static readonly Uri FinalizeArtifactUrl = new Uri("FinalizeArtifact", UriKind.Relative); + private static readonly Uri GetSignedArtifactURLUrl = new Uri("GetSignedArtifactURL", UriKind.Relative); + + internal async Task DownloadArtifactFiles( + string artifactName, + DirectoryPath directoryPath) + { + GetWorkflowBackendIds(out var workflowRunBackendId, out var workflowJobRunBackendId); + + var (_, signedUrl) = await GetSignedArtifactURL(workflowRunBackendId, workflowJobRunBackendId, artifactName); + + await DownloadArtifact(signedUrl, directoryPath); + } + + private async Task DownloadArtifact(string signedUrl, DirectoryPath directoryPath) + { + if (string.IsNullOrWhiteSpace(signedUrl)) + { + throw new ArgumentNullException(nameof(signedUrl)); + } + + using var downloadClient = GetStorageHttpClient(); + using var downloadResponse = await downloadClient.GetAsync(signedUrl); + + if (!downloadResponse.IsSuccessStatusCode) + { + throw new CakeException($"Artifact download failed {downloadResponse.StatusCode:F} ({downloadResponse.StatusCode:D})."); + } + + await using var downloadStream = await downloadResponse.Content.ReadAsStreamAsync(); + using var archive = new ZipArchive(downloadStream, ZipArchiveMode.Read); + foreach (var entry in archive.Entries) + { + var entryPath = directoryPath.CombineWithFilePath(entry.FullName); + if (FileSystem.GetFile(entryPath).Exists) + { + FileSystem.GetFile(entryPath).Delete(); + } + else if (FileSystem.GetDirectory(entryPath.GetDirectory()) is { Exists: false } entryDirectory) + { + entryDirectory.Create(); + } + using var entryStream = entry.Open(); + using var fileStream = FileSystem.GetFile(entryPath).OpenWrite(); + await entryStream.CopyToAsync(fileStream); + } + } + + private async Task GetSignedArtifactURL( + string workflowRunBackendId, + string workflowJobRunBackendId, + string artifactName) + { + var getSignedArtifactURLRequest = new GetSignedArtifactURLRequest( + workflowRunBackendId, + workflowJobRunBackendId, + artifactName); + + return await PostArtifactService( + GetSignedArtifactURLUrl, + getSignedArtifactURLRequest); + } + + internal async Task CreateAndUploadArtifactFiles( + string artifactName, + DirectoryPath rootPath, + params IFile[] files) + { + var tempArchivePath = Environment + .GetSpecialPath(SpecialPath.LocalTemp) + .CombineWithFilePath($"{Guid.NewGuid():n}.zip"); + + try + { + GetWorkflowBackendIds(out var workflowRunBackendId, out var workflowJobRunBackendId); + + await CreateArtifactArchive(rootPath, files, tempArchivePath); + + (long size, string hash) = GetArtifactArchiveSizeAndHash(tempArchivePath); + + var signedUploadUrl = await CreateArtifact(artifactName, workflowRunBackendId, workflowJobRunBackendId); + + await UploadArtifact(tempArchivePath, size, signedUploadUrl); + + var artifactId = await FinalizeArtifact(artifactName, hash, size, workflowRunBackendId, workflowJobRunBackendId); + + return artifactId; + } + finally + { + if (FileSystem.GetFile(tempArchivePath).Exists) + { + FileSystem.GetFile(tempArchivePath).Delete(); + } + } + } + + private async Task PostArtifactService( + Uri uri, + TParam param, + [System.Runtime.CompilerServices.CallerMemberName] string memberName = null) + { + using var httpClient = GetArtifactsHttpClient(); + + var jsonData = JsonSerializer.SerializeToUtf8Bytes(param); + + using var response = await httpClient.PostAsync( + uri, + new ByteArrayContent(jsonData) + { + Headers = { ContentType = MediaTypeHeaderValue.Parse(JsonContentType) } + }); + + if (!response.IsSuccessStatusCode) + { + throw new CakeException($"Artifact service call {memberName} failed {response.StatusCode:F} ({response.StatusCode:D})."); + } + + await using var responseStream = await response.Content.ReadAsStreamAsync(); + + return await JsonSerializer.DeserializeAsync(responseStream); + } + + private async Task CreateArtifact(string artifactName, string workflowRunBackendId, string workflowJobRunBackendId) + { + var createArtifactRequest = new CreateArtifactRequest( + 4, + artifactName, + workflowRunBackendId, + workflowJobRunBackendId); + + var (ok, signedUploadUrl) = await PostArtifactService( + CreateArtifactUrl, + createArtifactRequest); + + if (!ok) + { + throw new CakeException("Artifact creation failed."); + } + + if (string.IsNullOrWhiteSpace(signedUploadUrl)) + { + throw new CakeException("Artifact upload url missing."); + } + + return signedUploadUrl; + } + + private async Task UploadArtifact(FilePath contentPath, long contentLength, string signedUploadUrl) + { + using var uploadClient = GetStorageHttpClient(); + await using var uploadStream = FileSystem.GetFile(contentPath).OpenRead(); + using var uploadContent = new StreamContent(uploadStream) + { + Headers = + { + ContentType = MediaTypeHeaderValue.Parse(ZipContentType), + ContentLength = contentLength + } + }; + uploadContent.Headers.TryAddWithoutValidation("x-ms-blob-content-type", ZipContentType); + uploadContent.Headers.TryAddWithoutValidation("x-ms-blob-type", "BlockBlob"); + + using var response = await uploadClient.PutAsync( + signedUploadUrl, + uploadContent); + + if (!response.IsSuccessStatusCode) + { + throw new CakeException($"Artifact upload failed {response.StatusCode:F} ({response.StatusCode:D})."); + } + } + + private async Task FinalizeArtifact( + string artifactName, + string hash, + long contentLength, + string workflowRunBackendId, + string workflowJobRunBackendId) + { + var finalizeArtifactRequest = new FinalizeArtifactRequest( + artifactName, + hash, + contentLength, + workflowRunBackendId, + workflowJobRunBackendId); + + var (ok, artifactId) = await PostArtifactService( + FinalizeArtifactUrl, + finalizeArtifactRequest); + + if (!ok) + { + throw new CakeException("Artifact finalization failed."); + } + + if (string.IsNullOrWhiteSpace(artifactId)) + { + throw new CakeException("Artifact id missing."); + } + + return artifactId; + } + + private (long size, string hash) GetArtifactArchiveSizeAndHash(FilePath tempArchivePath) + { + var size = FileSystem.GetFile(tempArchivePath).Length; + using var stream = FileSystem.GetFile(tempArchivePath).OpenRead(); + using var sha256 = SHA256.Create(); + var hashBytes = sha256.ComputeHash(stream); + var hash = Convert.ToHexString(hashBytes).ToLowerInvariant(); + return (size, hash); + } + + private void GetWorkflowBackendIds(out string workflowRunBackendId, out string workflowJobRunBackendId) + { + try + { + var jwt = new JsonWebToken(ActionsEnvironment.Runtime.Token); + (workflowRunBackendId, workflowJobRunBackendId) = jwt.TryGetClaim("scp", out var scope) + ? scope.Value.Split(' ').FirstOrDefault(s => s.StartsWith("Actions.Results:"))?.Split(':') is { Length: 3 } workflowRunBackendParts + ? (workflowRunBackendParts[1], + workflowRunBackendParts[2]) + : default + : default; + + if (string.IsNullOrWhiteSpace(workflowRunBackendId)) + { + throw new CakeException("GitHub Actions Workflow Token workflowRunBackendId missing."); + } + + if (string.IsNullOrWhiteSpace(workflowJobRunBackendId)) + { + throw new CakeException("GitHub Actions Workflow Token workflowJobRunBackendId missing."); + } + } + catch (Exception ex) + { + throw new CakeException("GitHub Actions Workflow Token invalid.", ex); + } + } + + private async Task CreateArtifactArchive(DirectoryPath rootPath, IFile[] files, FilePath tempArchivePath) + { + if (FileSystem.GetDirectory(tempArchivePath.GetDirectory()) is { Exists: false } tempArchiveDirectory) + { + tempArchiveDirectory.Create(); + } + + using var archiveStream = FileSystem.GetFile(tempArchivePath).OpenWrite(); + using var archive = new ZipArchive(archiveStream, ZipArchiveMode.Create); + foreach (var file in files) + { + var relativePath = rootPath.GetRelativePath(file.Path.GetDirectory()); + var entry = archive.CreateEntry(relativePath.CombineWithFilePath(file.Path.GetFilename()).FullPath, CompressionLevel.SmallestSize); + using var entryStream = entry.Open(); + using var fileStream = file.OpenRead(); + await fileStream.CopyToAsync(entryStream); + } + } + + private HttpClient GetArtifactsHttpClient([System.Runtime.CompilerServices.CallerMemberName] string memberName = null) + { + var client = CreateHttpClient(memberName); + client.DefaultRequestHeaders.Accept.Add(MediaTypeWithQualityHeaderValue.Parse(JsonContentType)); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", ActionsEnvironment.Runtime.Token); + client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("Cake", Environment.Runtime.CakeVersion.ToString())); + client.BaseAddress = new Uri(string.Concat( + ActionsEnvironment.Runtime.ResultsUrl, + "twirp/github.actions.results.api.v1.ArtifactService/")); + return client; + } + + private HttpClient GetStorageHttpClient([System.Runtime.CompilerServices.CallerMemberName] string memberName = null) + { + var client = CreateHttpClient(memberName); + client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("Cake", Environment.Runtime.CakeVersion.ToString())); + return client; + } + } +} \ No newline at end of file diff --git a/src/Cake.Common/Build/GitHubActions/Commands/ArtifactResponse.cs b/src/Cake.Common/Build/GitHubActions/Commands/ArtifactResponse.cs deleted file mode 100644 index 7fc9ddd6c7..0000000000 --- a/src/Cake.Common/Build/GitHubActions/Commands/ArtifactResponse.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Text.Json.Serialization; - -namespace Cake.Common.Build.GitHubActions.Commands -{ - internal sealed class ArtifactResponse - { - [JsonPropertyName("containerId")] - public long ContainerId { get; set; } - - [JsonPropertyName("size")] - public long Size { get; set; } - - [JsonPropertyName("signedContent")] - public string SignedContent { get; set; } - - [JsonPropertyName("fileContainerResourceUrl")] - public string FileContainerResourceUrl { get; set; } - - [JsonPropertyName("type")] - public string Type { get; set; } - - [JsonPropertyName("name")] - public string Name { get; set; } - - [JsonPropertyName("url")] - public string Url { get; set; } - } -} diff --git a/src/Cake.Common/Build/GitHubActions/Commands/ContainerItem.cs b/src/Cake.Common/Build/GitHubActions/Commands/ContainerItem.cs deleted file mode 100644 index f9fd045659..0000000000 --- a/src/Cake.Common/Build/GitHubActions/Commands/ContainerItem.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Text.Json.Serialization; - -namespace Cake.Common.Build.GitHubActions.Commands -{ - internal sealed class ContainerItem - { - [JsonPropertyName("containerId")] - public long ContainerId { get; set; } - - [JsonPropertyName("size")] - public long Size { get; set; } - - [JsonPropertyName("fileContainerResourceUrl")] - public string FileContainerResourceUrl { get; set; } - - [JsonPropertyName("type")] - public string Type { get; set; } - - [JsonPropertyName("name")] - public string Name { get; set; } - - [JsonPropertyName("url")] - public string Url { get; set; } - - [JsonPropertyName("expiresOn")] - public DateTimeOffset ExpiresOn { get; set; } - } -} diff --git a/src/Cake.Common/Build/GitHubActions/Commands/ContainerItemResource.cs b/src/Cake.Common/Build/GitHubActions/Commands/ContainerItemResource.cs deleted file mode 100644 index 8065fc0a31..0000000000 --- a/src/Cake.Common/Build/GitHubActions/Commands/ContainerItemResource.cs +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Text.Json.Serialization; - -namespace Cake.Common.Build.GitHubActions.Commands -{ - internal class ContainerItemResource - { - [JsonPropertyName("containerId")] - public long ContainerId { get; set; } - - [JsonPropertyName("scopeIdentifier")] - public Guid ScopeIdentifier { get; set; } - - [JsonPropertyName("path")] - public string Path { get; set; } - - [JsonPropertyName("itemType")] - public string ItemType { get; set; } - - [JsonPropertyName("status")] - public string Status { get; set; } - - [JsonPropertyName("dateCreated")] - public DateTimeOffset DateCreated { get; set; } - - [JsonPropertyName("dateLastModified")] - public DateTimeOffset DateLastModified { get; set; } - - [JsonPropertyName("createdBy")] - public Guid CreatedBy { get; set; } - - [JsonPropertyName("lastModifiedBy")] - public Guid LastModifiedBy { get; set; } - - [JsonPropertyName("itemLocation")] - public string ItemLocation { get; set; } - - [JsonPropertyName("contentLocation")] - public string ContentLocation { get; set; } - - [JsonPropertyName("contentId")] - public string ContentId { get; set; } - - [JsonPropertyName("fileLength")] - public long? FileLength { get; set; } - - [JsonPropertyName("fileEncoding")] - public long? FileEncoding { get; set; } - - [JsonPropertyName("fileType")] - public long? FileType { get; set; } - - [JsonPropertyName("fileId")] - public long? FileId { get; set; } - } -} diff --git a/src/Cake.Common/Build/GitHubActions/Commands/CreateArtifactParameters.cs b/src/Cake.Common/Build/GitHubActions/Commands/CreateArtifactParameters.cs deleted file mode 100644 index 9930c25bbe..0000000000 --- a/src/Cake.Common/Build/GitHubActions/Commands/CreateArtifactParameters.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -namespace Cake.Common.Build.GitHubActions.Commands -{ - internal sealed class CreateArtifactParameters - { - public string Name { get; set; } - - public string Type { get; set; } - - public CreateArtifactParameters(string name) - { - Name = name; - Type = "actions_storage"; - } - } -} diff --git a/src/Cake.Common/Build/GitHubActions/Commands/GitHubActionsCommands.cs b/src/Cake.Common/Build/GitHubActions/Commands/GitHubActionsCommands.cs index 8a1fb0966f..8d4a141bb8 100644 --- a/src/Cake.Common/Build/GitHubActions/Commands/GitHubActionsCommands.cs +++ b/src/Cake.Common/Build/GitHubActions/Commands/GitHubActionsCommands.cs @@ -7,9 +7,8 @@ using System.IO; using System.Linq; using System.Net.Http; -using System.Net.Http.Headers; -using System.Text.Json; using System.Threading.Tasks; +using Cake.Common.Build.GitHubActions.Commands.Artifact; using Cake.Common.Build.GitHubActions.Data; using Cake.Core; using Cake.Core.IO; @@ -21,17 +20,11 @@ namespace Cake.Common.Build.GitHubActions.Commands /// public sealed class GitHubActionsCommands { - private const string ApiVersion = "6.0-preview"; - private const string AcceptHeader = "application/json; api-version=" + ApiVersion; - private const string ContentTypeHeader = "application/json"; - private const string AcceptGzip = "application/octet-stream; api-version=" + ApiVersion; - private const string AcceptEncodingGzip = "gzip"; - private readonly ICakeEnvironment _environment; private readonly IFileSystem _fileSystem; private readonly IBuildSystemServiceMessageWriter _writer; private readonly GitHubActionsEnvironmentInfo _actionsEnvironment; - private readonly Func _createHttpClient; + private readonly GitHubActionsArtifactService _artifactsService; /// /// Initializes a new instance of the class. @@ -52,7 +45,7 @@ public GitHubActionsCommands( _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); _writer = writer ?? throw new ArgumentNullException(nameof(writer)); _actionsEnvironment = actionsEnvironment ?? throw new ArgumentNullException(nameof(actionsEnvironment)); - _createHttpClient = createHttpClient ?? throw new ArgumentNullException(nameof(createHttpClient)); + _artifactsService = new GitHubActionsArtifactService(environment, fileSystem, actionsEnvironment, createHttpClient ?? throw new ArgumentNullException(nameof(createHttpClient))); } /// @@ -240,7 +233,7 @@ public async Task UploadArtifact(FilePath path, string artifactName) throw new FileNotFoundException("Artifact file not found.", file.Path.FullPath); } - await CreateAndUploadArtifactFiles(artifactName, file.Path.GetDirectory(), file); + await _artifactsService.CreateAndUploadArtifactFiles(artifactName, file.Path.GetDirectory(), file); } /// @@ -262,7 +255,7 @@ public async Task UploadArtifact(DirectoryPath path, string artifactName) .GetFiles("*", SearchScope.Recursive) .ToArray(); - await CreateAndUploadArtifactFiles(artifactName, directory.Path, files); + await _artifactsService.CreateAndUploadArtifactFiles(artifactName, directory.Path, files); } /// @@ -280,17 +273,7 @@ public async Task DownloadArtifact(string artifactName, DirectoryPath path) throw new DirectoryNotFoundException(FormattableString.Invariant($"Local directory {directory.Path.FullPath} not found.")); } - var client = GetRuntimeHttpClient(); - - var artifactResourceUrl = await GetArtifactResourceUrl(client, artifactName); - - var containerItemResources = await GetContainerItemResources( - client, - directory.Path, - artifactName, - artifactResourceUrl); - - await DownloadItemResources(client, containerItemResources); + await _artifactsService.DownloadArtifactFiles(artifactName, directory.Path); } internal void WriteCommand(string command, string message = null) @@ -309,116 +292,6 @@ internal void WriteCommand(string command, Dictionary parameters private static string EscapeCommandParameter(string value) => (value ?? string.Empty).Replace("%", "%25").Replace("\r", "%0D").Replace("\n", "%0A").Replace(":", "%3A").Replace(",", "%2C"); - private async Task DownloadItemResources(HttpClient client, (FilePath FilePath, string ContentLocation, long FileLength)[] containerItemResourceContent) - { - foreach (var (filePath, contentLocation, fileLength) in containerItemResourceContent) - { - await DownloadItemResource(client, filePath, contentLocation, fileLength); - } - } - - private async Task DownloadItemResource( - HttpClient client, - FilePath filePath, - string contentLocation, - long fileLength) - { - var contentDirectory = _fileSystem.GetDirectory(filePath.GetDirectory()); - - if (!contentDirectory.Exists) - { - contentDirectory.Create(); - } - - var contentFile = _fileSystem.GetFile(filePath); - - using var contentFileStream = contentFile.OpenWrite(); - - if (fileLength == 0) - { - return; - } - - using var contentResponse = await client.SendAsync( - new HttpRequestMessage( - HttpMethod.Get, - contentLocation) - { - Headers = - { - Accept = { MediaTypeWithQualityHeaderValue.Parse(AcceptGzip) }, - AcceptEncoding = { StringWithQualityHeaderValue.Parse(AcceptEncodingGzip) } - } - }); - - contentResponse.EnsureSuccessStatusCode(); - - using var contentResponseStream = await contentResponse.Content.ReadAsStreamAsync(); - - await contentResponseStream.CopyToAsync(contentFileStream); - } - - private async Task<(FilePath LocalPath, string ContentLocation, long FileLength)[]> GetContainerItemResources( - HttpClient client, - DirectoryPath path, - string artifactName, - string artifactResourceUrl) - { - using var resourceResponse = await client.GetAsync(artifactResourceUrl); - - resourceResponse.EnsureSuccessStatusCode(); - - using var resourceResponseStream = await resourceResponse.Content.ReadAsStreamAsync(); - - var containerItemResource = await JsonSerializer.DeserializeAsync>(resourceResponseStream); - - int artifactNameLength = artifactName.Length; - int relativePathStart = artifactNameLength + 1; - var containerItemResourceContent = - containerItemResource - ?.Value - .Where(content => content?.Path is string path - && path.Length > relativePathStart - && path[artifactNameLength] is char separator - && (separator == '/' || separator == '\\') - && path.StartsWith(artifactName) - && content.ItemType?.ToLowerInvariant() == "file") - .Select(content => (path.CombineWithFilePath(content.Path[relativePathStart..]), - content.ContentLocation, - content.FileLength ?? 0)) - .ToArray(); - - if (containerItemResourceContent == null || containerItemResourceContent.Length == 0) - { - throw new Exception($"Artifact \"{artifactName}\" content not found."); - } - - return containerItemResourceContent; - } - - private async Task GetArtifactResourceUrl(HttpClient client, string artifactName) - { - var artifactUrl = GetArtifactUrl(artifactName); - - using var containerResponse = await client.GetAsync(artifactUrl); - - containerResponse.EnsureSuccessStatusCode(); - - using var containerResponseStream = await containerResponse.Content.ReadAsStreamAsync(); - - var containerItems = await JsonSerializer.DeserializeAsync>(containerResponseStream); - - var artifactsLookup = (containerItems?.Value ?? Array.Empty()) - .Where(item => !string.IsNullOrWhiteSpace(item.FileContainerResourceUrl) - && !string.IsNullOrWhiteSpace(item.Name)) - .ToLookup( - key => key.Name, - item => string.Concat(item.FileContainerResourceUrl, "?itemPath=", Uri.EscapeDataString(item.Name))); - - return artifactsLookup[artifactName].FirstOrDefault() - ?? throw new Exception($"Artifact \"{artifactName}\" not found."); - } - private T ValidateArtifactParameters(T path, string artifactName) where T : IPath { if (path is null) @@ -448,116 +321,5 @@ private T ValidateArtifactParameters(T path, string artifactName) where T : I return path.MakeAbsolute(_environment); } - - private async Task CreateAndUploadArtifactFiles( - string artifactName, - DirectoryPath rootPath, - params IFile[] files) - { - var artifactUrl = GetArtifactUrl(artifactName); - - var client = GetRuntimeHttpClient(); - - var artifactResponse = await CreateArtifact(artifactName, client, artifactUrl); - - long totalFileSize = 0L; - foreach (var file in files) - { - using var artifactStream = file.OpenRead(); - await UploadFile(rootPath, artifactName, artifactResponse, client, artifactStream, file); - totalFileSize += file.Length; - } - - await FinalizeArtifact(client, artifactUrl, totalFileSize); - } - - private string GetArtifactUrl(string artifactName) - { - return string.Concat( - _actionsEnvironment.Runtime.Url, - "_apis/pipelines/workflows/", - _actionsEnvironment.Workflow.RunId, - "/artifacts?api-version=", - ApiVersion, - "&artifactName=", - Uri.EscapeDataString(artifactName)); - } - - private HttpClient GetRuntimeHttpClient([System.Runtime.CompilerServices.CallerMemberName] string memberName = null) - { - var client = _createHttpClient(memberName); - client.DefaultRequestHeaders.Accept.Add(MediaTypeWithQualityHeaderValue.Parse(AcceptHeader)); - client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _actionsEnvironment.Runtime.Token); - return client; - } - - private static async Task FinalizeArtifact(HttpClient client, string artifactUrl, long totalSize) - { - var jsonData = JsonSerializer.SerializeToUtf8Bytes(new PatchArtifactSize(totalSize)); - - var patchResponse = await client.SendAsync( - new HttpRequestMessage( - new HttpMethod("PATCH"), - artifactUrl) - { - Content = new ByteArrayContent(jsonData) - { - Headers = { ContentType = MediaTypeHeaderValue.Parse(ContentTypeHeader) } - } - }); - - patchResponse.EnsureSuccessStatusCode(); - } - - private static async Task UploadFile(DirectoryPath rootPath, string artifactName, ArtifactResponse artifactResponse, HttpClient client, Stream artifactStream, IFile file) - { - var itemPath = string.Concat( - artifactName, - "/", - rootPath.GetRelativePath(file.Path).FullPath); - - var putFileUrl = string.Concat( - artifactResponse?.FileContainerResourceUrl ?? throw new ArgumentNullException("FileContainerResourceUrl"), - $"?itemPath={Uri.EscapeDataString(itemPath)}"); - - var putResponse = await client.PutAsync( - putFileUrl, - new StreamContent(artifactStream) - { - Headers = - { - ContentType = MediaTypeHeaderValue.Parse("application/octet-stream"), - ContentLength = file.Length, - ContentRange = new ContentRangeHeaderValue(0, file.Length - 1L, file.Length) - } - }); - - if (!putResponse.IsSuccessStatusCode) - { - throw new CakeException( - FormattableString.Invariant($"Put artifact file {itemPath} failed."), - new HttpRequestException( - FormattableString.Invariant($"Response status code does not indicate success: {putResponse.StatusCode:d} ({putResponse.ReasonPhrase})."))); - } - } - - private static async Task CreateArtifact(string artifactName, HttpClient client, string artifactUrl) - { - var jsonData = JsonSerializer.SerializeToUtf8Bytes(new CreateArtifactParameters(artifactName)); - var response = await client.PostAsync( - artifactUrl, - new ByteArrayContent(jsonData) - { - Headers = { ContentType = MediaTypeHeaderValue.Parse(ContentTypeHeader) } - }); - - response.EnsureSuccessStatusCode(); - - using var responseStream = await response.Content.ReadAsStreamAsync(); - var artifactResponse = await JsonSerializer.DeserializeAsync(responseStream) - ?? throw new CakeException("Failed to parse ArtifactResponse"); - - return artifactResponse; - } } } diff --git a/src/Cake.Common/Build/GitHubActions/Commands/PatchArtifactSize.cs b/src/Cake.Common/Build/GitHubActions/Commands/PatchArtifactSize.cs deleted file mode 100644 index f70e245251..0000000000 --- a/src/Cake.Common/Build/GitHubActions/Commands/PatchArtifactSize.cs +++ /dev/null @@ -1,16 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -namespace Cake.Common.Build.GitHubActions.Commands -{ - internal sealed class PatchArtifactSize - { - public long Size { get; set; } - - public PatchArtifactSize(long size) - { - Size = size; - } - } -} \ No newline at end of file diff --git a/src/Cake.Common/Build/GitHubActions/Commands/Values.cs b/src/Cake.Common/Build/GitHubActions/Commands/Values.cs deleted file mode 100644 index 73c44ad61f..0000000000 --- a/src/Cake.Common/Build/GitHubActions/Commands/Values.cs +++ /dev/null @@ -1,17 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Text.Json.Serialization; - -namespace Cake.Common.Build.GitHubActions.Commands -{ - internal sealed class Values - { - [JsonPropertyName("count")] - public long Count { get; set; } - - [JsonPropertyName("value")] - public T[] Value { get; set; } - } -} diff --git a/src/Cake.Common/Build/GitHubActions/Data/GitHubActionsRuntimeInfo.cs b/src/Cake.Common/Build/GitHubActions/Data/GitHubActionsRuntimeInfo.cs index 238a07dd09..7c9ec5e7a0 100644 --- a/src/Cake.Common/Build/GitHubActions/Data/GitHubActionsRuntimeInfo.cs +++ b/src/Cake.Common/Build/GitHubActions/Data/GitHubActionsRuntimeInfo.cs @@ -28,7 +28,7 @@ public GitHubActionsRuntimeInfo(ICakeEnvironment environment) /// true if the GitHub Actions Runtime is available for the current build. /// public bool IsRuntimeAvailable - => !string.IsNullOrWhiteSpace(Token) && !string.IsNullOrWhiteSpace(Url); + => !string.IsNullOrWhiteSpace(Token) && !string.IsNullOrWhiteSpace(Url) && !string.IsNullOrWhiteSpace(ResultsUrl); /// /// Gets the current runtime API authorization token. @@ -46,6 +46,11 @@ public bool IsRuntimeAvailable /// public string Url => GetEnvironmentString("ACTIONS_RUNTIME_URL"); + /// + /// Gets the current runtime API endpoint url for the job. + /// + public string ResultsUrl => GetEnvironmentString("ACTIONS_RESULTS_URL"); + /// /// Gets the path to environment file to set an environment variable that the following steps in a job can use. /// diff --git a/src/Cake.Common/Cake.Common.csproj b/src/Cake.Common/Cake.Common.csproj index b83e2afa8c..b8746a591c 100644 --- a/src/Cake.Common/Cake.Common.csproj +++ b/src/Cake.Common/Cake.Common.csproj @@ -14,5 +14,6 @@ + \ No newline at end of file diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 66f40aec38..036812b69f 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -15,6 +15,7 @@ + From 7c90a29e9cb0b548c7cd52c2d11ecab21daf732f Mon Sep 17 00:00:00 2001 From: Mattias Karlsson Date: Fri, 4 Oct 2024 12:48:47 +0200 Subject: [PATCH 2/2] Address PR feedback --- .../Build/GitHubActions/Commands/GitHubActionsCommands.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Cake.Common/Build/GitHubActions/Commands/GitHubActionsCommands.cs b/src/Cake.Common/Build/GitHubActions/Commands/GitHubActionsCommands.cs index 8d4a141bb8..ac2c5700de 100644 --- a/src/Cake.Common/Build/GitHubActions/Commands/GitHubActionsCommands.cs +++ b/src/Cake.Common/Build/GitHubActions/Commands/GitHubActionsCommands.cs @@ -45,6 +45,8 @@ public GitHubActionsCommands( _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); _writer = writer ?? throw new ArgumentNullException(nameof(writer)); _actionsEnvironment = actionsEnvironment ?? throw new ArgumentNullException(nameof(actionsEnvironment)); + // Internal service class, keeping public API unchanged, + // introduced in pr https://github.com/cake-build/cake/pull/4350 _artifactsService = new GitHubActionsArtifactService(environment, fileSystem, actionsEnvironment, createHttpClient ?? throw new ArgumentNullException(nameof(createHttpClient))); }