Skip to content

Commit 90aec41

Browse files
grendellogrendello
authored
[xaprepare] cache .NET install artifacts (#7026)
Context: dotnet/install-scripts#15 Context: https://dot.net/v1/dotnet-install.sh Context: https://dot.net/v1/dotnet-install.ps1 We've been installing dotnet versions using the [`dotnet-install`][0] scripts for Unix & Windows. However, they do not cache the downloaded archive, and therefore we end up re-downloading the same archive over and over again. Additionally, if one finds themselves without an internet connection, there's no way to easily install the required version of dotnet. The installation scripts don't provide a way to cache the payloads and they appear to be in maintenance mode (dotnet/install-scripts#15), so there doesn't appear to be a chance to add caching support to them. Fortunately, we can "ask" the scripts what they're downloading: % curl -o dotnet-install.sh 'https://dot.net/v1/dotnet-install.sh' % ./dotnet-install.sh --version 7.0.100-preview.5.22273.1 --verbose --dry-run \ | grep 'dotnet-install: URL' This returns a list of URLs, which may or may not exist: dotnet-install: URL #0 - primary: https://dotnetcli.azureedge.net/dotnet/Sdk/7.0.100-preview.5.22273.1/dotnet-sdk-7.0.100-preview.5.22273.1-osx-x64.tar.gz dotnet-install: URL #1 - legacy: https://dotnetcli.azureedge.net/dotnet/Sdk/7.0.100-preview.5.22273.1/dotnet-dev-osx-x64.7.0.100-preview.5.22273.1.tar.gz dotnet-install: URL #2 - primary: https://dotnetbuilds.azureedge.net/public/Sdk/7.0.100-preview.5.22273.1/dotnet-sdk-7.0.100-preview.5.22273.1-osx-x64.tar.gz dotnet-install: URL #3 - legacy: https://dotnetbuilds.azureedge.net/public/Sdk/7.0.100-preview.5.22273.1/dotnet-dev-osx-x64.7.0.100-preview.5.22273.1.tar.gz We now parse this output, extract the URLs, then download and cache the URL contents into `$(AndroidToolchainCacheDirectory)`. When we need to install .NET, we just extract the cached archive into the appropriate directory. If no `dotnet-install: URL…` messages are generated, then we run the `dotnet-install` script as we previously did. This process lets us take a first step towards fully "offline" builds, along with smaller downloads on CI servers. [0]: https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script
1 parent 45997f2 commit 90aec41

File tree

1 file changed

+158
-40
lines changed

1 file changed

+158
-40
lines changed

‎build-tools/xaprepare/xaprepare/Steps/Step_InstallDotNetPreview.cs

+158-40
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,6 @@ protected override async Task<bool> Execute (Context context)
2222
dotnetPath = dotnetPath.TrimEnd (new char [] { Path.DirectorySeparatorChar });
2323
var dotnetTool = Path.Combine (dotnetPath, "dotnet");
2424

25-
// Always delete the bin/$(Configuration)/dotnet/ directory
26-
Utilities.DeleteDirectory (dotnetPath);
27-
2825
if (!await InstallDotNetAsync (context, dotnetPath, BuildToolVersion)) {
2926
Log.ErrorLine ($"Installation of dotnet SDK {BuildToolVersion} failed.");
3027
return false;
@@ -65,68 +62,189 @@ protected override async Task<bool> Execute (Context context)
6562
return true;
6663
}
6764

68-
async Task<bool> InstallDotNetAsync (Context context, string dotnetPath, string version, bool runtimeOnly = false)
65+
async Task<bool> DownloadDotNetInstallScript (Context context, string dotnetScriptPath, Uri dotnetScriptUrl)
6966
{
70-
if (Directory.Exists (Path.Combine (dotnetPath, "sdk", version)) && !runtimeOnly) {
71-
Log.Status ($"dotnet SDK version ");
72-
Log.Status (version, ConsoleColor.Yellow);
73-
Log.StatusLine (" already installed in: ", Path.Combine (dotnetPath, "sdk", version), tailColor: ConsoleColor.Cyan);
74-
return true;
67+
string tempDotnetScriptPath = dotnetScriptPath + "-tmp";
68+
Utilities.DeleteFile (tempDotnetScriptPath);
69+
70+
Log.StatusLine ("Downloading dotnet-install...");
71+
72+
(bool success, ulong size, HttpStatusCode status) = await Utilities.GetDownloadSizeWithStatus (dotnetScriptUrl);
73+
if (!success) {
74+
string message;
75+
if (status == HttpStatusCode.NotFound) {
76+
message = "dotnet-install URL not found";
77+
} else {
78+
message = $"Failed to obtain dotnet-install size. HTTP status code: {status} ({(int)status})";
79+
}
80+
81+
return ReportAndCheckCached (message, quietOnError: true);
7582
}
7683

77-
if (Directory.Exists (Path.Combine (dotnetPath, "shared", "Microsoft.NETCore.App", version)) && runtimeOnly) {
78-
Log.Status ($"dotnet runtime version ");
79-
Log.Status (version, ConsoleColor.Yellow);
80-
Log.StatusLine (" already installed in: ", Path.Combine (dotnetPath, "shared", "Microsoft.NETCore.App", version), tailColor: ConsoleColor.Cyan);
81-
return true;
84+
DownloadStatus downloadStatus = Utilities.SetupDownloadStatus (context, size, context.InteractiveSession);
85+
Log.StatusLine ($" {context.Characters.Link} {dotnetScriptUrl}", ConsoleColor.White);
86+
await Download (context, dotnetScriptUrl, tempDotnetScriptPath, "dotnet-install", Path.GetFileName (dotnetScriptUrl.LocalPath), downloadStatus);
87+
88+
if (!File.Exists (tempDotnetScriptPath)) {
89+
return ReportAndCheckCached ($"Download of dotnet-install from {dotnetScriptUrl} failed");
8290
}
8391

84-
Uri dotnetScriptUrl = Configurables.Urls.DotNetInstallScript;
85-
string dotnetScriptPath = Path.Combine (dotnetPath, Path.GetFileName (dotnetScriptUrl.LocalPath));
86-
if (File.Exists (dotnetScriptPath))
87-
Utilities.DeleteFile (dotnetScriptPath);
92+
Utilities.CopyFile (tempDotnetScriptPath, dotnetScriptPath);
93+
Utilities.DeleteFile (tempDotnetScriptPath);
94+
return true;
8895

89-
Log.StatusLine ("Downloading dotnet-install...");
96+
bool ReportAndCheckCached (string message, bool quietOnError = false)
97+
{
98+
if (File.Exists (dotnetScriptPath)) {
99+
Log.WarningLine (message);
100+
Log.WarningLine ($"Using cached installation script found in {dotnetScriptPath}");
101+
return true;
102+
}
90103

91-
(bool success, ulong size, HttpStatusCode status) = await Utilities.GetDownloadSizeWithStatus (dotnetScriptUrl);
104+
if (!quietOnError) {
105+
Log.ErrorLine (message);
106+
Log.ErrorLine ($"Cached installation script not found in {dotnetScriptPath}");
107+
}
108+
return false;
109+
}
110+
}
111+
112+
async Task<bool> DownloadDotNetArchive (Context context, string archiveDestinationPath, Uri archiveUrl)
113+
{
114+
Log.StatusLine ("Downloading dotnet archive...");
115+
116+
(bool success, ulong size, HttpStatusCode status) = await Utilities.GetDownloadSizeWithStatus (archiveUrl);
92117
if (!success) {
93-
if (status == HttpStatusCode.NotFound)
94-
Log.ErrorLine ("dotnet-install URL not found");
95-
else
96-
Log.ErrorLine ("Failed to obtain dotnet-install size. HTTP status code: {status} ({(int)status})");
118+
if (status == HttpStatusCode.NotFound) {
119+
Log.ErrorLine ($"dotnet archive URL {archiveUrl} not found");
120+
return false;
121+
} else {
122+
Log.WarningLine ($"Failed to obtain dotnet archive size. HTTP status code: {status} ({(int)status})");
123+
}
124+
97125
return false;
98126
}
99127

128+
string tempArchiveDestinationPath = archiveDestinationPath + "-tmp";
129+
Utilities.DeleteFile (tempArchiveDestinationPath);
130+
100131
DownloadStatus downloadStatus = Utilities.SetupDownloadStatus (context, size, context.InteractiveSession);
101-
Log.StatusLine ($" {context.Characters.Link} {dotnetScriptUrl}", ConsoleColor.White);
102-
await Download (context, dotnetScriptUrl, dotnetScriptPath, "dotnet-install", Path.GetFileName (dotnetScriptUrl.LocalPath), downloadStatus);
132+
Log.StatusLine ($" {context.Characters.Link} {archiveUrl}", ConsoleColor.White);
133+
await Download (context, archiveUrl, tempArchiveDestinationPath, "dotnet archive", Path.GetFileName (archiveUrl.LocalPath), downloadStatus);
103134

104-
if (!File.Exists (dotnetScriptPath)) {
105-
Log.ErrorLine ($"Download of dotnet-install from {dotnetScriptUrl} failed");
135+
if (!File.Exists (tempArchiveDestinationPath)) {
106136
return false;
107137
}
108138

109-
var type = runtimeOnly ? "runtime" : "SDK";
110-
Log.StatusLine ($"Installing dotnet {type} '{version}'...");
139+
Utilities.CopyFile (tempArchiveDestinationPath, archiveDestinationPath);
140+
Utilities.DeleteFile (tempArchiveDestinationPath);
141+
142+
return true;
143+
}
111144

145+
string[] GetInstallationScriptArgs (string version, string dotnetPath, string dotnetScriptPath, bool onlyGetUrls, bool runtimeOnly)
146+
{
147+
List<string> args;
112148
if (Context.IsWindows) {
113-
var args = new List<string> {
149+
args = new List<string> {
114150
"-NoProfile", "-ExecutionPolicy", "unrestricted", "-file", dotnetScriptPath,
115151
"-Version", version, "-InstallDir", dotnetPath, "-Verbose"
116152
};
117-
if (runtimeOnly)
153+
if (runtimeOnly) {
118154
args.AddRange (new string [] { "-Runtime", "dotnet" });
155+
}
156+
if (onlyGetUrls) {
157+
args.Add ("-DryRun");
158+
}
119159

120-
return Utilities.RunCommand ("powershell.exe", args.ToArray ());
121-
} else {
122-
var args = new List<string> {
123-
dotnetScriptPath, "--version", version, "--install-dir", dotnetPath, "--verbose"
124-
};
125-
if (runtimeOnly)
126-
args.AddRange (new string [] { "-Runtime", "dotnet" });
160+
return args.ToArray ();
161+
}
162+
163+
args = new List<string> {
164+
dotnetScriptPath, "--version", version, "--install-dir", dotnetPath, "--verbose"
165+
};
166+
167+
if (runtimeOnly) {
168+
args.AddRange (new string [] { "-Runtime", "dotnet" });
169+
}
170+
if (onlyGetUrls) {
171+
args.Add ("--dry-run");
172+
}
173+
174+
return args.ToArray ();
175+
}
176+
177+
async Task<bool> InstallDotNetAsync (Context context, string dotnetPath, string version, bool runtimeOnly = false)
178+
{
179+
string cacheDir = context.Properties.GetRequiredValue (KnownProperties.AndroidToolchainCacheDirectory);
127180

128-
return Utilities.RunCommand ("bash", args.ToArray ());
181+
// Always delete the bin/$(Configuration)/dotnet/ directory
182+
Utilities.DeleteDirectory (dotnetPath);
183+
184+
Uri dotnetScriptUrl = Configurables.Urls.DotNetInstallScript;
185+
string scriptFileName = Path.GetFileName (dotnetScriptUrl.LocalPath);
186+
string cachedDotnetScriptPath = Path.Combine (cacheDir, scriptFileName);
187+
if (!await DownloadDotNetInstallScript (context, cachedDotnetScriptPath, dotnetScriptUrl)) {
188+
return false;
129189
}
190+
191+
string dotnetScriptPath = Path.Combine (dotnetPath, scriptFileName);
192+
Utilities.CopyFile (cachedDotnetScriptPath, dotnetScriptPath);
193+
194+
var type = runtimeOnly ? "runtime" : "SDK";
195+
196+
Log.StatusLine ($"Discovering download URLs for dotnet {type} '{version}'...");
197+
string scriptCommand = Context.IsWindows ? "powershell.exe" : "bash";
198+
string[] scriptArgs = GetInstallationScriptArgs (version, dotnetPath, dotnetScriptPath, onlyGetUrls: true, runtimeOnly: runtimeOnly);
199+
string scriptReply = Utilities.GetStringFromStdout (scriptCommand, scriptArgs);
200+
var archiveUrls = new List<string> ();
201+
202+
char[] fieldSplitChars = new char[] { ':' };
203+
foreach (string l in scriptReply.Split (new char[] { '\n' })) {
204+
string line = l.Trim ();
205+
206+
if (!line.StartsWith ("dotnet-install: URL #", StringComparison.OrdinalIgnoreCase)) {
207+
continue;
208+
}
209+
210+
string[] parts = line.Split (fieldSplitChars, 3);
211+
if (parts.Length < 3) {
212+
Log.WarningLine ($"dotnet-install URL line has unexpected number of parts. Expected 3, got {parts.Length}");
213+
Log.WarningLine ($"Line: {line}");
214+
continue;
215+
}
216+
217+
archiveUrls.Add (parts[2].Trim ());
218+
}
219+
220+
if (archiveUrls.Count == 0) {
221+
Log.WarningLine ("No dotnet archive URLs discovered, attempting to run the installation script");
222+
scriptArgs = GetInstallationScriptArgs (version, dotnetPath, dotnetScriptPath, onlyGetUrls: false, runtimeOnly: runtimeOnly);
223+
return Utilities.RunCommand (scriptCommand, scriptArgs);
224+
}
225+
226+
string? archivePath = null;
227+
foreach (string url in archiveUrls) {
228+
var archiveUrl = new Uri (url);
229+
string archiveDestinationPath = Path.Combine (cacheDir, Path.GetFileName (archiveUrl.LocalPath));
230+
231+
if (File.Exists (archiveDestinationPath)) {
232+
archivePath = archiveDestinationPath;
233+
break;
234+
}
235+
236+
if (await DownloadDotNetArchive (context, archiveDestinationPath, archiveUrl)) {
237+
archivePath = archiveDestinationPath;
238+
break;
239+
}
240+
}
241+
242+
if (String.IsNullOrEmpty (archivePath)) {
243+
return false;
244+
}
245+
246+
Log.StatusLine ($"Installing dotnet {type} '{version}'...");
247+
return await Utilities.Unpack (archivePath, dotnetPath);
130248
}
131249

132250
bool TestDotNetSdk (string dotnetTool)

0 commit comments

Comments
 (0)