From 06ae7cabd42154f58ce9eae88a2da1a832468932 Mon Sep 17 00:00:00 2001 From: luiz Date: Sun, 1 Dec 2024 11:21:11 -0300 Subject: [PATCH 1/5] simple http client for downloading audio files from a CDN --- Robust.Client/HTTPClient/HTTPClient.cs | 147 +++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 Robust.Client/HTTPClient/HTTPClient.cs diff --git a/Robust.Client/HTTPClient/HTTPClient.cs b/Robust.Client/HTTPClient/HTTPClient.cs new file mode 100644 index 00000000000..70216c6acb3 --- /dev/null +++ b/Robust.Client/HTTPClient/HTTPClient.cs @@ -0,0 +1,147 @@ +using System; +using System.Json; +using System.Net.Http; +using System.Threading.Tasks; +using Robust.Shared.Configuration; +using Robust.Shared.IoC; + + + +namespace Robust.Client.HTTPClient +{ + /// + /// Interface for a Read-only client that can download files from a CDN. + /// This will be used to download and stream audio files from a whitelisted endpoint. + /// CDN must tell client what files are available to download by using a JSON file. + /// + public interface ICDNConsumer + { + Task GetFileAsync(string url, string outputPath); + } + + public class CDNConsumer : ICDNConsumer + { + [Dependency] private readonly IConfigurationManager _cfg = default!; + // The are so many error loggers, honestly don't know which one to use + [Dependency] private readonly IRuntimeLog _runtimeLog = default!; + + private readonly HttpClient _httpClient; + // move whiteListed domains to a configuration file and access it through IConfigurationManager + private readonly string[] _whitelistedDomains = { "example.com", "anotherexample.com" }; + + // this should also be in a configuration file + private readonly string _manifestFilename = "manifest.json"; + + + public CDNConsumer() + { + _httpClient = new HttpClient + { + Timeout = TimeSpan.FromSeconds(30), // timeouts prevent hanging requests. + MaxResponseContentBufferSize = 1_000_000 // Limit the response size (1MB here) + }; + } + + /// + /// Downloads a manifest file from a CDN. + /// The manifest file is a JSON file that contains a list of files that are available to download. + /// The manifest file will be cached to the local filesystem. (im not sure if this is a good idea) + /// + public async Task GetManifestAsync(string url) + { + // Check if the URL is valid and whitelisted + if (!IsValidUrl(url)) + { + _runtimeLog.LogException(e, "CDNConsumer: Invalid URL"); + return null; + } + try + { + var response = await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead); + response.EnsureSuccessStatusCode(); + } + + // Validate the response + // here we are checking if the response is a JSON file + var mediaType = response.Content.Headers.ContentType.MediaType; + if (!mediaType.Equals("application/json", StringComparison.OrdinalIgnoreCase)) + { + _runtimeLog.LogException(e, $"Invalid media type {nameof(mediaType)}"); + } + + using (var contentStream = await response.Content.ReadAsStreamAsync()) + using (var fileStream = new FileStream(_manifestFilename, FileMode.Create, FileAccess.Write, FileShare.None, 8192, true)) + { + await contentStream.CopyToAsync(fileStream); + } + + } + public async Task GetFileAsync(string url, string outputPath) + { + // Check if the URL is valid and whitelisted + if (!IsValidUrl(url)) + { + _runtimeLog.LogException(e, "CDNConsumer: Invalid URL"); + return null; + } + + try + { + var response = await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead); + response.EnsureSuccessStatusCode(); + } + catch (HttpRequestException e) + { + _runtimeLog.LogException(e, "CDNConsumer: Failed to download file"); + return null; + } + + // Validate the response + // here we are checking if the response is an audio file + // ideally this should verify integrity of the file using a hash + var mediaType = response.Content.Headers.ContentType.MediaType; + var fileExtension = Path.GetExtension(outputPath); + + + // this shouldn't be hardcoded, whitelisting file types should be in a configuration file + if (!mediaType.Equals("audio/ogg", StringComparison.OrdinalIgnoreCase)) + { + _runtimeLog.LogException(e, $"CDNConsumer: Invalid media type {nameof(mediaType)}"); + } + + using (var contentStream = await response.Content.ReadAsStreamAsync()) + using (var fileStream = new FileStream(outputPath, FileMode.Create, FileAccess.Write, FileShare.None, 8192, true)) + { + await contentStream.CopyToAsync(fileStream); + } + + } + + // Check if the URL is valid and whitelisted + private bool IsValidUrl(string url) + { + // Check if URL is valid + if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) + { + _runtimeLog.LogException(e, "CDNConsumer: Invalid URL"); + return false; + } + // Check if domain is whitelisted + var domain = uri.Host; + if (!Array.Exists(_whitelistedDomains, d => d.Equals(domain, StringComparison.OrdinalIgnoreCase))) + { + _runtimeLog.LogException(e, "CDNConsumer: Domain is not whitelisted"); + return false; + } + + // Ensure URL uses HTTPS + if (!url.StartsWith("https://")) + { + _runtimeLog.LogException(e, "CDNConsumer: URL must use HTTPS"); + return false; + } + + return true; + } + } +} From 3b5ae8c5c7d0149c0b0cf1bb6ec8c63f68b36031 Mon Sep 17 00:00:00 2001 From: luiz Date: Sun, 1 Dec 2024 11:33:35 -0300 Subject: [PATCH 2/5] added todo comments --- Robust.Client/HTTPClient/HTTPClient.cs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/Robust.Client/HTTPClient/HTTPClient.cs b/Robust.Client/HTTPClient/HTTPClient.cs index 70216c6acb3..40002a243a6 100644 --- a/Robust.Client/HTTPClient/HTTPClient.cs +++ b/Robust.Client/HTTPClient/HTTPClient.cs @@ -26,11 +26,11 @@ public class CDNConsumer : ICDNConsumer [Dependency] private readonly IRuntimeLog _runtimeLog = default!; private readonly HttpClient _httpClient; - // move whiteListed domains to a configuration file and access it through IConfigurationManager + // move whitelisted domains to a configuration file and access it through IConfigurationManager private readonly string[] _whitelistedDomains = { "example.com", "anotherexample.com" }; // this should also be in a configuration file - private readonly string _manifestFilename = "manifest.json"; + private readonly string _manifestFilename = "cdn_manifest.json"; public CDNConsumer() @@ -69,12 +69,9 @@ public async Task GetManifestAsync(string url) _runtimeLog.LogException(e, $"Invalid media type {nameof(mediaType)}"); } - using (var contentStream = await response.Content.ReadAsStreamAsync()) - using (var fileStream = new FileStream(_manifestFilename, FileMode.Create, FileAccess.Write, FileShare.None, 8192, true)) - { - await contentStream.CopyToAsync(fileStream); - } + var manifest = await response.Content.ReadAsStringAsync(); + //TODO: need to parse the JSON file and make it available to the ContentAudioSystem somehow } public async Task GetFileAsync(string url, string outputPath) { @@ -98,12 +95,12 @@ public async Task GetFileAsync(string url, string outputPath) // Validate the response // here we are checking if the response is an audio file - // ideally this should verify integrity of the file using a hash + // TODO: ideally this should verify integrity of the file using a hash var mediaType = response.Content.Headers.ContentType.MediaType; var fileExtension = Path.GetExtension(outputPath); - // this shouldn't be hardcoded, whitelisting file types should be in a configuration file + // TODO: this shouldn't be hardcoded, whitelisting file types should be in a configuration file if (!mediaType.Equals("audio/ogg", StringComparison.OrdinalIgnoreCase)) { _runtimeLog.LogException(e, $"CDNConsumer: Invalid media type {nameof(mediaType)}"); From d4694fb8757e118f22f525adb891d0f8e6d5a991 Mon Sep 17 00:00:00 2001 From: luiz Date: Wed, 4 Dec 2024 04:45:05 -0300 Subject: [PATCH 3/5] getfileasync downloads files to from cdn to disk --- Robust.Client/ClientIoC.cs | 3 + .../GameController/GameController.cs | 4 + .../HTTPClient/Commands/HTTPGetFileAsync.cs | 24 ++ Robust.Client/HTTPClient/HTTPClient.cs | 146 +++++++--- Robust.Client/HTTPClient/JSON.cs | 260 ++++++++++++++++++ 5 files changed, 394 insertions(+), 43 deletions(-) create mode 100644 Robust.Client/HTTPClient/Commands/HTTPGetFileAsync.cs create mode 100644 Robust.Client/HTTPClient/JSON.cs diff --git a/Robust.Client/ClientIoC.cs b/Robust.Client/ClientIoC.cs index e197f433fb7..00096b6efd0 100644 --- a/Robust.Client/ClientIoC.cs +++ b/Robust.Client/ClientIoC.cs @@ -8,6 +8,7 @@ using Robust.Client.GameStates; using Robust.Client.Graphics; using Robust.Client.Graphics.Clyde; +using Robust.Client.HTTPClient; using Robust.Client.HWId; using Robust.Client.Input; using Robust.Client.Map; @@ -102,6 +103,8 @@ public static void RegisterIoC(GameController.DisplayMode mode, IDependencyColle deps.Register(); deps.Register(); deps.Register(); + deps.Register(); + deps.Register(); switch (mode) { diff --git a/Robust.Client/GameController/GameController.cs b/Robust.Client/GameController/GameController.cs index 4116fb52c93..31c4c392f4a 100644 --- a/Robust.Client/GameController/GameController.cs +++ b/Robust.Client/GameController/GameController.cs @@ -10,6 +10,7 @@ using Robust.Client.GameObjects; using Robust.Client.GameStates; using Robust.Client.Graphics; +using Robust.Client.HTTPClient; using Robust.Client.Input; using Robust.Client.Placement; using Robust.Client.Replays.Loading; @@ -94,6 +95,8 @@ internal sealed partial class GameController : IGameControllerInternal [Dependency] private readonly IReplayRecordingManagerInternal _replayRecording = default!; [Dependency] private readonly IReflectionManager _reflectionManager = default!; + [Dependency] private readonly ICDNConsumer _cdnConsumer = default!; + private IWebViewManagerHook? _webViewHook; private CommandLineArgs? _commandLineArgs; @@ -206,6 +209,7 @@ internal bool StartupContinue(DisplayMode displayMode) _replayLoader.Initialize(); _replayPlayback.Initialize(); _replayRecording.Initialize(); + _cdnConsumer.Initialize(); _userInterfaceManager.PostInitialize(); _modLoader.BroadcastRunLevel(ModRunLevel.PostInit); diff --git a/Robust.Client/HTTPClient/Commands/HTTPGetFileAsync.cs b/Robust.Client/HTTPClient/Commands/HTTPGetFileAsync.cs new file mode 100644 index 00000000000..2279d422e49 --- /dev/null +++ b/Robust.Client/HTTPClient/Commands/HTTPGetFileAsync.cs @@ -0,0 +1,24 @@ +using Robust.Shared.Console; +using Robust.Shared.IoC; + +namespace Robust.Client.HTTPClient.Commands +{ + public sealed class HTTPGetFileAsync : LocalizedCommands + { + [Dependency] private readonly ICDNConsumer _consumer = default!; + + public override string Command => "getfileasync"; + + public override void Execute(IConsoleShell shell, string argStr, string[] args) + { + if (args.Length < 1) + { + shell.WriteLine("Usage: getfileasync "); + return; + } + + var url = args[0]; + _consumer.GetFileAsync(url); + } + } +} diff --git a/Robust.Client/HTTPClient/HTTPClient.cs b/Robust.Client/HTTPClient/HTTPClient.cs index 40002a243a6..84bdcf1f355 100644 --- a/Robust.Client/HTTPClient/HTTPClient.cs +++ b/Robust.Client/HTTPClient/HTTPClient.cs @@ -1,9 +1,12 @@ using System; -using System.Json; +using System.IO; using System.Net.Http; +using System.Net.Http.Headers; using System.Threading.Tasks; -using Robust.Shared.Configuration; using Robust.Shared.IoC; +using Robust.Shared.Configuration; +using Robust.Shared.Exceptions; +using Robust.Shared.Log; @@ -16,7 +19,9 @@ namespace Robust.Client.HTTPClient /// public interface ICDNConsumer { - Task GetFileAsync(string url, string outputPath); + + void Initialize(); + Task GetFileAsync(string url); } public class CDNConsumer : ICDNConsumer @@ -25,9 +30,16 @@ public class CDNConsumer : ICDNConsumer // The are so many error loggers, honestly don't know which one to use [Dependency] private readonly IRuntimeLog _runtimeLog = default!; + + [Dependency] private readonly IJSON _json = default!; + private ISawmill _sawmill = default!; + private readonly HttpClient _httpClient; // move whitelisted domains to a configuration file and access it through IConfigurationManager - private readonly string[] _whitelistedDomains = { "example.com", "anotherexample.com" }; + private readonly string[] _whitelistedDomains = { "ia902304.us.archive.org", "example.com", "anotherexample.com" }; + + //in windows this should be appdata of the game, should be in a configuration file + private readonly string _downloadDirectory = "%APPDATA%\\Space Station 14\\data\\Cache"; // this should also be in a configuration file // this should also be in a configuration file private readonly string _manifestFilename = "cdn_manifest.json"; @@ -38,8 +50,14 @@ public CDNConsumer() _httpClient = new HttpClient { Timeout = TimeSpan.FromSeconds(30), // timeouts prevent hanging requests. - MaxResponseContentBufferSize = 1_000_000 // Limit the response size (1MB here) + MaxResponseContentBufferSize = 5_000_000 // Limit the response size (5MB here) }; + + } + + public void Initialize() + { + _sawmill = Logger.GetSawmill("cdn"); } /// @@ -52,66 +70,108 @@ public async Task GetManifestAsync(string url) // Check if the URL is valid and whitelisted if (!IsValidUrl(url)) { - _runtimeLog.LogException(e, "CDNConsumer: Invalid URL"); - return null; + return "Invalid URL"; } try { var response = await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead); response.EnsureSuccessStatusCode(); + // Validate the response + // here we are checking if the response is a JSON file + var mediaType = response.Content.Headers.ContentType?.MediaType ?? ""; + if (!mediaType.Equals("application/json", StringComparison.OrdinalIgnoreCase)) + { + // _sawmill.Error($"Invalid media type {nameof(mediaType)}"); + return "Invalid media type"; + } + var manifest = await response.Content.ReadAsStringAsync(); + + var parsedManifest = _json.Parse(manifest); + if (parsedManifest == null) + { + // _sawmill.Error("CDNConsumer: Failed to parse manifest"); + return "Failed to parse manifest"; + } } - - // Validate the response - // here we are checking if the response is a JSON file - var mediaType = response.Content.Headers.ContentType.MediaType; - if (!mediaType.Equals("application/json", StringComparison.OrdinalIgnoreCase)) + catch (HttpRequestException e) { - _runtimeLog.LogException(e, $"Invalid media type {nameof(mediaType)}"); + _runtimeLog.LogException(e, "CDNConsumer: Failed to download file manifest"); + return "Failed to download file manifest"; } - var manifest = await response.Content.ReadAsStringAsync(); - - //TODO: need to parse the JSON file and make it available to the ContentAudioSystem somehow + return "Manifest downloaded"; } - public async Task GetFileAsync(string url, string outputPath) + public async Task GetFileAsync(string url) { + + string filename = Path.GetFileName(url); + string outputPath = Path.Combine(_downloadDirectory, filename); + + if (!Directory.Exists(_downloadDirectory)) + { + Directory.CreateDirectory(_downloadDirectory); + } + + // expand %appdata% to the actual path + string fullOutputPath = Environment.ExpandEnvironmentVariables(outputPath); + + // Check if the file already exists + if (File.Exists(fullOutputPath)) + { + _sawmill.Info($"File already exists at {outputPath}"); + return outputPath; + } + // Check if the URL is valid and whitelisted if (!IsValidUrl(url)) { - _runtimeLog.LogException(e, "CDNConsumer: Invalid URL"); - return null; + return "Invalid URL"; } - try { var response = await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead); response.EnsureSuccessStatusCode(); - } - catch (HttpRequestException e) - { - _runtimeLog.LogException(e, "CDNConsumer: Failed to download file"); - return null; - } + // Validate the response + // here we are checking if the response is an audio file + // TODO: ideally this should verify integrity of the file using a hash + var mediaType = response.Content.Headers.ContentType?.MediaType ?? ""; + var fileExtension = Path.GetExtension(url); - // Validate the response - // here we are checking if the response is an audio file - // TODO: ideally this should verify integrity of the file using a hash - var mediaType = response.Content.Headers.ContentType.MediaType; - var fileExtension = Path.GetExtension(outputPath); + // TODO: this shouldn't be hardcoded, whitelisting file types should be in a configuration file + if (!mediaType.Equals("application/ogg", StringComparison.OrdinalIgnoreCase)) + { + // _sawmill.Error($"CDNConsumer: Invalid media type {nameof(mediaType)}"); + return "Invalid media type"; + } - // TODO: this shouldn't be hardcoded, whitelisting file types should be in a configuration file - if (!mediaType.Equals("audio/ogg", StringComparison.OrdinalIgnoreCase)) - { - _runtimeLog.LogException(e, $"CDNConsumer: Invalid media type {nameof(mediaType)}"); - } - using (var contentStream = await response.Content.ReadAsStreamAsync()) - using (var fileStream = new FileStream(outputPath, FileMode.Create, FileAccess.Write, FileShare.None, 8192, true)) + + + using (var contentStream = await response.Content.ReadAsStreamAsync()) + using (var fileStream = new FileStream(fullOutputPath, FileMode.Create, FileAccess.Write, FileShare.None, 8192, true)) + { + try + { + + await contentStream.CopyToAsync(fileStream); + + } + catch (Exception e) + { + _sawmill.Error($"Failed to save file {e}"); + return "Failed to save file"; + } + } + + _sawmill.Info($"File downloaded to {outputPath}"); + return outputPath; + } + catch (HttpRequestException e) { - await contentStream.CopyToAsync(fileStream); + _runtimeLog.LogException(e, "CDNConsumer: Failed to download file"); + return "Failed to download file"; } - } // Check if the URL is valid and whitelisted @@ -120,21 +180,21 @@ private bool IsValidUrl(string url) // Check if URL is valid if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) { - _runtimeLog.LogException(e, "CDNConsumer: Invalid URL"); + // _sawmill.Error("CDNConsumer: Invalid URL"); return false; } // Check if domain is whitelisted var domain = uri.Host; if (!Array.Exists(_whitelistedDomains, d => d.Equals(domain, StringComparison.OrdinalIgnoreCase))) { - _runtimeLog.LogException(e, "CDNConsumer: Domain is not whitelisted"); + // _sawmill.Error("CDNConsumer: Domain is not whitelisted"); return false; } // Ensure URL uses HTTPS if (!url.StartsWith("https://")) { - _runtimeLog.LogException(e, "CDNConsumer: URL must use HTTPS"); + // _sawmill.Error("CDNConsumer: URL must use HTTPS"); return false; } diff --git a/Robust.Client/HTTPClient/JSON.cs b/Robust.Client/HTTPClient/JSON.cs new file mode 100644 index 00000000000..237d05532b4 --- /dev/null +++ b/Robust.Client/HTTPClient/JSON.cs @@ -0,0 +1,260 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Text; + + +namespace Robust.Client.HTTPClient +{ + public interface IJSON + { + string Serialize(object obj); + + object? Parse(string json); + + } + public class JSON : IJSON + { + private int index = 0; + + private string? json; + + public object? Parse(string json) + { + this.json = json; + + object? ParseValue() + { + SkipWhitespace(); + switch (this.json![this.index]) + { + case '{': + return ParseObject(); + case '[': + return ParseArray(); + case '"': + return ParseString(); + case 't': + return ParseTrue(); + case 'f': + return ParseFalse(); + case 'n': + return ParseNull(); + default: + return ParseNumber(); + } + } + + void SkipWhitespace() + { + while (char.IsWhiteSpace(this.json![this.index])) + { + this.index++; + } + } + + object ParseObject() + { + var obj = new Dictionary(); + this.index++; + while (this.json![this.index] != '}') + { + SkipWhitespace(); + var key = ParseString(); + SkipWhitespace(); + if (this.json[this.index] != ':') + { + throw new Exception("Expected ':'"); + } + this.index++; + SkipWhitespace(); + var value = ParseValue(); + obj[key] = value; + SkipWhitespace(); + if (this.json[this.index] == ',') + { + this.index++; + } + } + this.index++; + return obj; + } + + object ParseArray() + { + var list = new List(); + this.index++; + while (this.json![this.index] != ']') + { + SkipWhitespace(); + var value = ParseValue(); + list.Add(value); + SkipWhitespace(); + if (this.json[this.index] == ',') + { + this.index++; + } + } + this.index++; + return list; + } + + string ParseString() + { + var sb = new StringBuilder(); + this.index++; + while (this.json![this.index] != '"') + { + if (this.json[this.index] == '\\') + { + this.index++; + switch (this.json[this.index]) + { + case '"': + sb.Append('"'); + break; + case '\\': + sb.Append('\\'); + break; + case '/': + sb.Append('/'); + break; + case 'b': + sb.Append('\b'); + break; + case 'f': + sb.Append('\f'); + break; + case 'n': + sb.Append('\n'); + break; + case 'r': + sb.Append('\r'); + break; + case 't': + sb.Append('\t'); + break; + case 'u': + var hex = this.json.Substring(this.index + 1, 4); + sb.Append((char)Convert.ToInt32(hex, 16)); + this.index += 4; + break; + } + } + else + { + sb.Append(this.json[this.index]); + } + this.index++; + } + this.index++; + return sb.ToString(); + } + + object ParseTrue() + { + if (this.json!.Substring(this.index, 4) == "true") + { + this.index += 4; + return true; + } + throw new Exception("Expected 'true'"); + } + + object ParseFalse() + { + if (this.json!.Substring(this.index, 5) == "false") + { + this.index += 5; + return false; + } + throw new Exception("Expected 'false'"); + } + + object? ParseNull() + { + if (this.json!.Substring(this.index, 4) == "null") + { + this.index += 4; + return null; + } + throw new Exception("Expected 'null'"); + } + + object ParseNumber() + { + var start = this.index; + while (char.IsDigit(this.json![this.index]) || this.json[this.index] == '.' || this.json[this.index] == '-' || this.json[this.index] == 'e' || this.json[this.index] == 'E') + { + this.index++; + } + var str = this.json.Substring(start, this.index - start); + if (int.TryParse(str, out var i)) + { + return i; + } + if (double.TryParse(str, out var d)) + { + return d; + } + throw new Exception("Invalid number"); + } + + return ParseValue(); + } + + public string Serialize(object? obj) + { + if (obj == null) + { + return "null"; + } + + if (obj is string) + { + return $"\"{obj}\""; + + } + + if (obj is bool b) + { + return b.ToString().ToLower() ?? "false"; + } + + if (obj is IDictionary) + { + var dict = (IDictionary)obj; + var items = new List(); + foreach (var key in dict.Keys) + { + items.Add($"\"{key}\": {Serialize(dict[key])}"); + } + return $"{{{string.Join(", ", items)}}}"; + } + + if (obj is IEnumerable) + { + var items = new List(); + foreach (var item in (IEnumerable)obj) + { + items.Add(Serialize(item)); + } + return $"[{string.Join(", ", items)}]"; + } + + if (obj is ValueType v) + { + return v.ToString() ?? "null"; + } + + var properties = obj.GetType().GetProperties(); + var propItems = new List(); + foreach (var prop in properties) + { + var value = prop.GetValue(obj, null); + propItems.Add($"\"{prop.Name}\": {Serialize(value)}"); + } + return $"{{{string.Join(", ", propItems)}}}"; + } + } +} From 126842345e4a7f6d32ce4cb18cc1d63d58dfe9d8 Mon Sep 17 00:00:00 2001 From: luiz Date: Thu, 5 Dec 2024 18:05:00 -0300 Subject: [PATCH 4/5] added a way to play the files for testing purposes --- ...etFileAsync.cs => HTTPPlayAudioFromCDN.cs} | 8 +- Robust.Client/HTTPClient/HTTPClient.cs | 281 ++++++++++++++---- 2 files changed, 231 insertions(+), 58 deletions(-) rename Robust.Client/HTTPClient/Commands/{HTTPGetFileAsync.cs => HTTPPlayAudioFromCDN.cs} (64%) diff --git a/Robust.Client/HTTPClient/Commands/HTTPGetFileAsync.cs b/Robust.Client/HTTPClient/Commands/HTTPPlayAudioFromCDN.cs similarity index 64% rename from Robust.Client/HTTPClient/Commands/HTTPGetFileAsync.cs rename to Robust.Client/HTTPClient/Commands/HTTPPlayAudioFromCDN.cs index 2279d422e49..0f1a95bec0c 100644 --- a/Robust.Client/HTTPClient/Commands/HTTPGetFileAsync.cs +++ b/Robust.Client/HTTPClient/Commands/HTTPPlayAudioFromCDN.cs @@ -3,22 +3,22 @@ namespace Robust.Client.HTTPClient.Commands { - public sealed class HTTPGetFileAsync : LocalizedCommands + public sealed class HTTPPlayAudioFromCDN : LocalizedCommands { [Dependency] private readonly ICDNConsumer _consumer = default!; - public override string Command => "getfileasync"; + public override string Command => "playaudiocdn"; public override void Execute(IConsoleShell shell, string argStr, string[] args) { if (args.Length < 1) { - shell.WriteLine("Usage: getfileasync "); + shell.WriteLine("Usage: playaudiocdn "); return; } var url = args[0]; - _consumer.GetFileAsync(url); + _consumer.PlayAudioFromCDN(url); } } } diff --git a/Robust.Client/HTTPClient/HTTPClient.cs b/Robust.Client/HTTPClient/HTTPClient.cs index 84bdcf1f355..2776635c1b3 100644 --- a/Robust.Client/HTTPClient/HTTPClient.cs +++ b/Robust.Client/HTTPClient/HTTPClient.cs @@ -1,15 +1,21 @@ using System; using System.IO; +using System.Collections.Generic; using System.Net.Http; using System.Net.Http.Headers; using System.Threading.Tasks; +using Robust.Client.Audio; +using Robust.Client.Audio.Sources; +using Robust.Client.Graphics; +using Robust.Shared.Audio; +using Robust.Shared.Audio.Systems; +using Robust.Shared.Audio.Sources; using Robust.Shared.IoC; using Robust.Shared.Configuration; using Robust.Shared.Exceptions; using Robust.Shared.Log; - namespace Robust.Client.HTTPClient { /// @@ -21,17 +27,25 @@ public interface ICDNConsumer { void Initialize(); + + Task PlayAudioFromCDN(string url); + + void StopAudio(); + Task GetFileAsync(string url); + } public class CDNConsumer : ICDNConsumer { [Dependency] private readonly IConfigurationManager _cfg = default!; - // The are so many error loggers, honestly don't know which one to use [Dependency] private readonly IRuntimeLog _runtimeLog = default!; + [Dependency] private readonly IAudioManager _audioMgr = default!; [Dependency] private readonly IJSON _json = default!; + private readonly Dictionary _cachedFiles = new Dictionary(); + private List _activeSources = new List(); private ISawmill _sawmill = default!; private readonly HttpClient _httpClient; @@ -44,6 +58,9 @@ public class CDNConsumer : ICDNConsumer // this should also be in a configuration file private readonly string _manifestFilename = "cdn_manifest.json"; + // TODO: this is a temporary solution, + // public IBufferedAudioSource Source { get; set; } + public CDNConsumer() { @@ -61,11 +78,110 @@ public void Initialize() } /// + /// Plays fetches and plays audio from an URL using GetFileAsync + /// + /// + public async Task PlayAudioFromCDN(string url) + { + string filepath = await GetFileAsync(url); + + if (filepath != "Failed to download file") + { + PlayOggAudioFile(filepath); + return "Audio played"; + } + return "Failed to play audio"; + } + + + + /// + /// Downloads a file from a CDN. + /// The file is cached to the local filesystem inside _downloadDirectory. + /// The file is saved with the same name as the file on the CDN. + /// If the file already exists, it will not be downloaded again. + /// The file is validated to ensure it is an audio file and from a whitelisted domain. + /// + /// + /// + public async Task GetFileAsync(string url) + { + + string filename = Path.GetFileName(url); + // expand %appdata% to the actual path + string fullDownloadDir = Environment.ExpandEnvironmentVariables(_downloadDirectory); + string fullOutputPath = Path.Combine(fullDownloadDir, filename); + + // Create the download directory if it doesn't exist + if (!Directory.Exists(fullDownloadDir)) + { + Directory.CreateDirectory(fullDownloadDir); + } + + // Check if the file already exists. this could be less hacky. + if (File.Exists(fullOutputPath)) + { + _sawmill.Info($"File already exists at {_downloadDirectory}"); + return fullOutputPath; + } + + // Check if the URL is valid and whitelisted + if (!IsValidUrl(url)) + { + return "Invalid URL"; + } + + // Download the file + try + { + var response = await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead); + response.EnsureSuccessStatusCode(); + // Validate the response + // here we are checking if the response is an audio file + // TODO: ideally this should verify integrity of the file using a hash + var mediaType = response.Content.Headers.ContentType?.MediaType ?? ""; + var fileExtension = Path.GetExtension(url); + + + // TODO: this shouldn't be hardcoded, whitelisting file types should be in a configuration file + if (!(mediaType.Equals("application/ogg", StringComparison.OrdinalIgnoreCase) + || mediaType.Equals("audio/ogg", StringComparison.OrdinalIgnoreCase))) + { + _sawmill.Error($"Invalid media type {nameof(mediaType)}"); + return "Invalid media type"; + } + + using (var contentStream = await response.Content.ReadAsStreamAsync()) + using (var fileStream = new FileStream(fullOutputPath, FileMode.Create, FileAccess.Write, FileShare.None, 8192, true)) + { + try + { + await contentStream.CopyToAsync(fileStream); + } + catch (Exception e) + { + _sawmill.Error($"Failed to save file {e}"); + return "Failed to save file"; + } + } + + _sawmill.Info($"File downloaded to {fullOutputPath}"); + return fullOutputPath; + } + catch (HttpRequestException e) + { + _sawmill.Error(e, "Failed to download file"); + return "Failed to download file"; + } + } + + /// + /// This is not implemented yet. /// Downloads a manifest file from a CDN. /// The manifest file is a JSON file that contains a list of files that are available to download. - /// The manifest file will be cached to the local filesystem. (im not sure if this is a good idea) + /// The manifest file is parsed and loaded into memory. /// - public async Task GetManifestAsync(string url) + private async Task GetManifestAsync(string url) { // Check if the URL is valid and whitelisted if (!IsValidUrl(url)) @@ -101,17 +217,19 @@ public async Task GetManifestAsync(string url) return "Manifest downloaded"; } - public async Task GetFileAsync(string url) - { + /// + /// This is not implemented yet. + /// Downloads a file from a CDN in chunks. + /// This is useful for downloading large files. + /// Has a callback to play audio while the file is being downloaded. + /// + /// + /// + public async Task GetChunkedFileAsync(string url, int chunkSize = -1) + { string filename = Path.GetFileName(url); string outputPath = Path.Combine(_downloadDirectory, filename); - - if (!Directory.Exists(_downloadDirectory)) - { - Directory.CreateDirectory(_downloadDirectory); - } - // expand %appdata% to the actual path string fullOutputPath = Environment.ExpandEnvironmentVariables(outputPath); @@ -119,86 +237,141 @@ public async Task GetFileAsync(string url) if (File.Exists(fullOutputPath)) { _sawmill.Info($"File already exists at {outputPath}"); - return outputPath; + + PlayOggAudioFile(fullOutputPath); + + return "File already exists"; } // Check if the URL is valid and whitelisted if (!IsValidUrl(url)) { + _sawmill.Error("Invalid URL"); return "Invalid URL"; } - try - { - var response = await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead); - response.EnsureSuccessStatusCode(); - // Validate the response - // here we are checking if the response is an audio file - // TODO: ideally this should verify integrity of the file using a hash - var mediaType = response.Content.Headers.ContentType?.MediaType ?? ""; - var fileExtension = Path.GetExtension(url); + HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, url); - // TODO: this shouldn't be hardcoded, whitelisting file types should be in a configuration file - if (!mediaType.Equals("application/ogg", StringComparison.OrdinalIgnoreCase)) + if (chunkSize != -1) + request.Headers.Range = new RangeHeaderValue(0, chunkSize); + + long totalBytes = 0; + long bytesDownloaded = 0; + + // TODO: getting the total size of the file should already be available in a manifest file inside cdn + // this should be refactored to get the total size from the manifest file so we don't have to make a HEAD request + try + { + // Get the total size of the file + using (var response = await _httpClient.SendAsync(new HttpRequestMessage(HttpMethod.Head, url))) { - // _sawmill.Error($"CDNConsumer: Invalid media type {nameof(mediaType)}"); - return "Invalid media type"; + response.EnsureSuccessStatusCode(); + totalBytes = response.Content.Headers.ContentLength ?? 0; + } + } + catch (HttpRequestException e) + { + _sawmill.Error("Failed to get file size"); + return "Failed to get file size"; + } + while (bytesDownloaded < totalBytes) + { + var chunkRequest = new HttpRequestMessage(HttpMethod.Get, url); + if (chunkSize == -1) + chunkSize = (int)totalBytes; - using (var contentStream = await response.Content.ReadAsStreamAsync()) - using (var fileStream = new FileStream(fullOutputPath, FileMode.Create, FileAccess.Write, FileShare.None, 8192, true)) + chunkRequest.Headers.Range = new RangeHeaderValue(bytesDownloaded, bytesDownloaded + chunkSize - 1); + + try { - try + + using (var response = await _httpClient.SendAsync(chunkRequest)) { + response.EnsureSuccessStatusCode(); + var content = await response.Content.ReadAsByteArrayAsync(); - await contentStream.CopyToAsync(fileStream); + // // Queue the buffer to the audio source + // QueueBuffer(content); - } - catch (Exception e) - { - _sawmill.Error($"Failed to save file {e}"); - return "Failed to save file"; + MemoryStream stream = new MemoryStream(); + + await stream.WriteAsync(content, (int)bytesDownloaded, content.Length); + + //_audioMgr.LoadAudioPartialOggVorbis(stream, (int)bytesDownloaded, content.Length, filename); + + bytesDownloaded += content.Length; } } + catch (HttpRequestException e) + { + _sawmill.Error("Failed to download file in chunks"); + return "Failed to download file in chunks"; + } + - _sawmill.Info($"File downloaded to {outputPath}"); - return outputPath; + _sawmill.Info($"Downloaded {bytesDownloaded}/{totalBytes} bytes"); } - catch (HttpRequestException e) + + _sawmill.Info("File downloaded in chunks"); + return "File downloaded in chunks"; + } + + + // for testing purposes only + private void PlayOggAudioFile(string filepath) + { + string filename = Path.GetFileName(filepath); + using FileStream fileStream = new FileStream(filepath, FileMode.Open, FileAccess.Read); + AudioStream audioStream = _audioMgr.LoadAudioOggVorbis(fileStream, filename); + var source = _audioMgr.CreateAudioSource(audioStream); + + if (source != null) { - _runtimeLog.LogException(e, "CDNConsumer: Failed to download file"); - return "Failed to download file"; + _activeSources.Add(source); + source.StartPlaying(); + + // add an event listener to remove the source from the list when it finishes playing + source.PlaybackFinished += (src) => + { + src.Dispose(); + _activeSources.Remove(src); + }; } + else + _sawmill.Error("Failed to create audio source"); } - // Check if the URL is valid and whitelisted private bool IsValidUrl(string url) { // Check if URL is valid if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) { - // _sawmill.Error("CDNConsumer: Invalid URL"); + _sawmill.Error("Invalid URL"); return false; } + // uncomment this if to enforce domain whitelist // Check if domain is whitelisted - var domain = uri.Host; - if (!Array.Exists(_whitelistedDomains, d => d.Equals(domain, StringComparison.OrdinalIgnoreCase))) - { - // _sawmill.Error("CDNConsumer: Domain is not whitelisted"); - return false; - } - + // var domain = uri.Host; + // if (!Array.Exists(_whitelistedDomains, d => d.Equals(domain, StringComparison.OrdinalIgnoreCase))) + // { + // _sawmill.Error("Requested domain is not whitelisted"); + // return false; + // } + + // uncomment this if to enforce HTTPS // Ensure URL uses HTTPS - if (!url.StartsWith("https://")) - { - // _sawmill.Error("CDNConsumer: URL must use HTTPS"); - return false; - } + // if (!url.StartsWith("https://")) + // { + // // _sawmill.Error("CDNConsumer: URL must use HTTPS"); + // return false; + // } return true; } + } } From e13c2c274d50e8f2d201a2333d21d46097dd6765 Mon Sep 17 00:00:00 2001 From: luiz Date: Thu, 5 Dec 2024 18:47:59 -0300 Subject: [PATCH 5/5] adds NVorbis package to Robus.Client.csproj --- Robust.Client/HTTPClient/HTTPClient.cs | 20 ++++++++++---------- Robust.Client/Robust.Client.csproj | 1 + 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Robust.Client/HTTPClient/HTTPClient.cs b/Robust.Client/HTTPClient/HTTPClient.cs index 2776635c1b3..621ba6d5a4d 100644 --- a/Robust.Client/HTTPClient/HTTPClient.cs +++ b/Robust.Client/HTTPClient/HTTPClient.cs @@ -15,6 +15,8 @@ using Robust.Shared.Exceptions; using Robust.Shared.Log; +using NVorbis; + namespace Robust.Client.HTTPClient { @@ -30,8 +32,6 @@ public interface ICDNConsumer Task PlayAudioFromCDN(string url); - void StopAudio(); - Task GetFileAsync(string url); } @@ -160,7 +160,7 @@ public async Task GetFileAsync(string url) } catch (Exception e) { - _sawmill.Error($"Failed to save file {e}"); + _sawmill.Error($"Failed to save file: {e}"); return "Failed to save file"; } } @@ -170,7 +170,7 @@ public async Task GetFileAsync(string url) } catch (HttpRequestException e) { - _sawmill.Error(e, "Failed to download file"); + _sawmill.Error($"Failed to download file: {e}"); return "Failed to download file"; } } @@ -334,12 +334,12 @@ private void PlayOggAudioFile(string filepath) _activeSources.Add(source); source.StartPlaying(); - // add an event listener to remove the source from the list when it finishes playing - source.PlaybackFinished += (src) => - { - src.Dispose(); - _activeSources.Remove(src); - }; + // TODO: add an event listener to remove the source from the list when it finishes playing + // source.PlaybackFinished += (src) => + // { + // src.Dispose(); + // _activeSources.Remove(src); + // }; } else _sawmill.Error("Failed to create audio source"); diff --git a/Robust.Client/Robust.Client.csproj b/Robust.Client/Robust.Client.csproj index 9cc646e6748..6a08cbd9f97 100644 --- a/Robust.Client/Robust.Client.csproj +++ b/Robust.Client/Robust.Client.csproj @@ -19,6 +19,7 @@ +