From fdab54448faf52c349069fef37f9746ad7dcba44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=B3=E9=88=9E?= Date: Thu, 23 May 2024 19:25:31 +0800 Subject: [PATCH 1/3] feat: implement missing job status verification and handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Added `IsJobMissing` functionality to `IJobService.cs` interface to verify if a job is missing. - Incorporated handling for `VideoStatus.Missing` in `FC2Service.cs`, `TwitcastingService.cs` & `TwitchService.cs` to warn about possible server malfunction during previous recording and revert video status to `WaitingToRecord`. - Added methods to `ACIService.cs` & `KubernetesService.cs` to check if a job is missing. - Improved readability and safety of `GetResourceByKeywordAsync` function in `ACIService.cs`. - Included checking for missing job status and updating video status accordingly in `RecordService.cs`. Signed-off-by: 陳鈞 --- Interfaces/IJobService.cs | 2 ++ ScopedServices/PlatformService/FC2Service.cs | 9 ++++++- .../PlatformService/TwitcastingService.cs | 20 ++++++++++++++ .../PlatformService/TwitchService.cs | 27 +++++++++++++++++++ SingletonServices/ACIService.cs | 27 +++++++++++++++---- SingletonServices/KubernetesService.cs | 10 +++++++ SingletonServices/RecordService.cs | 17 +++++++++++- 7 files changed, 105 insertions(+), 7 deletions(-) diff --git a/Interfaces/IJobService.cs b/Interfaces/IJobService.cs index f84ca35..04109c5 100644 --- a/Interfaces/IJobService.cs +++ b/Interfaces/IJobService.cs @@ -4,6 +4,8 @@ namespace LivestreamRecorderService.Interfaces; public interface IJobService { + Task IsJobMissing(Video video, CancellationToken cancellation); + Task IsJobMissing(string keyword, CancellationToken cancellation); Task IsJobFailedAsync(Video video, CancellationToken cancellation = default); Task IsJobFailedAsync(string keyword, CancellationToken cancellation = default); Task IsJobSucceededAsync(Video video, CancellationToken cancellation = default); diff --git a/ScopedServices/PlatformService/FC2Service.cs b/ScopedServices/PlatformService/FC2Service.cs index 33132e4..bb61c04 100644 --- a/ScopedServices/PlatformService/FC2Service.cs +++ b/ScopedServices/PlatformService/FC2Service.cs @@ -84,6 +84,13 @@ public override async Task UpdateVideosDataAsync(Channel channel, CancellationTo case VideoStatus.Skipped: logger.LogTrace("{videoId} is rejected for recording.", video.id); return; + case VideoStatus.Missing: + logger.LogWarning( + "{videoId} has been marked missing. It is possible that a server malfunction occurred during the previous recording. Changed its state back to Recording.", + video.id); + + video.Status = VideoStatus.WaitingToRecord; + break; case VideoStatus.Archived: case VideoStatus.PermanentArchived: logger.LogWarning( @@ -98,8 +105,8 @@ public override async Task UpdateVideosDataAsync(Channel channel, CancellationTo case VideoStatus.Pending: case VideoStatus.WaitingToDownload: case VideoStatus.Downloading: + //case VideoStatus.Uploading: case VideoStatus.Expired: - case VideoStatus.Missing: case VideoStatus.Error: case VideoStatus.Exist: case VideoStatus.Edited: diff --git a/ScopedServices/PlatformService/TwitcastingService.cs b/ScopedServices/PlatformService/TwitcastingService.cs index 9919d73..4dfeb3f 100644 --- a/ScopedServices/PlatformService/TwitcastingService.cs +++ b/ScopedServices/PlatformService/TwitcastingService.cs @@ -71,6 +71,13 @@ public override async Task UpdateVideosDataAsync(Channel channel, CancellationTo case VideoStatus.Skipped: logger.LogTrace("{videoId} is rejected for recording.", video.id); return; + case VideoStatus.Missing: + logger.LogWarning( + "{videoId} has been marked missing. It is possible that a server malfunction occurred during the previous recording. Changed its state back to Recording.", + video.id); + + video.Status = VideoStatus.WaitingToRecord; + break; case VideoStatus.Archived: case VideoStatus.PermanentArchived: logger.LogWarning( @@ -79,7 +86,20 @@ public override async Task UpdateVideosDataAsync(Channel channel, CancellationTo video.Status = VideoStatus.WaitingToRecord; break; + + case VideoStatus.Unknown: + case VideoStatus.Scheduled: + case VideoStatus.Pending: + case VideoStatus.WaitingToDownload: + case VideoStatus.Downloading: + //case VideoStatus.Uploading: + case VideoStatus.Expired: + case VideoStatus.Error: + case VideoStatus.Exist: + case VideoStatus.Edited: + case VideoStatus.Deleted: default: + // All cases should be handled logger.LogWarning("{videoId} is in {status}.", video.id, Enum.GetName(typeof(VideoStatus), video.Status)); return; } diff --git a/ScopedServices/PlatformService/TwitchService.cs b/ScopedServices/PlatformService/TwitchService.cs index f63ecbb..7df6dd3 100644 --- a/ScopedServices/PlatformService/TwitchService.cs +++ b/ScopedServices/PlatformService/TwitchService.cs @@ -82,6 +82,33 @@ public override async Task UpdateVideosDataAsync(Channel channel, CancellationTo case VideoStatus.Skipped: logger.LogTrace("{videoId} is rejected for recording.", video.id); return; + case VideoStatus.Missing: + logger.LogWarning( + "{videoId} has been marked missing. It is possible that a server malfunction occurred during the previous recording. Changed its state back to Recording.", + video.id); + + video.Status = VideoStatus.WaitingToRecord; + break; + case VideoStatus.Archived: + case VideoStatus.PermanentArchived: + logger.LogWarning( + "{videoId} has already been archived. It is possible that an internet disconnect occurred during the process. Changed its state back to Recording.", + video.id); + + video.Status = VideoStatus.WaitingToRecord; + break; + + case VideoStatus.Unknown: + case VideoStatus.Scheduled: + case VideoStatus.Pending: + case VideoStatus.WaitingToDownload: + case VideoStatus.Downloading: + //case VideoStatus.Uploading: + case VideoStatus.Expired: + case VideoStatus.Error: + case VideoStatus.Exist: + case VideoStatus.Edited: + case VideoStatus.Deleted: default: logger.LogWarning("{videoId} is in {status}, skip.", video.id, Enum.GetName(typeof(VideoStatus), video.Status)); return; diff --git a/SingletonServices/ACIService.cs b/SingletonServices/ACIService.cs index b4d6ea8..5d0c70b 100644 --- a/SingletonServices/ACIService.cs +++ b/SingletonServices/ACIService.cs @@ -21,6 +21,16 @@ public class AciService(ILogger logger, private const string FallbackRegistry = "recordermoe/"; private readonly string _resourceGroupName = options.Value.ContainerInstance!.ResourceGroupName; + public Task IsJobMissing(Video video, CancellationToken cancellation) + { + return IsJobMissing(NameHelper.CleanUpInstanceName(video.id), cancellation); + } + + public async Task IsJobMissing(string keyword, CancellationToken cancellation) + { + return null == (await GetResourceByKeywordAsync(keyword, cancellation)); + } + public Task IsJobSucceededAsync(Video video, CancellationToken cancellation = default) { return IsJobSucceededAsync(NameHelper.CleanUpInstanceName(video.id), cancellation); @@ -29,7 +39,9 @@ public Task IsJobSucceededAsync(Video video, CancellationToken cancellatio public async Task IsJobSucceededAsync(string keyword, CancellationToken cancellation = default) { ContainerGroupResource? resource = await GetResourceByKeywordAsync(keyword, cancellation); - return null != resource && resource.HasData && resource.Data.InstanceView.State == "Succeeded"; + return null != resource + && resource.HasData + && resource.Data.InstanceView.State == "Succeeded"; } public Task IsJobFailedAsync(Video video, CancellationToken cancellation = default) @@ -40,7 +52,9 @@ public Task IsJobFailedAsync(Video video, CancellationToken cancellation = public async Task IsJobFailedAsync(string keyword, CancellationToken cancellation) { ContainerGroupResource? resource = await GetResourceByKeywordAsync(keyword, cancellation); - return null == resource || !resource.HasData || resource.Data.InstanceView.State == "Failed"; + return null != resource + && (!resource.HasData + || resource.Data.InstanceView.State == "Failed"); } /// @@ -166,9 +180,12 @@ await armDeploymentCollection.CreateOrUpdateAsync( resourceGroupResource.GetContainerGroups() .FirstOrDefault(p => p.Id.Name.Contains(NameHelper.CleanUpInstanceName(keyword))); - return null == containerGroupResourceTemp - ? null - : (await resourceGroupResource.GetContainerGroupAsync(containerGroupResourceTemp.Id.Name, cancellation)).Value; + if (null == containerGroupResourceTemp) return null; + + Response response = + (await resourceGroupResource.GetContainerGroupAsync(containerGroupResourceTemp.Id.Name, cancellation)); + + return response.HasValue ? response.Value : null; } private async Task GetResourceGroupAsync(CancellationToken cancellation = default) diff --git a/SingletonServices/KubernetesService.cs b/SingletonServices/KubernetesService.cs index 44ede2f..cc61e46 100644 --- a/SingletonServices/KubernetesService.cs +++ b/SingletonServices/KubernetesService.cs @@ -19,6 +19,16 @@ public class KubernetesService( private readonly string _kubernetesNamespace = options.Value.Namespace ?? "recordermoe"; + public Task IsJobMissing(Video video, CancellationToken cancellation) + { + return IsJobMissing(NameHelper.CleanUpInstanceName(video.id), cancellation); + } + + public async Task IsJobMissing(string keyword, CancellationToken cancellation) + { + return (await GetJobsByKeywordAsync(keyword, cancellation)).Count == 0; + } + public Task IsJobSucceededAsync(Video video, CancellationToken cancellation = default) { return IsJobSucceededAsync(NameHelper.CleanUpInstanceName(video.id), cancellation); diff --git a/SingletonServices/RecordService.cs b/SingletonServices/RecordService.cs index 9de387b..622112e 100644 --- a/SingletonServices/RecordService.cs +++ b/SingletonServices/RecordService.cs @@ -66,7 +66,22 @@ public async Task HandledFailedJobsAsync(VideoService videoService, Cancellation _logger.LogTrace("No videos recording/downloading"); foreach (Video video in videos) - if (await _jobService.IsJobFailedAsync(video, stoppingToken)) + if (await _jobService.IsJobMissing(video, stoppingToken)) + switch (video.Source) + { + case "Youtube": + await videoService.UpdateVideoStatusAsync(video, VideoStatus.Pending); + _logger.LogWarning("{videoId} is missing. Set status to {status}", video.id, video.Status); + break; + default: + await videoService.UpdateVideoStatusAsync(video, VideoStatus.Missing); + await videoService.UpdateVideoNoteAsync(video, + "This video archive is missing. If you would like to provide it, please contact admin."); + + _logger.LogWarning("{videoId} is missing.", video.id); + break; + } + else if (await _jobService.IsJobFailedAsync(video, stoppingToken)) switch (video.Source) { case "Youtube": From bc0deb320b2942020b10f16923fd077eae7ec900 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=B3=E9=88=9E?= Date: Thu, 23 May 2024 22:56:36 +0800 Subject: [PATCH 2/3] chore: fix docker debug configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 陳鈞 --- LivestreamRecorderService.csproj | 3 +++ LivestreamRecorderService.sln | 6 ------ docker-compose.dcproj | 19 ------------------- docker-compose.override.yml | 9 --------- docker-compose.yml | 13 ------------- launchSettings.json | 11 ----------- 6 files changed, 3 insertions(+), 58 deletions(-) delete mode 100644 docker-compose.dcproj delete mode 100644 docker-compose.override.yml delete mode 100644 docker-compose.yml delete mode 100644 launchSettings.json diff --git a/LivestreamRecorderService.csproj b/LivestreamRecorderService.csproj index 19d5155..223d975 100644 --- a/LivestreamRecorderService.csproj +++ b/LivestreamRecorderService.csproj @@ -12,6 +12,7 @@ false Linux . + Fast debug CouchDB;CosmosDB;AzureCosmosDB_Release;ApacheCouchDB_Release @@ -23,6 +24,8 @@ $(DefineConstants);COUCHDB;RELEASE True + + diff --git a/LivestreamRecorderService.sln b/LivestreamRecorderService.sln index 9c0c041..00f707e 100644 --- a/LivestreamRecorderService.sln +++ b/LivestreamRecorderService.sln @@ -7,8 +7,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LivestreamRecorderService", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LivestreamRecorder.DB", "LivestreamRecorder.DB\LivestreamRecorder.DB.csproj", "{194EA7A4-7735-40ED-B89A-AFAC094B91C0}" EndProject -Project("{E53339B2-1760-4266-BCC7-CA923CBCF16C}") = "docker-compose", "docker-compose.dcproj", "{D9705463-12BD-4589-B8E0-53C6EE792EE8}" -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "方案項目", "方案項目", "{05516183-EF2E-4C8F-B2F3-97C2222C1723}" ProjectSection(SolutionItems) = preProject .github\workflows\docker_publish.yml = .github\workflows\docker_publish.yml @@ -38,10 +36,6 @@ Global {194EA7A4-7735-40ED-B89A-AFAC094B91C0}.CosmosDB|Any CPU.Build.0 = CosmosDB|Any CPU {194EA7A4-7735-40ED-B89A-AFAC094B91C0}.CouchDB|Any CPU.ActiveCfg = CouchDB|Any CPU {194EA7A4-7735-40ED-B89A-AFAC094B91C0}.CouchDB|Any CPU.Build.0 = CouchDB|Any CPU - {D9705463-12BD-4589-B8E0-53C6EE792EE8}.ApacheCouchDB_Release|Any CPU.ActiveCfg = Release|Any CPU - {D9705463-12BD-4589-B8E0-53C6EE792EE8}.AzureCosmosDB_Release|Any CPU.ActiveCfg = Release|Any CPU - {D9705463-12BD-4589-B8E0-53C6EE792EE8}.CosmosDB|Any CPU.ActiveCfg = Release|Any CPU - {D9705463-12BD-4589-B8E0-53C6EE792EE8}.CouchDB|Any CPU.ActiveCfg = CouchDB|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/docker-compose.dcproj b/docker-compose.dcproj deleted file mode 100644 index 216452e..0000000 --- a/docker-compose.dcproj +++ /dev/null @@ -1,19 +0,0 @@ - - - - 2.1 - Linux - d9705463-12bd-4589-b8e0-53c6ee792ee8 - - - livestreamrecorderservice - - - - - docker-compose.yml - - - - - \ No newline at end of file diff --git a/docker-compose.override.yml b/docker-compose.override.yml deleted file mode 100644 index 97bf4ed..0000000 --- a/docker-compose.override.yml +++ /dev/null @@ -1,9 +0,0 @@ -services: - livestreamrecorderservice: - image: ghcr.io/recorder-moe/livestreamrecorderservice:debug - build: - target: debug - environment: - - DOTNET_ENVIRONMENT=Development - volumes: - - ${APPDATA}/Microsoft/UserSecrets:/root/.microsoft/usersecrets:ro \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 63d32ae..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,13 +0,0 @@ -services: - livestreamrecorderservice: - container_name: livestreamrecorderservice - image: ghcr.io/recorder-moe/livestreamrecorderservice:latest - user: "1654:0" - env_file: - - .env - volumes: - - ~/.kube/config:/.kube/config - # - ./appsettings.json:/app/appsettings.json - extra_hosts: - - "host.docker.internal:host-gateway" - restart: unless-stopped diff --git a/launchSettings.json b/launchSettings.json deleted file mode 100644 index 63f8eed..0000000 --- a/launchSettings.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "profiles": { - "Docker Compose": { - "commandName": "DockerCompose", - "commandVersion": "1.0", - "serviceActions": { - "livestreamrecorderservice": "StartDebugging" - } - } - } -} \ No newline at end of file From 8ded74dcbe6c9c39899a97ca3467c2ad2fec8989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=B3=E9=88=9E?= Date: Thu, 23 May 2024 22:57:09 +0800 Subject: [PATCH 3/3] refactor: improve logging and exception handling in S3Service MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Change log messages in `YoutubeDLHelper.cs` to eliminate `.exe` for `yt-dlp` and `ffmpeg` - Add a new entry to user dictionary in `LivestreamRecorderService.sln.DotSettings` - Refactor variable declarations and method calls in `PlatformService.cs` for clarity - Expand file upload logging in `S3Service.cs` by adding size and etag information and warning for empty etag, indicating potential failed upload Signed-off-by: 陳鈞 --- Helper/YoutubeDLHelper.cs | 4 +-- LivestreamRecorderService.sln.DotSettings | 1 + .../PlatformService/PlatformService.cs | 31 ++++++++++--------- SingletonServices/S3Service.cs | 26 +++++++++++----- 4 files changed, 38 insertions(+), 24 deletions(-) diff --git a/Helper/YoutubeDLHelper.cs b/Helper/YoutubeDLHelper.cs index 007164f..24289a4 100644 --- a/Helper/YoutubeDLHelper.cs +++ b/Helper/YoutubeDLHelper.cs @@ -111,8 +111,8 @@ from e in extensions where File.Exists(path) select path)?.FirstOrDefault(); - Log.Debug("Found yt-dlp.exe at {YtdlpPath}", _YtdlpPath); - Log.Debug("Found ffmpeg.exe at {FFmpegPath}", _FFmpegPath); + Log.Debug("Found yt-dlp at {YtdlpPath}", _YtdlpPath); + Log.Debug("Found ffmpeg at {FFmpegPath}", _FFmpegPath); return (_YtdlpPath, _FFmpegPath); } diff --git a/LivestreamRecorderService.sln.DotSettings b/LivestreamRecorderService.sln.DotSettings index c61ab2f..433c5f3 100644 --- a/LivestreamRecorderService.sln.DotSettings +++ b/LivestreamRecorderService.sln.DotSettings @@ -1,4 +1,5 @@  + True True True True diff --git a/ScopedServices/PlatformService/PlatformService.cs b/ScopedServices/PlatformService/PlatformService.cs index 3d2db8a..d41f128 100644 --- a/ScopedServices/PlatformService/PlatformService.cs +++ b/ScopedServices/PlatformService/PlatformService.cs @@ -104,7 +104,7 @@ public bool StepInterval(int elapsedTime) $"Failed to fetch video data from yt-dlp for URL: {url}. Errors: {string.Join(' ', res.ErrorOutput)}"); } - var videoData = res.Data; + YtdlpVideoData? videoData = res.Data; return videoData; } catch (Exception e) @@ -190,21 +190,21 @@ public bool StepInterval(int elapsedTime) throw new ArgumentNullException(nameof(path)); } - string? extension, contentType, pathInStorage, tempPath; + string? contentType, pathInStorage, tempPath; try { - using var client = HttpClientFactory.CreateClient(); - var response = await client.GetAsync(url, cancellation); + using HttpClient client = HttpClientFactory.CreateClient(); + HttpResponseMessage response = await client.GetAsync(url, cancellation); response.EnsureSuccessStatusCode(); contentType = response.Content.Headers.ContentType?.MediaType; - extension = MimeUtility.GetExtensions(contentType)?.FirstOrDefault(); + string? extension = MimeUtility.GetExtensions(contentType)?.FirstOrDefault(); extension = extension == "jpeg" ? "jpg" : extension; pathInStorage = $"{path}.{extension}"; tempPath = Path.GetTempFileName(); tempPath = Path.ChangeExtension(tempPath, extension); - await using var contentStream = await response.Content.ReadAsStreamAsync(cancellation); + await using Stream contentStream = await response.Content.ReadAsStreamAsync(cancellation); await using var fileStream = new FileStream(tempPath, FileMode.Create); await contentStream.CopyToAsync(fileStream, cancellation); } @@ -216,16 +216,16 @@ public bool StepInterval(int elapsedTime) try { - List tasks = - [ - StorageService.UploadPublicFileAsync(contentType, pathInStorage, tempPath, cancellation), - StorageService.UploadPublicFileAsync(KnownMimeTypes.Avif, - $"{path}.avif", - await ImageHelper.ConvertToAvifAsync(tempPath), - cancellation) - ]; + await StorageService.UploadPublicFileAsync(contentType: contentType, + pathInStorage: pathInStorage, + filePathToUpload: tempPath, + cancellation: cancellation); + + await StorageService.UploadPublicFileAsync(contentType: KnownMimeTypes.Avif, + pathInStorage: $"{path}.avif", + filePathToUpload: await ImageHelper.ConvertToAvifAsync(tempPath), + cancellation: cancellation); - await Task.WhenAll(tasks); return pathInStorage; } catch (Exception e) @@ -242,6 +242,7 @@ await ImageHelper.ConvertToAvifAsync(tempPath), } catch (IOException) { + // ignored } } } diff --git a/SingletonServices/S3Service.cs b/SingletonServices/S3Service.cs index 9c3323d..d6d7365 100644 --- a/SingletonServices/S3Service.cs +++ b/SingletonServices/S3Service.cs @@ -4,6 +4,7 @@ using Minio; using Minio.DataModel; using Minio.DataModel.Args; +using Minio.DataModel.Response; using Minio.Exceptions; namespace LivestreamRecorderService.SingletonServices; @@ -71,14 +72,25 @@ public async Task UploadPublicFileAsync(string? contentType, string pathInStorag { try { - await minioClient.PutObjectAsync(new PutObjectArgs() - .WithBucket(_options.BucketName_Public) - .WithObject(pathInStorage) - .WithFileName(tempPath) - .WithContentType(contentType), - cancellation); + string bucketNamePublic = _options.BucketName_Public; + PutObjectArgs putObjectArgs = new PutObjectArgs().WithBucket(bucketNamePublic) + .WithObject(pathInStorage) + .WithFileName(tempPath) + .WithContentType(contentType); + + PutObjectResponse result = await minioClient.PutObjectAsync(putObjectArgs, cancellation); + + logger.LogInformation("Uploaded to S3 {S3Server} {bucket}/{filePath}, {size}, {etag}", + minioClient.Config.Endpoint, + bucketNamePublic, + result.ObjectName, + result.Size, + result.Etag); + + if (string.IsNullOrEmpty(result.Etag)) + logger.LogWarning("The Etag is empty for the uploaded file at {filePath}.", pathInStorage); } - catch (MinioException e) + catch (Exception e) { logger.LogError(e, "Failed to upload public file: {filePath}", pathInStorage); }