RTN-025 and title category parsing (#195)

* update rtn to 025 * Implement movie / show type parsing * switch to RTN in collectors * ensure env for pythonnet is loaded, and that requirements copy for qbit * version bump
knightcrawler-stremio · Mar 31, 2024 · 684dbba · 684dbba
1 parent c75ecd2
commit 684dbba
Show file tree

Hide file tree

Showing 37 changed files with 218 additions and 112 deletions.
diff --git a/.gitignore b/.gitignore
@@ -612,3 +612,7 @@ fabric.properties
 # Mac directory indexes
 .DS_Store
 deployment/docker/stack.env
+
+src/producer/src/python/
+src/debrid-collector/python/
+src/qbit-collector/python/
diff --git a/deployment/docker/docker-compose.yaml b/deployment/docker/docker-compose.yaml
@@ -94,7 +94,7 @@ services:
         condition: service_healthy
     env_file: stack.env
     hostname: knightcrawler-addon
-    image: gabisonfire/knightcrawler-addon:2.0.17
+    image: gabisonfire/knightcrawler-addon:2.0.18
     labels:
       logging: promtail
     networks:
@@ -117,7 +117,7 @@ services:
       redis:
         condition: service_healthy
     env_file: stack.env
-    image: gabisonfire/knightcrawler-consumer:2.0.17
+    image: gabisonfire/knightcrawler-consumer:2.0.18
     labels:
       logging: promtail
     networks:
@@ -138,7 +138,7 @@ services:
       redis:
         condition: service_healthy
     env_file: stack.env
-    image: gabisonfire/knightcrawler-debrid-collector:2.0.17
+    image: gabisonfire/knightcrawler-debrid-collector:2.0.18
     labels:
       logging: promtail
     networks:
@@ -152,7 +152,7 @@ services:
       migrator:
         condition: service_completed_successfully
     env_file: stack.env
-    image: gabisonfire/knightcrawler-metadata:2.0.17
+    image: gabisonfire/knightcrawler-metadata:2.0.18
     networks:
       - knightcrawler-network
     restart: "no"
@@ -163,7 +163,7 @@ services:
       postgres:
         condition: service_healthy
     env_file: stack.env
-    image: gabisonfire/knightcrawler-migrator:2.0.17
+    image: gabisonfire/knightcrawler-migrator:2.0.18
     networks:
       - knightcrawler-network
     restart: "no"
@@ -182,7 +182,7 @@ services:
       redis:
         condition: service_healthy
     env_file: stack.env
-    image: gabisonfire/knightcrawler-producer:2.0.17
+    image: gabisonfire/knightcrawler-producer:2.0.18
     labels:
       logging: promtail
     networks:
@@ -207,7 +207,7 @@ services:
     deploy:
       replicas: ${QBIT_REPLICAS:-0}
     env_file: stack.env
-    image: gabisonfire/knightcrawler-qbit-collector:2.0.17
+    image: gabisonfire/knightcrawler-qbit-collector:2.0.18
     labels:
       logging: promtail
     networks:

diff --git a/deployment/docker/src/components/knightcrawler.yaml b/deployment/docker/src/components/knightcrawler.yaml
@@ -20,7 +20,7 @@ x-depends: &knightcrawler-app-depends
 
 services:
   metadata:
-    image: gabisonfire/knightcrawler-metadata:2.0.17
+    image: gabisonfire/knightcrawler-metadata:2.0.18
     env_file: ../../.env
     networks:
       - knightcrawler-network
@@ -30,7 +30,7 @@ services:
         condition: service_completed_successfully
 
   migrator:
-    image: gabisonfire/knightcrawler-migrator:2.0.17
+    image: gabisonfire/knightcrawler-migrator:2.0.18
     env_file: ../../.env
     networks:
       - knightcrawler-network
@@ -40,30 +40,30 @@ services:
         condition: service_healthy
 
   addon:
-    image: gabisonfire/knightcrawler-addon:2.0.17
+    image: gabisonfire/knightcrawler-addon:2.0.18
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
     hostname: knightcrawler-addon
     ports:
       - "7000:7000"
 
   consumer:
-    image: gabisonfire/knightcrawler-consumer:2.0.17
+    image: gabisonfire/knightcrawler-consumer:2.0.18
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
 
   debridcollector:
-    image: gabisonfire/knightcrawler-debrid-collector:2.0.17
+    image: gabisonfire/knightcrawler-debrid-collector:2.0.18
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
 
   producer:
-    image: gabisonfire/knightcrawler-producer:2.0.17
+    image: gabisonfire/knightcrawler-producer:2.0.18
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
 
   qbitcollector:
-    image: gabisonfire/knightcrawler-qbit-collector:2.0.17
+    image: gabisonfire/knightcrawler-qbit-collector:2.0.18
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
     depends_on:

diff --git a/src/debrid-collector/DebridCollector.csproj b/src/debrid-collector/DebridCollector.csproj
@@ -17,7 +17,6 @@
       <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
       <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="8.0.3" />
       <PackageReference Include="Polly" Version="8.3.1" />
-      <PackageReference Include="PromKnight.ParseTorrentTitle" Version="1.0.4" />
       <PackageReference Include="Serilog" Version="3.1.1" />
       <PackageReference Include="Serilog.AspNetCore" Version="8.0.1" />
       <PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
@@ -29,10 +28,30 @@
       <None Include="Configuration\logging.json">
         <CopyToOutputDirectory>Always</CopyToOutputDirectory>
       </None>
+      <None Update="requirements.txt">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      </None>
+      <Content Remove="eng\**" />
+      <None Remove="eng\**" />
+    </ItemGroup>
+
+    <ItemGroup Condition="'$(Configuration)' == 'Debug'">
+        <Content Remove="python\**" />
+        <None Include="python\**">
+            <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+        </None>
     </ItemGroup>
 
     <ItemGroup>
       <ProjectReference Include="..\shared\SharedContracts.csproj" />
     </ItemGroup>
 
+    <ItemGroup>
+      <Compile Remove="eng\**" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <EmbeddedResource Remove="eng\**" />
+    </ItemGroup>
+
 </Project>
diff --git a/src/debrid-collector/DebridCollector.sln b/src/debrid-collector/DebridCollector.sln
@@ -6,6 +6,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedContracts", "..\share
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shared", "shared", "{2C0A0F53-28E6-404F-9EFE-DADFBEF8338B}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "eng", "eng", "{72A042C3-B4F3-45C5-AC20-041FE8F41EFC}"
+	ProjectSection(SolutionItems) = preProject
+		eng\install-python-reqs.ps1 = eng\install-python-reqs.ps1
+		eng\install-python-reqs.sh = eng\install-python-reqs.sh
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU

diff --git a/src/debrid-collector/Dockerfile b/src/debrid-collector/Dockerfile
@@ -9,12 +9,23 @@ RUN dotnet restore -a $TARGETARCH
 RUN dotnet publish -c Release --no-restore -o /src/out -a $TARGETARCH
 
 
-FROM mcr.microsoft.com/dotnet/aspnet:8.0-alpine
+FROM mcr.microsoft.com/dotnet/aspnet:8.0-alpine3.19
 
 WORKDIR /app
+
+ENV PYTHONUNBUFFERED=1
+
+RUN apk add --update --no-cache python3=~3.11.8-r0 py3-pip && ln -sf python3 /usr/bin/python
+
 COPY --from=build /src/out .
+
+RUN rm -rf /app/python && mkdir -p /app/python
+
+RUN pip3 install -r /app/requirements.txt -t /app/python
+
 RUN addgroup -S debrid && adduser -S -G debrid debrid
 USER debrid
 HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
   CMD pgrep -f dotnet || exit 1
+ENV PYTHONNET_PYDLL=/usr/lib/libpython3.11.so.1.0
 ENTRYPOINT ["dotnet", "DebridCollector.dll"]
diff --git a/src/debrid-collector/Extensions/ServiceCollectionExtensions.cs b/src/debrid-collector/Extensions/ServiceCollectionExtensions.cs
@@ -1,5 +1,3 @@
-using DebridCollector.Features.Configuration;
-
 namespace DebridCollector.Extensions;
 
 public static class ServiceCollectionExtensions
@@ -17,7 +15,8 @@ internal static IServiceCollection AddServiceConfiguration(this IServiceCollecti
         var serviceConfiguration = services.LoadConfigurationFromEnv<DebridCollectorConfiguration>();
 
         services.AddRealDebridClient(serviceConfiguration);
-        services.AddSingleton<IParseTorrentTitle, ParseTorrentTitle>();
+        services.RegisterPythonEngine();
+        services.AddSingleton<IRankTorrentName, RankTorrentName>();
         services.AddHostedService<DebridRequestProcessor>();
 
         return services;

diff --git a/src/debrid-collector/Features/Debrid/ServiceCollectionExtensions.cs b/src/debrid-collector/Features/Debrid/ServiceCollectionExtensions.cs
@@ -1,6 +1,4 @@
-using DebridCollector.Features.Configuration;
-
-namespace DebridCollector.Features.Debrid;
+namespace DebridCollector.Features.Debrid;
 
 public static class ServiceCollectionExtensions
 {

diff --git a/src/debrid-collector/Features/Worker/DebridMetaToTorrentMeta.cs b/src/debrid-collector/Features/Worker/DebridMetaToTorrentMeta.cs
@@ -3,10 +3,11 @@ namespace DebridCollector.Features.Worker;
 public static class DebridMetaToTorrentMeta
 {
     public static IReadOnlyList<TorrentFile> MapMetadataToFilesCollection(
-        IParseTorrentTitle torrentTitle,
+        IRankTorrentName rankTorrentName,
         Torrent torrent,
         string ImdbId,
-        FileDataDictionary Metadata)
+        FileDataDictionary Metadata,
+        ILogger<WriteMetadataConsumer> logger)
     {
         try
         {
@@ -26,23 +27,30 @@ public static IReadOnlyList<TorrentFile> MapMetadataToFilesCollection(
                     Size = metadataEntry.Value.Filesize.GetValueOrDefault(),
                 };
 
-                var parsedTitle = torrentTitle.Parse(file.Title);
+                var parsedTitle = rankTorrentName.Parse(file.Title, false);
 
-                file.ImdbSeason = parsedTitle.Seasons.FirstOrDefault();
-                file.ImdbEpisode = parsedTitle.Episodes.FirstOrDefault();
+                if (!parsedTitle.Success)
+                {
+                    logger.LogWarning("Failed to parse title {Title} for metadata mapping", file.Title);
+                    continue;
+                }
+
+                file.ImdbSeason = parsedTitle.Response?.Season?.FirstOrDefault() ?? 0;
+                file.ImdbEpisode = parsedTitle.Response?.Episode?.FirstOrDefault() ?? 0;
 
                 files.Add(file);
             }
 
             return files;
         }
-        catch (Exception)
+        catch (Exception ex)
         {
+            logger.LogWarning("Failed to map metadata to files collection: {Exception}", ex.Message);
             return [];
         }
     }
 
-    public static async Task<IReadOnlyList<SubtitleFile>> MapMetadataToSubtitlesCollection(IDataStorage storage, string InfoHash, FileDataDictionary Metadata)
+    public static async Task<IReadOnlyList<SubtitleFile>> MapMetadataToSubtitlesCollection(IDataStorage storage, string InfoHash, FileDataDictionary Metadata, ILogger<WriteMetadataConsumer> logger)
     {
         try
         {
@@ -74,8 +82,9 @@ public static async Task<IReadOnlyList<SubtitleFile>> MapMetadataToSubtitlesColl
 
             return files;
         }
-        catch (Exception)
+        catch (Exception ex)
         {
+            logger.LogWarning("Failed to map metadata to subtitles collection: {Exception}", ex.Message);
             return [];
         }
     }

diff --git a/src/debrid-collector/Features/Worker/InfohashMetadataSagaStateMachine.cs b/src/debrid-collector/Features/Worker/InfohashMetadataSagaStateMachine.cs
@@ -53,6 +53,12 @@ public InfohashMetadataSagaStateMachine(ILogger<InfohashMetadataSagaStateMachine
                 .Then(
                     context =>
                     {
+                        if (!context.Message.WithFiles)
+                        {
+                            logger.LogInformation("No files written for torrent {InfoHash} in Saga {SagaId}", context.Saga.Torrent.InfoHash, context.Saga.CorrelationId);
+                            return;
+                        }
+
                         logger.LogInformation("Metadata Written for torrent {InfoHash} in Saga {SagaId}", context.Saga.Torrent.InfoHash, context.Saga.CorrelationId);
                     })
                 .TransitionTo(Completed)

diff --git a/src/debrid-collector/Features/Worker/Requests.cs b/src/debrid-collector/Features/Worker/Requests.cs
@@ -16,7 +16,7 @@ public record WriteMetadata(Torrent Torrent, TorrentMetadataResponse Metadata, s
 }
 
 [EntityName("metadata-written-debrid-colloctor")]
-public record MetadataWritten(TorrentMetadataResponse Metadata) : CorrelatedBy<Guid>
+public record MetadataWritten(TorrentMetadataResponse Metadata, bool WithFiles) : CorrelatedBy<Guid>
 {
     public Guid CorrelationId { get; init; } = Metadata.CorrelationId;
 }
diff --git a/src/debrid-collector/Features/Worker/WriteMetadataConsumer.cs b/src/debrid-collector/Features/Worker/WriteMetadataConsumer.cs
@@ -1,25 +1,28 @@
 namespace DebridCollector.Features.Worker;
 
-public class WriteMetadataConsumer(IParseTorrentTitle parseTorrentTitle, IDataStorage dataStorage) : IConsumer<WriteMetadata>
+public class WriteMetadataConsumer(IRankTorrentName rankTorrentName, IDataStorage dataStorage, ILogger<WriteMetadataConsumer> logger) : IConsumer<WriteMetadata>
 {
     public async Task Consume(ConsumeContext<WriteMetadata> context)
     {
         var request = context.Message;
 
-        var torrentFiles = DebridMetaToTorrentMeta.MapMetadataToFilesCollection(parseTorrentTitle, request.Torrent, request.ImdbId, request.Metadata.Metadata);
+        var torrentFiles = DebridMetaToTorrentMeta.MapMetadataToFilesCollection(rankTorrentName, request.Torrent, request.ImdbId, request.Metadata.Metadata, logger);
 
-        if (torrentFiles.Any())
+        if (!torrentFiles.Any())
         {
-            await dataStorage.InsertFiles(torrentFiles);
+            await context.Publish(new MetadataWritten(request.Metadata, false));
+            return;
+        }
+
+        await dataStorage.InsertFiles(torrentFiles);
 
-            var subtitles = await DebridMetaToTorrentMeta.MapMetadataToSubtitlesCollection(dataStorage, request.Torrent.InfoHash, request.Metadata.Metadata);
+        var subtitles = await DebridMetaToTorrentMeta.MapMetadataToSubtitlesCollection(dataStorage, request.Torrent.InfoHash, request.Metadata.Metadata, logger);
 
-            if (subtitles.Any())
-            {
-                await dataStorage.InsertSubtitles(subtitles);
-            }
+        if (subtitles.Any())
+        {
+            await dataStorage.InsertSubtitles(subtitles);
         }
 
-        await context.Publish(new MetadataWritten(request.Metadata));
+        await context.Publish(new MetadataWritten(request.Metadata, true));
     }
 }
diff --git a/src/debrid-collector/GlobalUsings.cs b/src/debrid-collector/GlobalUsings.cs
@@ -4,17 +4,18 @@
 global using System.Text.Json.Serialization;
 global using System.Threading.Channels;
 global using DebridCollector.Extensions;
+global using DebridCollector.Features.Configuration;
 global using DebridCollector.Features.Debrid;
 global using DebridCollector.Features.Worker;
 global using MassTransit;
-global using MassTransit.Mediator;
 global using Microsoft.AspNetCore.Builder;
 global using Microsoft.Extensions.DependencyInjection;
 global using Polly;
 global using Polly.Extensions.Http;
-global using PromKnight.ParseTorrentTitle;
 global using SharedContracts.Configuration;
 global using SharedContracts.Dapper;
 global using SharedContracts.Extensions;
 global using SharedContracts.Models;
+global using SharedContracts.Python;
+global using SharedContracts.Python.RTN;
 global using SharedContracts.Requests;
diff --git a/src/debrid-collector/eng/install-python-reqs.ps1 b/src/debrid-collector/eng/install-python-reqs.ps1
@@ -0,0 +1,2 @@
+mkdir -p ../python
+python -m pip install -r ../requirements.txt -t ../python/
diff --git a/src/debrid-collector/eng/install-python-reqs.sh b/src/debrid-collector/eng/install-python-reqs.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+rm -rf ../python
+mkdir -p ../python
+python3 -m pip install -r ../requirements.txt -t ../python/
diff --git a/src/debrid-collector/requirements.txt b/src/debrid-collector/requirements.txt
@@ -0,0 +1 @@
+rank-torrent-name==0.2.5
diff --git a/src/producer/src/requirements.txt b/src/producer/src/requirements.txt
@@ -1 +1 @@
-rank-torrent-name==0.1.8
+rank-torrent-name==0.2.5
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		mkdir -p ../python
		python -m pip install -r ../requirements.txt -t ../python/