[wip] bridge python and c# and bring in rank torrent name (#177)

* [wip] bridge python and c# and bring in rank torrent name * Container restores package now Includes two dev scripts to install the python packages locally for debugging purposes. * Introduce slightly turned title matching scoring, by making it length aware this should help with sequels such as Terminator 2, vs Terminator etc * Version bump Also fixes postgres healthcheck so that it utilises the user from the stack.env file
knightcrawler-stremio · Mar 28, 2024 · 4308a0e · 4308a0e
1 parent cc15a69
commit 4308a0e
Show file tree

Hide file tree

Showing 24 changed files with 314 additions and 108 deletions.
diff --git a/deployment/docker/docker-compose.yaml b/deployment/docker/docker-compose.yaml
@@ -17,12 +17,8 @@ services:
   ## All downloaded metadata is stored in this database.
   postgres:
     env_file: stack.env
-    environment:
-      PGUSER: ${POSTGRES_USER}
     healthcheck:
-      test:
-        - CMD-SHELL
-        - pg_isready
+      test: [ "CMD", "sh", "-c", "pg_isready -h localhost -U $$POSTGRES_USER" ]
       timeout: 10s
       interval: 10s
       retries: 3
@@ -44,9 +40,7 @@ services:
   redis:
     env_file: stack.env
     healthcheck:
-      test:
-        - CMD-SHELL
-        - redis-cli ping
+      test: ["CMD-SHELL", "redis-cli ping"]
       timeout: 10s
       interval: 10s
       retries: 3
@@ -66,9 +60,7 @@ services:
   rabbitmq:
     env_file: stack.env
     healthcheck:
-      test:
-        - CMD-SHELL
-        - rabbitmq-diagnostics -q ping
+      test: ["CMD-SHELL", "rabbitmq-diagnostics -q ping"]
       timeout: 10s
       interval: 10s
       retries: 3
@@ -91,22 +83,17 @@ services:
     depends_on:
       metadata:
         condition: service_completed_successfully
-        required: true
       migrator:
         condition: service_completed_successfully
-        required: true
       postgres:
         condition: service_healthy
-        required: true
       rabbitmq:
         condition: service_healthy
-        required: true
       redis:
         condition: service_healthy
-        required: true
     env_file: stack.env
     hostname: knightcrawler-addon
-    image: gabisonfire/knightcrawler-addon:2.0.7
+    image: gabisonfire/knightcrawler-addon:2.0.8
     labels:
       logging: promtail
     networks:
@@ -120,21 +107,16 @@ services:
     depends_on:
       metadata:
         condition: service_completed_successfully
-        required: true
       migrator:
         condition: service_completed_successfully
-        required: true
       postgres:
         condition: service_healthy
-        required: true
       rabbitmq:
         condition: service_healthy
-        required: true
       redis:
         condition: service_healthy
-        required: true
     env_file: stack.env
-    image: gabisonfire/knightcrawler-consumer:2.0.7
+    image: gabisonfire/knightcrawler-consumer:2.0.8
     labels:
       logging: promtail
     networks:
@@ -146,21 +128,16 @@ services:
     depends_on:
       metadata:
         condition: service_completed_successfully
-        required: true
       migrator:
         condition: service_completed_successfully
-        required: true
       postgres:
         condition: service_healthy
-        required: true
       rabbitmq:
         condition: service_healthy
-        required: true
       redis:
         condition: service_healthy
-        required: true
     env_file: stack.env
-    image: gabisonfire/knightcrawler-debrid-collector:2.0.7
+    image: gabisonfire/knightcrawler-debrid-collector:2.0.8
     labels:
       logging: promtail
     networks:
@@ -173,9 +150,8 @@ services:
     depends_on:
       migrator:
         condition: service_completed_successfully
-        required: true
     env_file: stack.env
-    image: gabisonfire/knightcrawler-metadata:2.0.7
+    image: gabisonfire/knightcrawler-metadata:2.0.8
     networks:
       - knightcrawler-network
     restart: "no"
@@ -185,9 +161,8 @@ services:
     depends_on:
       postgres:
         condition: service_healthy
-        required: true
     env_file: stack.env
-    image: gabisonfire/knightcrawler-migrator:2.0.7
+    image: gabisonfire/knightcrawler-migrator:2.0.8
     networks:
       - knightcrawler-network
     restart: "no"
@@ -197,21 +172,16 @@ services:
     depends_on:
       metadata:
         condition: service_completed_successfully
-        required: true
       migrator:
         condition: service_completed_successfully
-        required: true
       postgres:
         condition: service_healthy
-        required: true
       rabbitmq:
         condition: service_healthy
-        required: true
       redis:
         condition: service_healthy
-        required: true
     env_file: stack.env
-    image: gabisonfire/knightcrawler-producer:2.0.7
+    image: gabisonfire/knightcrawler-producer:2.0.8
     labels:
       logging: promtail
     networks:
@@ -223,11 +193,10 @@ services:
     depends_on:
       qbittorrent:
         condition: service_healthy
-        required: true
     deploy:
       replicas: ${QBIT_REPLICAS:-0}
     env_file: stack.env
-    image: gabisonfire/knightcrawler-qbit-collector:2.0.7
+    image: gabisonfire/knightcrawler-qbit-collector:2.0.8
     labels:
       logging: promtail
     networks:
@@ -246,9 +215,7 @@ services:
       TORRENTING_PORT: "6881"
       WEBUI_PORT: "8080"
     healthcheck:
-      test:
-        - CMD-SHELL
-        - curl --fail http://localhost:8080
+      test: ["CMD-SHELL", "curl --fail http://localhost:8080"]
       timeout: 10s
       interval: 10s
       retries: 3

diff --git a/deployment/docker/src/components/infrastructure.yaml b/deployment/docker/src/components/infrastructure.yaml
@@ -13,7 +13,7 @@ x-redishealth: &redis-health
   <<: *base-health
 
 x-postgreshealth: &postgresdb-health
-  test: pg_isready
+  test: [ "CMD", "sh", "-c", "pg_isready -h localhost -U $$POSTGRES_USER" ]
   <<: *base-health
 
 x-qbit: &qbit-health

diff --git a/deployment/docker/src/components/knightcrawler.yaml b/deployment/docker/src/components/knightcrawler.yaml
@@ -20,7 +20,7 @@ x-depends: &knightcrawler-app-depends
 
 services:
   metadata:
-    image: gabisonfire/knightcrawler-metadata:2.0.7
+    image: gabisonfire/knightcrawler-metadata:2.0.8
     env_file: ../../.env
     networks:
       - knightcrawler-network
@@ -30,7 +30,7 @@ services:
         condition: service_completed_successfully
 
   migrator:
-    image: gabisonfire/knightcrawler-migrator:2.0.7
+    image: gabisonfire/knightcrawler-migrator:2.0.8
     env_file: ../../.env
     networks:
       - knightcrawler-network
@@ -40,30 +40,30 @@ services:
         condition: service_healthy
 
   addon:
-    image: gabisonfire/knightcrawler-addon:2.0.7
+    image: gabisonfire/knightcrawler-addon:2.0.8
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
     hostname: knightcrawler-addon
     ports:
       - "7000:7000"
 
   consumer:
-    image: gabisonfire/knightcrawler-consumer:2.0.7
+    image: gabisonfire/knightcrawler-consumer:2.0.8
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
 
   debridcollector:
-    image: gabisonfire/knightcrawler-debrid-collector:2.0.7
+    image: gabisonfire/knightcrawler-debrid-collector:2.0.8
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
 
   producer:
-    image: gabisonfire/knightcrawler-producer:2.0.7
+    image: gabisonfire/knightcrawler-producer:2.0.8
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
 
   qbitcollector:
-    image: gabisonfire/knightcrawler-qbit-collector:2.0.7
+    image: gabisonfire/knightcrawler-qbit-collector:2.0.8
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
     depends_on:

diff --git a/src/producer/.dockerignore b/src/producer/.dockerignore
@@ -0,0 +1,2 @@
+**/python/
+.idea/
diff --git a/src/producer/Producer.sln b/src/producer/Producer.sln
@@ -6,6 +6,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedContracts", "..\share
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shared", "shared", "{FF5CA857-51E8-4446-8840-2A1D24ED3952}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "eng", "eng", "{1AE7F597-24C4-4575-B59F-67A625D95C1E}"
+	ProjectSection(SolutionItems) = preProject
+		eng\install-python-reqs.ps1 = eng\install-python-reqs.ps1
+		eng\install-python-reqs.sh = eng\install-python-reqs.sh
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU

diff --git a/src/producer/eng/install-python-reqs.ps1 b/src/producer/eng/install-python-reqs.ps1
@@ -0,0 +1,2 @@
+mkdir -p ../src/python
+pip install --force-reinstall rank-torrent-name==0.1.6 -t ../src/python/
diff --git a/src/producer/eng/install-python-reqs.sh b/src/producer/eng/install-python-reqs.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+mkdir -p ../src/python
+pip install --force-reinstall rank-torrent-name==0.1.6 -t ../src/python/
diff --git a/src/producer/src/.dockerignore b/src/producer/src/.dockerignore
@@ -0,0 +1,2 @@
+**/python/
+.idea/
diff --git a/src/producer/src/Dockerfile b/src/producer/src/Dockerfile
@@ -8,13 +8,21 @@ WORKDIR /src/producer/src
 RUN dotnet restore -a $TARGETARCH
 RUN dotnet publish -c Release --no-restore -o /src/out -a $TARGETARCH
 
-
-FROM mcr.microsoft.com/dotnet/aspnet:8.0-alpine
+FROM mcr.microsoft.com/dotnet/aspnet:8.0-alpine3.19
 
 WORKDIR /app
+
+ENV PYTHONUNBUFFERED=1
+RUN apk add --update --no-cache python3=~3.11.8-r0 py3-pip && ln -sf python3 /usr/bin/python
+
 COPY --from=build /src/out .
+RUN rm -rf /app/python && mkdir -p /app/python
+RUN pip3 install --force-reinstall rank-torrent-name==0.1.6 -t /app/python
 RUN addgroup -S producer && adduser -S -G producer producer
 USER producer
 HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
   CMD pgrep -f dotnet || exit 1
+
+ENV PYTHONNET_PYDLL=/usr/lib/libpython3.11.so.1.0
+
 ENTRYPOINT ["dotnet", "Producer.dll"]
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		mkdir -p ../src/python
		pip install --force-reinstall rank-torrent-name==0.1.6 -t ../src/python/