feat: add playlist sticher service setup

Author: cieslarmichal

apps/api/tsconfig.json

@@ -3,7 +3,7 @@
   "compilerOptions": {
     "outDir": "./dist",
   },
-  "include": ["src/**/*.ts", "../../libs/uuid/src/uuidService.ts"],
+  "include": ["src/**/*.ts", "tests/**/*.ts"],
   "references": [
     {
       "path": "../../libs/amqp/tsconfig.json"

apps/downloader/tsconfig.json

@@ -3,7 +3,7 @@
   "compilerOptions": {
     "outDir": "./dist",
   },
-  "include": ["src/**/*.ts", "tests/**/*.ts", "src/actions/downloadVideoAction"],
+  "include": ["src/**/*.ts", "tests/**/*.ts"],
   "exclude": [],
   "references": [
     {

apps/encoder/tsconfig.json

@@ -3,7 +3,7 @@
   "compilerOptions": {
     "outDir": "./dist",
   },
-  "include": ["src/**/*.ts", "tests/**/*.ts", "src/actions/encodeVideoAction"],
+  "include": ["src/**/*.ts", "tests/**/*.ts"],
   "exclude": [],
   "references": [
     {

apps/encoding-director/tsconfig.json

@@ -3,7 +3,7 @@
   "compilerOptions": {
     "outDir": "./dist",
   },
-  "include": ["src/**/*.ts", "tests/**/*.ts", "src/actions/requestVideoEncodingsAction"],
+  "include": ["src/**/*.ts", "tests/**/*.ts"],
   "exclude": [],
   "references": [
     {

apps/playlist-sticher/.dockerignore

@@ -0,0 +1,46 @@
+# See http://help.github.com/ignore-files/ for more about ignoring files.
+
+# Compiled output
+/dist
+/tmp
+/out-tsc
+/bazel-out
+
+# Node
+/node_modules
+npm-debug.log
+yarn-error.log
+
+# IDEs and editors
+.idea/
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+
+# Visual Studio Code
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+.history/*
+
+# Miscellaneous
+/.angular/cache
+.sass-cache/
+/connect.lock
+/coverage
+/libpeerconnection.log
+testem.log
+/typings
+
+# System files
+.DS_Store
+Thumbs.db
+
+Dockerfile
+.dockerignore
+.git

apps/playlist-sticher/.gitignore

@@ -0,0 +1,18 @@
+.idea
+node_modules
+
+dist
+*.log
+
+.eslintcache
+
+.DS_Store
+
+.vscode
+!.vscode/extensions.json
+
+.tsbuildinfo
+
+.turbo
+
+**/local.json

apps/playlist-sticher/Dockerfile

@@ -0,0 +1,57 @@
+FROM node:20.17.0-alpine AS base
+
+RUN apk update
+
+WORKDIR /app
+
+FROM base AS builder
+
+RUN npm i -g turbo
+
+COPY package*.json ./
+COPY turbo.json ./
+COPY apps/ ./apps
+COPY libs/ ./libs
+COPY tsconfig.json ./
+
+RUN turbo prune --scope="@apps/playlist-sticher" --docker
+
+FROM base AS installer
+
+COPY --from=builder /app/out/json/ .
+COPY --from=builder /app/out/package-lock.json ./package-lock.json
+COPY tsconfig.json ./
+
+RUN npm ci
+
+COPY --from=builder /app/out/full/ .
+
+RUN npm run build --filter="@apps/playlist-sticher"
+
+FROM base AS runner
+
+ENV NODE_CONFIG_DIR=/app/apps/playlist-sticher/config
+
+COPY package*.json ./
+COPY --from=installer /app/apps/playlist-sticher/dist ./apps/playlist-sticher
+COPY --from=installer /app/apps/playlist-sticher/config ./apps/playlist-sticher/config
+COPY --from=installer /app/apps/playlist-sticher/package.json ./apps/playlist-sticher/package.json
+
+COPY --from=installer /app/libs/ ./libs/.temp
+
+RUN for folder in /app/libs/.temp/*; do \
+  package_name=$(basename $folder); \
+  mkdir -p /app/libs/$package_name; \
+  cp -r $folder/dist /app/libs/$package_name/dist; \
+  cp $folder/package.json /app/libs/$package_name/package.json; \
+  done
+
+RUN rm -rf ./libs/.temp
+
+RUN npm ci
+
+RUN mkdir -p /shared && chmod -R 777 /shared
+
+USER node
+
+CMD node apps/playlist-sticher/src/index.js

apps/playlist-sticher/README.md

@@ -0,0 +1 @@
+# Playlist Sticher

apps/playlist-sticher/config/custom-environment-variables.json

@@ -0,0 +1,13 @@
+{
+  "logLevel": "LOG_LEVEL",
+  "aws": {
+    "accessKeyId": "AWS_ACCESS_KEY_ID",
+    "secretAccessKey": "AWS_SECRET_ACCESS_KEY",
+    "s3": {
+      "encodingArtifactsBucket": "AWS_S3_ENCODING_ARTIFACTS_BUCKET"
+    }
+  },
+  "amqp": {
+    "url": "AMQP_URL"
+  }
+}

apps/playlist-sticher/config/default.json

@@ -0,0 +1,17 @@
+{
+  "appName": "playlist-sticher",
+  "logLevel": "debug",
+  "aws": {
+    "accessKeyId": null,
+    "secretAccessKey": null,
+    "region": "eu-central-1",
+    "s3": {
+      "encodingArtifactsBucket": null
+    }
+  },
+  "amqp": {
+    "url": "amqp://test:test@transcoder-rabbitmq:5672",
+    "redeliveryDropThreshold": 3,
+    "messageTtl": 30000
+  }
+}

apps/playlist-sticher/config/development.json

@@ -0,0 +1,2 @@
+{
+}

apps/playlist-sticher/config/production.json

@@ -0,0 +1,3 @@
+{
+  "logLevel": "info"
+}

apps/playlist-sticher/config/test.json

@@ -0,0 +1,13 @@
+{
+  "aws": {
+    "accessKeyId": "test",
+    "secretAccessKey": "test",
+    "endpoint": "http://127.0.0.1:4566",
+    "s3": {
+      "encodingArtifactsBucket": "encoding-artifacts"
+    }
+  },
+  "amqp": {
+    "url": "amqp://test:test@localhost:5672"
+  }
+}

apps/playlist-sticher/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@apps/playlist-sticher",
+  "type": "module",
+  "scripts": {
+    "start": "node src/index.js",
+    "start:dev": "node --watch --inspect --no-warnings=ExperimentalWarning --experimental-specifier-resolution=node --loader ts-node/esm/transpile-only src/index.ts",
+    "build": "tsc --build tsconfig.prod.json",
+    "build:dev": "tsc --build tsconfig.json",
+    "test:unit": "vitest .unit.test.ts --config vitest.unit.config.js",
+    "test:unit:run": "npm run test:unit -- --run",
+    "test:integration": "NODE_ENV=test vitest --config vitest.integration.config.js .integration.test.ts",
+    "test:integration:run": "npm run test:integration -- --run",
+    "lint": "eslint . -c ../../eslint.config.mjs --max-warnings 0",
+    "lint:fix": "npm run lint -- --fix"
+  },
+  "dependencies": {
+    "@libs/amqp": "*",
+    "@libs/contracts": "*",
+    "@libs/errors": "*",
+    "@libs/logger": "*",
+    "@libs/s3": "*"
+  },
+  "volta": {
+    "node": "20.17.0",
+    "npm": "10.8.3"
+  },
+  "engines": {
+    "node": "20.17.0"
+  }
+}

apps/playlist-sticher/src/actions/uploadVideoArtifactsAction/createMasterPlaylistAction.ts

@@ -0,0 +1,35 @@
+import { type Logger } from '@libs/logger';
+import { type Config } from '../../config.js';
+import { type EncodingId } from '@libs/contracts';
+import { type S3Service } from '@libs/s3';
+
+export interface CreateMasterPlaylstActionPayload {
+  readonly videoId: string;
+  readonly encodingId: EncodingId;
+}
+
+export class CreateMasterPlaylistAction {
+  public constructor(
+    private readonly s3Service: S3Service,
+    private readonly logger: Logger,
+    private readonly config: Config,
+  ) {}
+
+  public async execute(payload: CreateMasterPlaylstActionPayload): Promise<void> {
+    const { videoId, encodingId } = payload;
+
+    this.logger.debug({
+      message: 'Creating master HLS playlist...',
+      videoId,
+      encodingId,
+    });
+
+    console.log({ s3: this.s3Service, config: this.config });
+
+    this.logger.info({
+      message: 'HLS master playlist created.',
+      videoId,
+      encodingId,
+    });
+  }
+}

apps/playlist-sticher/src/api/messageConsumers/videoArtifactsUploadedConsumer.ts

@@ -0,0 +1,17 @@
+import { type ConsumePayload, type MessageConsumer } from '@libs/amqp';
+import { Value } from '@sinclair/typebox/value';
+import { videoArtifactsUploadedMessageSchema } from '@libs/contracts';
+import { type CreateMasterPlaylistAction } from '../../actions/uploadVideoArtifactsAction/createMasterPlaylistAction.js';
+
+export class VideoArtifactsUploadedMessageConsumer implements MessageConsumer {
+  public constructor(private readonly createMasterPlaylistAction: CreateMasterPlaylistAction) {}
+
+  public async consume(payload: ConsumePayload): Promise<void> {
+    const { videoId, encodingId } = Value.Decode(videoArtifactsUploadedMessageSchema, payload.message);
+
+    await this.createMasterPlaylistAction.execute({
+      videoId,
+      encodingId,
+    });
+  }
+}

apps/playlist-sticher/src/application.ts

@@ -0,0 +1,69 @@
+import { type AmqpChannel, type AmqpConnection, AmqpProvisioner, MessageConsumerExecutor } from '@libs/amqp';
+import { type Logger, LoggerFactory } from '@libs/logger';
+
+import { type Config, ConfigFactory } from './config.js';
+import { exchangeName, queueNames, routingKeys } from '@libs/contracts';
+import { VideoArtifactsUploadedMessageConsumer } from './api/messageConsumers/videoArtifactsUploadedConsumer.js';
+import { S3Service, S3ClientFactory } from '@libs/s3';
+import { CreateMasterPlaylistAction } from './actions/uploadVideoArtifactsAction/createMasterPlaylistAction.js';
+
+export class Application {
+  private readonly config: Config;
+  private readonly logger: Logger;
+  private amqpConnection: AmqpConnection | undefined;
+  private amqpChannel: AmqpChannel | undefined;
+  private readonly amqpProvisioner: AmqpProvisioner;
+
+  public constructor() {
+    this.config = ConfigFactory.create();
+
+    this.logger = LoggerFactory.create({
+      appName: this.config.appName,
+      logLevel: this.config.logLevel,
+    });
+
+    this.amqpProvisioner = new AmqpProvisioner(this.logger);
+  }
+
+  public async start(): Promise<void> {
+    await this.setupAmqp();
+
+    const s3Service = new S3Service(S3ClientFactory.create(this.config.aws));
+
+    const createMasterPlaylistAction = new CreateMasterPlaylistAction(s3Service, this.logger, this.config);
+
+    const messageConsumer = new VideoArtifactsUploadedMessageConsumer(createMasterPlaylistAction);
+
+    const messageConsumerExecutor = new MessageConsumerExecutor(
+      messageConsumer,
+      this.amqpChannel as AmqpChannel,
+      this.logger,
+      queueNames.uploadedArtifacts,
+      this.config.amqp.redeliveryDropThreshold,
+    );
+
+    await messageConsumerExecutor.startConsuming();
+  }
+
+  public async stop(): Promise<void> {
+    await this.amqpConnection?.close();
+  }
+
+  private async setupAmqp(): Promise<void> {
+    this.amqpConnection = await this.amqpProvisioner.createConnection({
+      url: this.config.amqp.url,
+    });
+
+    this.amqpChannel = await this.amqpProvisioner.createChannel({
+      connection: this.amqpConnection,
+    });
+
+    await this.amqpProvisioner.createQueue({
+      channel: this.amqpChannel,
+      exchangeName,
+      queueName: queueNames.uploadedArtifacts,
+      pattern: routingKeys.videoArtifactsUploaded,
+      dlqMessageTtl: this.config.amqp.messageTtl,
+    });
+  }
+}