Add support for saving video files to object storage

.github/workflows/test.yml

@@ -31,6 +31,11 @@ jobs:
         ports:
           - 10389:10389
 
+      s3ninja:
+        image: scireum/s3-ninja
+        ports:
+          - 9444:9000
+
     strategy:
       fail-fast: false
       matrix:

config/default.yaml

@@ -97,20 +97,33 @@ storage:
 
 object_storage:
   enabled: false
-  # Maximum amount of MiB to upload in one request to object storage
-  max_upload_part: 2048
-  # Will always use https with default URL generation (see below)
+
+  # Without protocol, will default to HTTPS
   endpoint: 's3.amazonaws.com'
+
+  region: 'us-east-1'
+
+  credentials:
+    access_key_id: 'access-key'
+    secret_access_key: 'secret-access-key'
+
+  # Maximum amount to upload in one request to object storage
+  max_upload_part: 2MB
+
   streaming_playlists:
-    bucket_name: ''
+    bucket_name: 'streaming-playlists'
+
     # Allows setting all buckets to the same value but with a different prefix
-    prefix: ''
-    # Base url for object URL generation, path in bucket is appended to this url
-    base_url: ''
+    prefix: '' # Example: 'streaming-playlists:'
+
+    # Base url for object URL generation, scheme and host will be replaced by this URL
+    # Useful when you want to use a CDN/external proxy
+    base_url: '' # Example: 'https://mirror.example.com'
+
   # Same settings but for webtorrent videos
   videos:
-    bucket_name: ''
-    prefix: '' 
+    bucket_name: 'videos'
+    prefix: ''
     base_url: ''
 
 log:

scripts/ci.sh

@@ -89,9 +89,10 @@ elif [ "$1" = "api-4" ]; then
 
     moderationFiles=$(findTestFiles ./dist/server/tests/api/moderation)
     redundancyFiles=$(findTestFiles ./dist/server/tests/api/redundancy)
+    objectStorageFiles=$(findTestFiles ./dist/server/tests/api/object-storage)
     activitypubFiles=$(findTestFiles ./dist/server/tests/api/activitypub)
 
-    MOCHA_PARALLEL=true TS_NODE_FILES=true runTest "$1" 2 $moderationFiles $redundancyFiles $activitypubFiles
+    MOCHA_PARALLEL=true TS_NODE_FILES=true runTest "$1" 2 $moderationFiles $redundancyFiles $activitypubFiles $objectStorageFiles
 elif [ "$1" = "external-plugins" ]; then
     npm run build:server
 

scripts/optimize-old-videos.ts

@@ -1,16 +1,18 @@
-import { registerTSPaths } from '../server/helpers/register-ts-paths'
-registerTSPaths()
-
-import { getDurationFromVideoFile, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../server/helpers/ffprobe-utils'
-import { VideoModel } from '../server/models/video/video'
-import { optimizeOriginalVideofile } from '../server/lib/transcoding/video-transcoding'
-import { initDatabaseModels } from '../server/initializers/database'
-import { basename, dirname } from 'path'
 import { copy, move, remove } from 'fs-extra'
+import { basename, dirname } from 'path'
 import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
-import { getMaxBitrate } from '@shared/core-utils'
+import { CONFIG } from '@server/initializers/config'
+import { processMoveToObjectStorage } from '@server/lib/job-queue/handlers/move-to-object-storage'
 import { getVideoFilePath, getVideoFilePathMakeAvailable } from '@server/lib/video-paths'
-import { moveWebTorrentFiles } from '@server/lib/job-queue/handlers/move-to-object-storage'
+import { getMaxBitrate } from '@shared/core-utils'
+import { MoveObjectStoragePayload } from '@shared/models'
+import { getDurationFromVideoFile, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../server/helpers/ffprobe-utils'
+import { registerTSPaths } from '../server/helpers/register-ts-paths'
+import { initDatabaseModels } from '../server/initializers/database'
+import { optimizeOriginalVideofile } from '../server/lib/transcoding/video-transcoding'
+import { VideoModel } from '../server/models/video/video'
+
+registerTSPaths()
 
 run()
   .then(() => process.exit(0))
@@ -68,7 +70,6 @@ async function run () {
 
         if (originalDuration === newDuration) {
           console.log('Finished optimizing %s', basename(currentFilePath))
-          await moveWebTorrentFiles(video, file.id)
           await remove(backupFile)
           continue
         }
@@ -79,6 +80,10 @@ async function run () {
         await file.save()
       }
     }
+
+    if (CONFIG.OBJECT_STORAGE.ENABLED === true) {
+      await processMoveToObjectStorage({ data: { videoUUID: video.uuid } as MoveObjectStoragePayload } as any)
+    }
   }
 
   console.log('Finished optimizing videos')

server/controllers/api/videos/upload.ts

@@ -6,9 +6,10 @@ import { uuidToShort } from '@server/helpers/uuid'
 import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
 import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
 import {
-  addMoveToObjectStorageJob as addMoveToObjectStorageJobIfNeeded,
+  addMoveToObjectStorageJob,
   addOptimizeOrMergeAudioJob,
   buildLocalVideoFromReq,
+  buildNextVideoState,
   buildVideoThumbnailsFromReq,
   setVideoTags
 } from '@server/lib/video'
@@ -145,10 +146,7 @@ async function addVideo (options: {
 
   const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id)
 
-  videoData.state = CONFIG.TRANSCODING.ENABLED
-    ? VideoState.TO_TRANSCODE
-    : VideoState.PUBLISHED
-
+  videoData.state = buildNextVideoState()
   videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware
 
   const video = new VideoModel(videoData) as MVideoFullLight
@@ -216,14 +214,13 @@ async function addVideo (options: {
 
   createTorrentFederate(video, videoFile)
     .then(() => {
-      if (video.state !== VideoState.TO_TRANSCODE) {
-        // Video will be published before move is complete which may cause some video connections to drop
-        // But it's recommended to enable transcoding anyway, so this is the tradeoff
-        addMoveToObjectStorageJobIfNeeded(video, videoFile)
-        return
+      if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
+        return addMoveToObjectStorageJob(video)
       }
 
-      return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user)
+      if (video.state === VideoState.TO_TRANSCODE) {
+        return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user)
+      }
     })
     .catch(err => logger.error('Cannot add optimize/merge audio job for %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) }))
 

server/controllers/download.ts

@@ -5,7 +5,7 @@ import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache
 import { Hooks } from '@server/lib/plugins/hooks'
 import { getVideoFilePath } from '@server/lib/video-paths'
 import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
-import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models'
+import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models'
 import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants'
 import { asyncMiddleware, videosDownloadValidator } from '../middlewares'
 
@@ -81,6 +81,10 @@ async function downloadVideoFile (req: express.Request, res: express.Response) {
 
   if (!checkAllowResult(res, allowParameters, allowedResult)) return
 
+  if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
+    return res.redirect(videoFile.getObjectStorageUrl())
+  }
+
   return res.download(getVideoFilePath(video, videoFile), `${video.name}-${videoFile.resolution}p${videoFile.extname}`)
 }
 
@@ -107,6 +111,10 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response
 
   if (!checkAllowResult(res, allowParameters, allowedResult)) return
 
+  if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
+    return res.redirect(videoFile.getObjectStorageUrl())
+  }
+
   const filename = `${video.name}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}`
   return res.download(getVideoFilePath(streamingPlaylist, videoFile), filename)
 }

server/initializers/checker-after-init.ts

@@ -155,14 +155,19 @@ function checkConfig () {
 
   // Object storage
   if (CONFIG.OBJECT_STORAGE.ENABLED === true) {
+
     if (!CONFIG.OBJECT_STORAGE.VIDEOS.BUCKET_NAME) {
       return 'videos_bucket should be set when object storage support is enabled.'
     }
+
     if (!CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME) {
       return 'streaming_playlists_bucket should be set when object storage support is enabled.'
     }
-    if (CONFIG.OBJECT_STORAGE.VIDEOS.BUCKET_NAME === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME &&
-        CONFIG.OBJECT_STORAGE.VIDEOS.PREFIX === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.PREFIX) {
+
+    if (
+      CONFIG.OBJECT_STORAGE.VIDEOS.BUCKET_NAME === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME &&
+      CONFIG.OBJECT_STORAGE.VIDEOS.PREFIX === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.PREFIX
+    ) {
       if (CONFIG.OBJECT_STORAGE.VIDEOS.PREFIX === '') {
         return 'Object storage bucket prefixes should be set when the same bucket is used for both types of video.'
       } else {

server/initializers/config.ts

@@ -75,10 +75,13 @@ const CONFIG = {
   },
   OBJECT_STORAGE: {
     ENABLED: config.get<boolean>('object_storage.enabled'),
-    MAX_UPLOAD_PART: config.get<number>('object_storage.max_upload_part'),
-    ENDPOINT: new URL(/^https?:\/\//i.test(config.get<string>('object_storage.endpoint'))
-      ? config.get<string>('object_storage.endpoint')
-      : 'https://' + config.get<string>('object_storage.endpoint')),
+    MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')),
+    ENDPOINT: config.get<string>('object_storage.endpoint'),
+    REGION: config.get<string>('object_storage.region'),
+    CREDENTIALS: {
+      ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'),
+      SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key')
+    },
     VIDEOS: {
       BUCKET_NAME: config.get<string>('object_storage.videos.bucket_name'),
       PREFIX: config.get<string>('object_storage.videos.prefix'),

server/initializers/constants.ts

@@ -415,7 +415,8 @@ const VIDEO_STATES: { [ id in VideoState ]: string } = {
   [VideoState.TO_TRANSCODE]: 'To transcode',
   [VideoState.TO_IMPORT]: 'To import',
   [VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream',
-  [VideoState.LIVE_ENDED]: 'Livestream ended'
+  [VideoState.LIVE_ENDED]: 'Livestream ended',
+  [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE]: 'To move to an external storage'
 }
 
 const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = {

server/initializers/migrations/0660-object-storage.ts

@@ -1,5 +1,5 @@
-import { VideoStorageType } from '@server/types/models'
 import * as Sequelize from 'sequelize'
+import { VideoStorage } from '@shared/models'
 
 async function up (utils: {
   transaction: Sequelize.Transaction
@@ -12,6 +12,7 @@ async function up (utils: {
     CREATE TABLE IF NOT EXISTS "videoJobInfo" (
       "id" serial,
       "pendingMove" INTEGER NOT NULL,
+      "pendingTranscoding" INTEGER NOT NULL,
       "videoId" serial UNIQUE NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
       "createdAt" timestamp WITH time zone NOT NULL,
       "updatedAt" timestamp WITH time zone NOT NULL,
@@ -27,7 +28,7 @@ async function up (utils: {
   }
   {
     await utils.sequelize.query(
-      `UPDATE "videoFile" SET "storage" = ${VideoStorageType.LOCAL}`
+      `UPDATE "videoFile" SET "storage" = ${VideoStorage.LOCAL}`
     )
   }
   {
@@ -39,7 +40,7 @@ async function up (utils: {
   }
   {
     await utils.sequelize.query(
-      `UPDATE "videoStreamingPlaylist" SET "storage" = ${VideoStorageType.LOCAL}`
+      `UPDATE "videoStreamingPlaylist" SET "storage" = ${VideoStorage.LOCAL}`
     )
   }
   {