Add support for saving video files to object storage

.github/workflows/test.yml

@@ -31,6 +31,11 @@ jobs:
         ports:
           - 10389:10389
 
+      s3ninja:
+        image: scireum/s3-ninja
+        ports:
+          - 9444:9000
+
     strategy:
       fail-fast: false
       matrix:
@@ -40,6 +45,7 @@ jobs:
       PGUSER: peertube
       PGHOST: localhost
       NODE_PENDING_JOB_WAIT: 250
+      ENABLE_OBJECT_STORAGE_TESTS: true
 
     steps:
       - uses: actions/checkout@v2

client/src/app/+admin/system/jobs/jobs.component.ts

@@ -36,7 +36,8 @@ export class JobsComponent extends RestTable implements OnInit {
     'video-live-ending',
     'video-redundancy',
     'video-transcoding',
-    'videos-views'
+    'videos-views',
+    'move-to-object-storage'
   ]
 
   jobs: Job[] = []

client/src/app/+videos/+video-watch/shared/information/video-alert.component.html

@@ -6,6 +6,10 @@
   The video is being transcoded, it may not work properly.
 </div>
 
+<div i18n class="alert alert-warning" *ngIf="isVideoToMoveToExternalStorage()">
+  The video is being moved to an external server, it may not work properly.
+</div>
+
 <div i18n class="alert alert-info" *ngIf="hasVideoScheduledPublication()">
   This video will be published on {{ video.scheduledUpdate.updateAt | date: 'full' }}.
 </div>

client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts

@@ -18,6 +18,10 @@ export class VideoAlertComponent {
     return this.video && this.video.state.id === VideoState.TO_IMPORT
   }
 
+  isVideoToMoveToExternalStorage () {
+    return this.video && this.video.state.id === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE
+  }
+
   hasVideoScheduledPublication () {
     return this.video && this.video.scheduledUpdate !== undefined
   }

config/default.yaml

@@ -95,6 +95,39 @@ storage:
   # If not, peertube will fallback to the default fil
   client_overrides: 'storage/client-overrides/'
 
+object_storage:
+  enabled: false
+
+  # Without protocol, will default to HTTPS
+  endpoint: '' # 's3.amazonaws.com' or 's3.fr-par.scw.cloud' for example
+
+  region: 'us-east-1'
+
+  credentials:
+    # You can also use AWS_ACCESS_KEY_ID env variable
+    access_key_id: ''
+    # You can also use AWS_SECRET_ACCESS_KEY env variable
+    secret_access_key: ''
+
+  # Maximum amount to upload in one request to object storage
+  max_upload_part: 2GB
+
+  streaming_playlists:
+    bucket_name: 'streaming-playlists'
+
+    # Allows setting all buckets to the same value but with a different prefix
+    prefix: '' # Example: 'streaming-playlists:'
+
+    # Base url for object URL generation, scheme and host will be replaced by this URL
+    # Useful when you want to use a CDN/external proxy
+    base_url: '' # Example: 'https://mirror.example.com'
+
+  # Same settings but for webtorrent videos
+  videos:
+    bucket_name: 'videos'
+    prefix: ''
+    base_url: ''
+
 log:
   level: 'info' # 'debug' | 'info' | 'warn' | 'error'
   rotation:

config/production.yaml.example

@@ -93,6 +93,39 @@ storage:
   # If not, peertube will fallback to the default file
   client_overrides: '/var/www/peertube/storage/client-overrides/'
 
+object_storage:
+  enabled: false
+
+  # Without protocol, will default to HTTPS
+  endpoint: '' # 's3.amazonaws.com' or 's3.fr-par.scw.cloud' for example
+
+  region: 'us-east-1'
+
+  credentials:
+    # You can also use AWS_ACCESS_KEY_ID env variable
+    access_key_id: ''
+    # You can also use AWS_SECRET_ACCESS_KEY env variable
+    secret_access_key: ''
+
+  # Maximum amount to upload in one request to object storage
+  max_upload_part: 2GB
+
+  streaming_playlists:
+    bucket_name: 'streaming-playlists'
+
+    # Allows setting all buckets to the same value but with a different prefix
+    prefix: '' # Example: 'streaming-playlists:'
+
+    # Base url for object URL generation, scheme and host will be replaced by this URL
+    # Useful when you want to use a CDN/external proxy
+    base_url: '' # Example: 'https://mirror.example.com'
+
+  # Same settings but for webtorrent videos
+  videos:
+    bucket_name: 'videos'
+    prefix: ''
+    base_url: ''
+
 log:
   level: 'info' # 'debug' | 'info' | 'warn' | 'error'
   rotation:

package.json

@@ -73,6 +73,7 @@
     "swagger-cli": "swagger-cli"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.23.0",
     "@uploadx/core": "^4.4.0",
     "async": "^3.0.1",
     "async-lru": "^1.1.1",

scripts/ci.sh

@@ -89,9 +89,10 @@ elif [ "$1" = "api-4" ]; then
 
     moderationFiles=$(findTestFiles ./dist/server/tests/api/moderation)
     redundancyFiles=$(findTestFiles ./dist/server/tests/api/redundancy)
+    objectStorageFiles=$(findTestFiles ./dist/server/tests/api/object-storage)
     activitypubFiles=$(findTestFiles ./dist/server/tests/api/activitypub)
 
-    MOCHA_PARALLEL=true TS_NODE_FILES=true runTest "$1" 2 $moderationFiles $redundancyFiles $activitypubFiles
+    MOCHA_PARALLEL=true TS_NODE_FILES=true runTest "$1" 2 $moderationFiles $redundancyFiles $activitypubFiles $objectStorageFiles
 elif [ "$1" = "external-plugins" ]; then
     npm run build:server
 

scripts/create-transcoding-job.ts

@@ -6,9 +6,10 @@ import { VideoModel } from '../server/models/video/video'
 import { initDatabaseModels } from '../server/initializers/database'
 import { JobQueue } from '../server/lib/job-queue'
 import { computeResolutionsToTranscode } from '@server/helpers/ffprobe-utils'
-import { VideoTranscodingPayload } from '@shared/models'
+import { VideoState, VideoTranscodingPayload } from '@shared/models'
 import { CONFIG } from '@server/initializers/config'
 import { isUUIDValid } from '@server/helpers/custom-validators/misc'
+import { addTranscodingJob } from '@server/lib/video'
 
 program
   .option('-v, --video [videoUUID]', 'Video UUID')
@@ -47,7 +48,7 @@ async function run () {
   if (!video) throw new Error('Video not found.')
 
   const dataInput: VideoTranscodingPayload[] = []
-  const { resolution } = await video.getMaxQualityResolution()
+  const resolution = video.getMaxQualityFile().resolution
 
   // Generate HLS files
   if (options.generateHls || CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
@@ -62,6 +63,7 @@ async function run () {
         resolution,
         isPortraitMode: false,
         copyCodecs: false,
+        isNewVideo: false,
         isMaxQuality: false
       })
     }
@@ -87,10 +89,13 @@ async function run () {
     }
   }
 
-  await JobQueue.Instance.init()
+  JobQueue.Instance.init()
+
+  video.state = VideoState.TO_TRANSCODE
+  await video.save()
 
   for (const d of dataInput) {
-    await JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: d })
+    await addTranscodingJob(d, {})
     console.log('Transcoding job for video %s created.', video.uuid)
   }
 }

scripts/optimize-old-videos.ts

@@ -1,15 +1,18 @@
 import { registerTSPaths } from '../server/helpers/register-ts-paths'
 registerTSPaths()
 
-import { getDurationFromVideoFile, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../server/helpers/ffprobe-utils'
-import { VideoModel } from '../server/models/video/video'
-import { optimizeOriginalVideofile } from '../server/lib/transcoding/video-transcoding'
-import { initDatabaseModels } from '../server/initializers/database'
-import { basename, dirname } from 'path'
 import { copy, move, remove } from 'fs-extra'
+import { basename, dirname } from 'path'
 import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
-import { getVideoFilePath } from '@server/lib/video-paths'
+import { CONFIG } from '@server/initializers/config'
+import { processMoveToObjectStorage } from '@server/lib/job-queue/handlers/move-to-object-storage'
+import { VideoPathManager } from '@server/lib/video-path-manager'
 import { getMaxBitrate } from '@shared/core-utils'
+import { MoveObjectStoragePayload } from '@shared/models'
+import { getDurationFromVideoFile, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../server/helpers/ffprobe-utils'
+import { initDatabaseModels } from '../server/initializers/database'
+import { optimizeOriginalVideofile } from '../server/lib/transcoding/video-transcoding'
+import { VideoModel } from '../server/models/video/video'
 
 run()
   .then(() => process.exit(0))
@@ -39,43 +42,49 @@ async function run () {
     currentVideoId = video.id
 
     for (const file of video.VideoFiles) {
-      currentFilePath = getVideoFilePath(video, file)
-
-      const [ videoBitrate, fps, dataResolution ] = await Promise.all([
-        getVideoFileBitrate(currentFilePath),
-        getVideoFileFPS(currentFilePath),
-        getVideoFileResolution(currentFilePath)
-      ])
-
-      const maxBitrate = getMaxBitrate({ ...dataResolution, fps })
-      const isMaxBitrateExceeded = videoBitrate > maxBitrate
-      if (isMaxBitrateExceeded) {
-        console.log(
-          'Optimizing video file %s with bitrate %s kbps (max: %s kbps)',
-          basename(currentFilePath), videoBitrate / 1000, maxBitrate / 1000
-        )
-
-        const backupFile = `${currentFilePath}_backup`
-        await copy(currentFilePath, backupFile)
-
-        await optimizeOriginalVideofile(video, file)
-        // Update file path, the video filename changed
-        currentFilePath = getVideoFilePath(video, file)
-
-        const originalDuration = await getDurationFromVideoFile(backupFile)
-        const newDuration = await getDurationFromVideoFile(currentFilePath)
-
-        if (originalDuration === newDuration) {
-          console.log('Finished optimizing %s', basename(currentFilePath))
-          await remove(backupFile)
-          continue
+      await VideoPathManager.Instance.makeAvailableVideoFile(video, file, async path => {
+        currentFilePath = path
+
+        const [ videoBitrate, fps, dataResolution ] = await Promise.all([
+          getVideoFileBitrate(currentFilePath),
+          getVideoFileFPS(currentFilePath),
+          getVideoFileResolution(currentFilePath)
+        ])
+
+        const maxBitrate = getMaxBitrate({ ...dataResolution, fps })
+        const isMaxBitrateExceeded = videoBitrate > maxBitrate
+        if (isMaxBitrateExceeded) {
+          console.log(
+            'Optimizing video file %s with bitrate %s kbps (max: %s kbps)',
+            basename(currentFilePath), videoBitrate / 1000, maxBitrate / 1000
+          )
+
+          const backupFile = `${currentFilePath}_backup`
+          await copy(currentFilePath, backupFile)
+
+          await optimizeOriginalVideofile(video, file)
+          // Update file path, the video filename changed
+          currentFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
+
+          const originalDuration = await getDurationFromVideoFile(backupFile)
+          const newDuration = await getDurationFromVideoFile(currentFilePath)
+
+          if (originalDuration === newDuration) {
+            console.log('Finished optimizing %s', basename(currentFilePath))
+            await remove(backupFile)
+            return
+          }
+
+          console.log('Failed to optimize %s, restoring original', basename(currentFilePath))
+          await move(backupFile, currentFilePath, { overwrite: true })
+          await createTorrentAndSetInfoHash(video, file)
+          await file.save()
         }
+      })
+    }
 
-        console.log('Failed to optimize %s, restoring original', basename(currentFilePath))
-        await move(backupFile, currentFilePath, { overwrite: true })
-        await createTorrentAndSetInfoHash(video, file)
-        await file.save()
-      }
+    if (CONFIG.OBJECT_STORAGE.ENABLED === true) {
+      await processMoveToObjectStorage({ data: { videoUUID: video.uuid } as MoveObjectStoragePayload } as any)
     }
   }
 

server/controllers/api/videos/upload.ts

@@ -1,12 +1,21 @@
 import * as express from 'express'
 import { move } from 'fs-extra'
+import { basename } from 'path'
 import { getLowercaseExtension } from '@server/helpers/core-utils'
 import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload'
 import { uuidToShort } from '@server/helpers/uuid'
 import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
 import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
-import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video'
-import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths'
+import { generateWebTorrentVideoFilename } from '@server/lib/paths'
+import {
+  addMoveToObjectStorageJob,
+  addOptimizeOrMergeAudioJob,
+  buildLocalVideoFromReq,
+  buildVideoThumbnailsFromReq,
+  setVideoTags
+} from '@server/lib/video'
+import { VideoPathManager } from '@server/lib/video-path-manager'
+import { buildNextVideoState } from '@server/lib/video-state'
 import { openapiOperationDoc } from '@server/middlewares/doc'
 import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
 import { uploadx } from '@uploadx/core'
@@ -139,23 +148,20 @@ async function addVideo (options: {
 
   const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id)
 
-  videoData.state = CONFIG.TRANSCODING.ENABLED
-    ? VideoState.TO_TRANSCODE
-    : VideoState.PUBLISHED
-
+  videoData.state = buildNextVideoState()
   videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware
 
   const video = new VideoModel(videoData) as MVideoFullLight
   video.VideoChannel = videoChannel
   video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object
 
-  const videoFile = await buildNewFile(video, videoPhysicalFile)
+  const videoFile = await buildNewFile(videoPhysicalFile)
 
   // Move physical file
-  const destination = getVideoFilePath(video, videoFile)
+  const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile)
   await move(videoPhysicalFile.path, destination)
   // This is important in case if there is another attempt in the retry process
-  videoPhysicalFile.filename = getVideoFilePath(video, videoFile)
+  videoPhysicalFile.filename = basename(destination)
   videoPhysicalFile.path = destination
 
   const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
@@ -210,9 +216,13 @@ async function addVideo (options: {
 
   createTorrentFederate(video, videoFile)
     .then(() => {
-      if (video.state !== VideoState.TO_TRANSCODE) return
+      if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
+        return addMoveToObjectStorageJob(video)
+      }
 
-      return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user)
+      if (video.state === VideoState.TO_TRANSCODE) {
+        return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user)
+      }
     })
     .catch(err => logger.error('Cannot add optimize/merge audio job for %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) }))
 
@@ -227,7 +237,7 @@ async function addVideo (options: {
   })
 }
 
-async function buildNewFile (video: MVideo, videoPhysicalFile: express.VideoUploadFile) {
+async function buildNewFile (videoPhysicalFile: express.VideoUploadFile) {
   const videoFile = new VideoFileModel({
     extname: getLowercaseExtension(videoPhysicalFile.filename),
     size: videoPhysicalFile.size,