Add video path manager

.github/workflows/test.yml

@@ -45,6 +45,7 @@ jobs:
       PGUSER: peertube
       PGHOST: localhost
       NODE_PENDING_JOB_WAIT: 250
+      ENABLE_OBJECT_STORAGE_TESTS: true
 
     steps:
       - uses: actions/checkout@v2

config/default.yaml

@@ -99,16 +99,18 @@ object_storage:
   enabled: false
 
   # Without protocol, will default to HTTPS
-  endpoint: 's3.amazonaws.com'
+  endpoint: '' # 's3.amazonaws.com' or 's3.fr-par.scw.cloud' for example
 
   region: 'us-east-1'
 
   credentials:
-    access_key_id: 'access-key'
-    secret_access_key: 'secret-access-key'
+    # You can also use AWS_ACCESS_KEY_ID env variable
+    access_key_id: ''
+    # You can also use AWS_SECRET_ACCESS_KEY env variable
+    secret_access_key: ''
 
   # Maximum amount to upload in one request to object storage
-  max_upload_part: 2MB
+  max_upload_part: 2GB
 
   streaming_playlists:
     bucket_name: 'streaming-playlists'

config/production.yaml.example

@@ -93,6 +93,39 @@ storage:
   # If not, peertube will fallback to the default file
   client_overrides: '/var/www/peertube/storage/client-overrides/'
 
+object_storage:
+  enabled: false
+
+  # Without protocol, will default to HTTPS
+  endpoint: '' # 's3.amazonaws.com' or 's3.fr-par.scw.cloud' for example
+
+  region: 'us-east-1'
+
+  credentials:
+    # You can also use AWS_ACCESS_KEY_ID env variable
+    access_key_id: ''
+    # You can also use AWS_SECRET_ACCESS_KEY env variable
+    secret_access_key: ''
+
+  # Maximum amount to upload in one request to object storage
+  max_upload_part: 2GB
+
+  streaming_playlists:
+    bucket_name: 'streaming-playlists'
+
+    # Allows setting all buckets to the same value but with a different prefix
+    prefix: '' # Example: 'streaming-playlists:'
+
+    # Base url for object URL generation, scheme and host will be replaced by this URL
+    # Useful when you want to use a CDN/external proxy
+    base_url: '' # Example: 'https://mirror.example.com'
+
+  # Same settings but for webtorrent videos
+  videos:
+    bucket_name: 'videos'
+    prefix: ''
+    base_url: ''
+
 log:
   level: 'info' # 'debug' | 'info' | 'warn' | 'error'
   rotation:

scripts/create-transcoding-job.ts

@@ -6,7 +6,7 @@ import { VideoModel } from '../server/models/video/video'
 import { initDatabaseModels } from '../server/initializers/database'
 import { JobQueue } from '../server/lib/job-queue'
 import { computeResolutionsToTranscode } from '@server/helpers/ffprobe-utils'
-import { VideoTranscodingPayload } from '@shared/models'
+import { VideoState, VideoTranscodingPayload } from '@shared/models'
 import { CONFIG } from '@server/initializers/config'
 import { isUUIDValid } from '@server/helpers/custom-validators/misc'
 import { addTranscodingJob } from '@server/lib/video'
@@ -48,7 +48,7 @@ async function run () {
   if (!video) throw new Error('Video not found.')
 
   const dataInput: VideoTranscodingPayload[] = []
-  const { resolution } = await video.getMaxQualityResolution()
+  const resolution = video.getMaxQualityFile().resolution
 
   // Generate HLS files
   if (options.generateHls || CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
@@ -63,6 +63,7 @@ async function run () {
         resolution,
         isPortraitMode: false,
         copyCodecs: false,
+        isNewVideo: false,
         isMaxQuality: false
       })
     }
@@ -88,7 +89,10 @@ async function run () {
     }
   }
 
-  await JobQueue.Instance.init()
+  JobQueue.Instance.init()
+
+  video.state = VideoState.TO_TRANSCODE
+  await video.save()
 
   for (const d of dataInput) {
     await addTranscodingJob(d, {})

scripts/optimize-old-videos.ts

@@ -1,19 +1,19 @@
+import { registerTSPaths } from '../server/helpers/register-ts-paths'
+registerTSPaths()
+
 import { copy, move, remove } from 'fs-extra'
 import { basename, dirname } from 'path'
 import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
 import { CONFIG } from '@server/initializers/config'
 import { processMoveToObjectStorage } from '@server/lib/job-queue/handlers/move-to-object-storage'
-import { getVideoFilePath, getVideoFilePathMakeAvailable } from '@server/lib/video-paths'
+import { VideoPathManager } from '@server/lib/video-path-manager'
 import { getMaxBitrate } from '@shared/core-utils'
 import { MoveObjectStoragePayload } from '@shared/models'
 import { getDurationFromVideoFile, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../server/helpers/ffprobe-utils'
-import { registerTSPaths } from '../server/helpers/register-ts-paths'
 import { initDatabaseModels } from '../server/initializers/database'
 import { optimizeOriginalVideofile } from '../server/lib/transcoding/video-transcoding'
 import { VideoModel } from '../server/models/video/video'
 
-registerTSPaths()
-
 run()
   .then(() => process.exit(0))
   .catch(err => {
@@ -42,43 +42,45 @@ async function run () {
     currentVideoId = video.id
 
     for (const file of video.VideoFiles) {
-      currentFilePath = await getVideoFilePathMakeAvailable(video, file)
-
-      const [ videoBitrate, fps, dataResolution ] = await Promise.all([
-        getVideoFileBitrate(currentFilePath),
-        getVideoFileFPS(currentFilePath),
-        getVideoFileResolution(currentFilePath)
-      ])
-
-      const maxBitrate = getMaxBitrate({ ...dataResolution, fps })
-      const isMaxBitrateExceeded = videoBitrate > maxBitrate
-      if (isMaxBitrateExceeded) {
-        console.log(
-          'Optimizing video file %s with bitrate %s kbps (max: %s kbps)',
-          basename(currentFilePath), videoBitrate / 1000, maxBitrate / 1000
-        )
-
-        const backupFile = `${currentFilePath}_backup`
-        await copy(currentFilePath, backupFile)
-
-        await optimizeOriginalVideofile(video, file)
-        // Update file path, the video filename changed
-        currentFilePath = getVideoFilePath(video, file)
-
-        const originalDuration = await getDurationFromVideoFile(backupFile)
-        const newDuration = await getDurationFromVideoFile(currentFilePath)
-
-        if (originalDuration === newDuration) {
-          console.log('Finished optimizing %s', basename(currentFilePath))
-          await remove(backupFile)
-          continue
+      await VideoPathManager.Instance.makeAvailableVideoFile(video, file, async path => {
+        currentFilePath = path
+
+        const [ videoBitrate, fps, dataResolution ] = await Promise.all([
+          getVideoFileBitrate(currentFilePath),
+          getVideoFileFPS(currentFilePath),
+          getVideoFileResolution(currentFilePath)
+        ])
+
+        const maxBitrate = getMaxBitrate({ ...dataResolution, fps })
+        const isMaxBitrateExceeded = videoBitrate > maxBitrate
+        if (isMaxBitrateExceeded) {
+          console.log(
+            'Optimizing video file %s with bitrate %s kbps (max: %s kbps)',
+            basename(currentFilePath), videoBitrate / 1000, maxBitrate / 1000
+          )
+
+          const backupFile = `${currentFilePath}_backup`
+          await copy(currentFilePath, backupFile)
+
+          await optimizeOriginalVideofile(video, file)
+          // Update file path, the video filename changed
+          currentFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
+
+          const originalDuration = await getDurationFromVideoFile(backupFile)
+          const newDuration = await getDurationFromVideoFile(currentFilePath)
+
+          if (originalDuration === newDuration) {
+            console.log('Finished optimizing %s', basename(currentFilePath))
+            await remove(backupFile)
+            return
+          }
+
+          console.log('Failed to optimize %s, restoring original', basename(currentFilePath))
+          await move(backupFile, currentFilePath, { overwrite: true })
+          await createTorrentAndSetInfoHash(video, file)
+          await file.save()
         }
-
-        console.log('Failed to optimize %s, restoring original', basename(currentFilePath))
-        await move(backupFile, currentFilePath, { overwrite: true })
-        await createTorrentAndSetInfoHash(video, file)
-        await file.save()
-      }
+      })
     }
 
     if (CONFIG.OBJECT_STORAGE.ENABLED === true) {

server/controllers/api/videos/upload.ts

@@ -1,18 +1,20 @@
 import * as express from 'express'
 import { move } from 'fs-extra'
+import { basename } from 'path'
 import { getLowercaseExtension } from '@server/helpers/core-utils'
 import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload'
 import { uuidToShort } from '@server/helpers/uuid'
 import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
 import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
+import { generateWebTorrentVideoFilename } from '@server/lib/paths'
 import {
   addMoveToObjectStorageJob,
   addOptimizeOrMergeAudioJob,
   buildLocalVideoFromReq,
   buildVideoThumbnailsFromReq,
   setVideoTags
 } from '@server/lib/video'
-import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths'
+import { VideoPathManager } from '@server/lib/video-path-manager'
 import { buildNextVideoState } from '@server/lib/video-state'
 import { openapiOperationDoc } from '@server/middlewares/doc'
 import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
@@ -153,13 +155,13 @@ async function addVideo (options: {
   video.VideoChannel = videoChannel
   video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object
 
-  const videoFile = await buildNewFile(video, videoPhysicalFile)
+  const videoFile = await buildNewFile(videoPhysicalFile)
 
   // Move physical file
-  const destination = getVideoFilePath(video, videoFile)
+  const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile)
   await move(videoPhysicalFile.path, destination)
   // This is important in case if there is another attempt in the retry process
-  videoPhysicalFile.filename = getVideoFilePath(video, videoFile)
+  videoPhysicalFile.filename = basename(destination)
   videoPhysicalFile.path = destination
 
   const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
@@ -235,7 +237,7 @@ async function addVideo (options: {
   })
 }
 
-async function buildNewFile (video: MVideo, videoPhysicalFile: express.VideoUploadFile) {
+async function buildNewFile (videoPhysicalFile: express.VideoUploadFile) {
   const videoFile = new VideoFileModel({
     extname: getLowercaseExtension(videoPhysicalFile.filename),
     size: videoPhysicalFile.size,

server/controllers/download.ts

@@ -3,7 +3,7 @@ import * as express from 'express'
 import { logger } from '@server/helpers/logger'
 import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache'
 import { Hooks } from '@server/lib/plugins/hooks'
-import { getVideoFilePath } from '@server/lib/video-paths'
+import { VideoPathManager } from '@server/lib/video-path-manager'
 import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
 import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models'
 import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants'
@@ -85,7 +85,11 @@ async function downloadVideoFile (req: express.Request, res: express.Response) {
     return res.redirect(videoFile.getObjectStorageUrl())
   }
 
-  return res.download(getVideoFilePath(video, videoFile), `${video.name}-${videoFile.resolution}p${videoFile.extname}`)
+  await VideoPathManager.Instance.makeAvailableVideoFile(video, videoFile, path => {
+    const filename = `${video.name}-${videoFile.resolution}p${videoFile.extname}`
+
+    return res.download(path, filename)
+  })
 }
 
 async function downloadHLSVideoFile (req: express.Request, res: express.Response) {
@@ -115,8 +119,11 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response
     return res.redirect(videoFile.getObjectStorageUrl())
   }
 
-  const filename = `${video.name}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}`
-  return res.download(getVideoFilePath(streamingPlaylist, videoFile), filename)
+  await VideoPathManager.Instance.makeAvailableVideoFile(streamingPlaylist, videoFile, path => {
+    const filename = `${video.name}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}`
+
+    return res.download(path, filename)
+  })
 }
 
 function getVideoFile (req: express.Request, files: MVideoFile[]) {

server/helpers/webtorrent.ts

@@ -6,7 +6,8 @@ import { dirname, join } from 'path'
 import * as WebTorrent from 'webtorrent'
 import { isArray } from '@server/helpers/custom-validators/misc'
 import { WEBSERVER } from '@server/initializers/constants'
-import { generateTorrentFileName, getVideoFilePath } from '@server/lib/video-paths'
+import { generateTorrentFileName } from '@server/lib/paths'
+import { VideoPathManager } from '@server/lib/video-path-manager'
 import { MVideo } from '@server/types/models/video/video'
 import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file'
 import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist'
@@ -78,7 +79,7 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName
   })
 }
 
-async function createTorrentAndSetInfoHash (
+function createTorrentAndSetInfoHash (
   videoOrPlaylist: MVideo | MStreamingPlaylistVideo,
   videoFile: MVideoFile
 ) {
@@ -95,22 +96,24 @@ async function createTorrentAndSetInfoHash (
     urlList: [ videoFile.getFileUrl(video) ]
   }
 
-  const torrent = await createTorrentPromise(getVideoFilePath(videoOrPlaylist, videoFile), options)
+  return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, videoFile, async videoPath => {
+    const torrent = await createTorrentPromise(videoPath, options)
 
-  const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution)
-  const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename)
-  logger.info('Creating torrent %s.', torrentPath)
+    const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution)
+    const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename)
+    logger.info('Creating torrent %s.', torrentPath)
 
-  await writeFile(torrentPath, torrent)
+    await writeFile(torrentPath, torrent)
 
-  // Remove old torrent file if it existed
-  if (videoFile.hasTorrent()) {
-    await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename))
-  }
+    // Remove old torrent file if it existed
+    if (videoFile.hasTorrent()) {
+      await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename))
+    }
 
-  const parsedTorrent = parseTorrent(torrent)
-  videoFile.infoHash = parsedTorrent.infoHash
-  videoFile.torrentFilename = torrentFilename
+    const parsedTorrent = parseTorrent(torrent)
+    videoFile.infoHash = parsedTorrent.infoHash
+    videoFile.torrentFilename = torrentFilename
+  })
 }
 
 function generateMagnetUri (

server/initializers/migrations/0065-video-file-size.ts

@@ -1,38 +1,12 @@
 import * as Sequelize from 'sequelize'
-import { stat } from 'fs-extra'
-import { VideoModel } from '../../models/video/video'
-import { getVideoFilePath } from '@server/lib/video-paths'
 
 function up (utils: {
   transaction: Sequelize.Transaction
   queryInterface: Sequelize.QueryInterface
   sequelize: Sequelize.Sequelize
   db: any
 }): Promise<void> {
-  return utils.db.Video.listOwnedAndPopulateAuthorAndTags()
-    .then((videos: VideoModel[]) => {
-      const tasks: Promise<any>[] = []
-
-      videos.forEach(video => {
-        video.VideoFiles.forEach(videoFile => {
-          const p = new Promise((res, rej) => {
-            stat(getVideoFilePath(video, videoFile), (err, stats) => {
-              if (err) return rej(err)
-
-              videoFile.size = stats.size
-              videoFile.save().then(res).catch(rej)
-            })
-          })
-
-          tasks.push(p)
-        })
-      })
-
-      return tasks
-    })
-    .then((tasks: Promise<any>[]) => {
-      return Promise.all(tasks)
-    })
+  throw new Error('Removed, please upgrade from a previous version first.')
 }
 
 function down (options) {