Add detection of container/video_codec/audio_codec compatibility for …

…live file streaming or transcoding (#384)

* add forceMKV, forceHEVC config options
* drop audio stream instead of trying to transcode for ffmpeg unsupported/unknown audio codecs

graphql/documents/data/config.graphql

@@ -4,6 +4,8 @@ fragment ConfigGeneralData on ConfigGeneralResult {
   generatedPath
   maxTranscodeSize
   maxStreamingTranscodeSize
+  forceMkv
+  forceHevc
   username
   password
   maxSessionAge

graphql/schema/types/config.graphql

@@ -18,6 +18,10 @@ input ConfigGeneralInput {
   maxTranscodeSize: StreamingResolutionEnum
   """Max streaming transcode size"""
   maxStreamingTranscodeSize: StreamingResolutionEnum
+  """Force MKV as supported format"""
+  forceMkv: Boolean!
+  """Force HEVC as a supported codec"""
+  forceHevc: Boolean!
   """Username"""
   username: String
   """Password"""
@@ -49,6 +53,10 @@ type ConfigGeneralResult {
   maxTranscodeSize: StreamingResolutionEnum
   """Max streaming transcode size"""
   maxStreamingTranscodeSize: StreamingResolutionEnum
+  """Force MKV as supported format"""
+  forceMkv: Boolean!
+  """Force HEVC as a supported codec"""
+  forceHevc: Boolean!
   """Username"""
   username: String!
   """Password"""

pkg/api/resolver_mutation_configure.go

@@ -45,6 +45,8 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co
 	if input.MaxStreamingTranscodeSize != nil {
 		config.Set(config.MaxStreamingTranscodeSize, input.MaxStreamingTranscodeSize.String())
 	}
+	config.Set(config.ForceMKV, input.ForceMkv)
+	config.Set(config.ForceHEVC, input.ForceHevc)
 
 	if input.Username != nil {
 		config.Set(config.Username, input.Username)

pkg/api/resolver_query_configuration.go

@@ -41,6 +41,8 @@ func makeConfigGeneralResult() *models.ConfigGeneralResult {
 		GeneratedPath:             config.GetGeneratedPath(),
 		MaxTranscodeSize:          &maxTranscodeSize,
 		MaxStreamingTranscodeSize: &maxStreamingTranscodeSize,
+		ForceMkv:                  config.GetForceMKV(),
+		ForceHevc:                 config.GetForceHEVC(),
 		Username:                  config.GetUsername(),
 		Password:                  config.GetPasswordHash(),
 		MaxSessionAge:             config.GetMaxSessionAge(),

pkg/api/routes_scene.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"io"
 	"net/http"
+	"os"
 	"strconv"
 	"strings"
 
@@ -42,13 +43,32 @@ func (rs sceneRoutes) Routes() chi.Router {
 // region Handlers
 
 func (rs sceneRoutes) Stream(w http.ResponseWriter, r *http.Request) {
+
 	scene := r.Context().Value(sceneKey).(*models.Scene)
 
+	container := ""
+	if scene.Format.Valid {
+		container = scene.Format.String
+	} else { // container isn't in the DB
+		// shouldn't happen, fallback to ffprobe
+		tmpVideoFile, err := ffmpeg.NewVideoFile(manager.GetInstance().FFProbePath, scene.Path)
+		if err != nil {
+			logger.Errorf("[transcode] error reading video file: %s", err.Error())
+			return
+		}
+
+		container = string(ffmpeg.MatchContainer(tmpVideoFile.Container, scene.Path))
+	}
+
 	// detect if not a streamable file and try to transcode it instead
 	filepath := manager.GetInstance().Paths.Scene.GetStreamPath(scene.Path, scene.Checksum)
 	videoCodec := scene.VideoCodec.String
+	audioCodec := ffmpeg.MissingUnsupported
+	if scene.AudioCodec.Valid {
+		audioCodec = ffmpeg.AudioCodec(scene.AudioCodec.String)
+	}
 	hasTranscode, _ := manager.HasTranscode(scene)
-	if ffmpeg.IsValidCodec(videoCodec) || hasTranscode {
+	if ffmpeg.IsValidCodec(videoCodec) && ffmpeg.IsValidCombo(videoCodec, ffmpeg.Container(container)) && ffmpeg.IsValidAudioForContainer(audioCodec, ffmpeg.Container(container)) || hasTranscode {
 		manager.RegisterStream(filepath, &w)
 		http.ServeFile(w, r, filepath)
 		manager.WaitAndDeregisterStream(filepath, &w, r)
@@ -69,16 +89,50 @@ func (rs sceneRoutes) Stream(w http.ResponseWriter, r *http.Request) {
 
 	encoder := ffmpeg.NewEncoder(manager.GetInstance().FFMPEGPath)
 
-	stream, process, err := encoder.StreamTranscode(*videoFile, startTime, config.GetMaxStreamingTranscodeSize())
+	var stream io.ReadCloser
+	var process *os.Process
+	mimeType := ffmpeg.MimeWebm
+
+	if audioCodec == ffmpeg.MissingUnsupported {
+		//ffmpeg fails if it trys to transcode a non supported audio codec
+		stream, process, err = encoder.StreamTranscodeVideo(*videoFile, startTime, config.GetMaxStreamingTranscodeSize())
+	} else {
+		copyVideo := false // try to be smart if the video to be transcoded is in a Matroska container
+		//  mp4 has always supported audio so it doesn't need to be checked
+		//  while mpeg_ts has seeking issues if we don't reencode the video
+
+		if config.GetForceMKV() { // If MKV is forced as supported and video codec is also supported then only transcode audio
+			if ffmpeg.Container(container) == ffmpeg.Matroska {
+				switch videoCodec {
+				case ffmpeg.H264, ffmpeg.Vp9, ffmpeg.Vp8:
+					copyVideo = true
+				case ffmpeg.Hevc:
+					if config.GetForceHEVC() {
+						copyVideo = true
+					}
+
+				}
+			}
+		}
+
+		if copyVideo { // copy video stream instead of transcoding it
+			stream, process, err = encoder.StreamMkvTranscodeAudio(*videoFile, startTime, config.GetMaxStreamingTranscodeSize())
+			mimeType = ffmpeg.MimeMkv
+
+		} else {
+			stream, process, err = encoder.StreamTranscode(*videoFile, startTime, config.GetMaxStreamingTranscodeSize())
+		}
+	}
+
 	if err != nil {
 		logger.Errorf("[stream] error transcoding video file: %s", err.Error())
 		return
 	}
 
 	w.WriteHeader(http.StatusOK)
-	w.Header().Set("Content-Type", "video/webm")
+	w.Header().Set("Content-Type", mimeType)
 
-	logger.Info("[stream] transcoding video file")
+	logger.Infof("[stream] transcoding video file to %s", mimeType)
 
 	// handle if client closes the connection
 	notify := r.Context().Done()

pkg/database/database.go

@@ -19,7 +19,7 @@ import (
 
 var DB *sqlx.DB
 var dbPath string
-var appSchemaVersion uint = 5
+var appSchemaVersion uint = 6
 var databaseSchemaVersion uint
 
 const sqlite3Driver = "sqlite3_regexp"

pkg/database/migrations/6_scenes_format.up.sql

@@ -0,0 +1 @@
+ALTER TABLE `scenes` ADD COLUMN `format` varchar(255);

pkg/ffmpeg/encoder_transcode.go

@@ -69,6 +69,49 @@ func (e *Encoder) Transcode(probeResult VideoFile, options TranscodeOptions) {
 	_, _ = e.run(probeResult, args)
 }
 
+//transcode the video, remove the audio
+//in some videos where the audio codec is not supported by ffmpeg
+//ffmpeg fails if you try to transcode the audio
+func (e *Encoder) TranscodeVideo(probeResult VideoFile, options TranscodeOptions) {
+	scale := calculateTranscodeScale(probeResult, options.MaxTranscodeSize)
+	args := []string{
+		"-i", probeResult.Path,
+		"-an",
+		"-c:v", "libx264",
+		"-pix_fmt", "yuv420p",
+		"-profile:v", "high",
+		"-level", "4.2",
+		"-preset", "superfast",
+		"-crf", "23",
+		"-vf", "scale=" + scale,
+		options.OutputPath,
+	}
+	_, _ = e.run(probeResult, args)
+}
+
+//copy the video stream as is, transcode audio
+func (e *Encoder) TranscodeAudio(probeResult VideoFile, options TranscodeOptions) {
+	args := []string{
+		"-i", probeResult.Path,
+		"-c:v", "copy",
+		"-c:a", "aac",
+		"-strict", "-2",
+		options.OutputPath,
+	}
+	_, _ = e.run(probeResult, args)
+}
+
+//copy the video stream as is, drop audio
+func (e *Encoder) CopyVideo(probeResult VideoFile, options TranscodeOptions) {
+	args := []string{
+		"-i", probeResult.Path,
+		"-an",
+		"-c:v", "copy",
+		options.OutputPath,
+	}
+	_, _ = e.run(probeResult, args)
+}
+
 func (e *Encoder) StreamTranscode(probeResult VideoFile, startTime string, maxTranscodeSize models.StreamingResolutionEnum) (io.ReadCloser, *os.Process, error) {
 	scale := calculateTranscodeScale(probeResult, maxTranscodeSize)
 	args := []string{}
@@ -92,3 +135,53 @@ func (e *Encoder) StreamTranscode(probeResult VideoFile, startTime string, maxTr
 
 	return e.stream(probeResult, args)
 }
+
+//transcode the video, remove the audio
+//in some videos where the audio codec is not supported by ffmpeg
+//ffmpeg fails if you try to transcode the audio
+func (e *Encoder) StreamTranscodeVideo(probeResult VideoFile, startTime string, maxTranscodeSize models.StreamingResolutionEnum) (io.ReadCloser, *os.Process, error) {
+	scale := calculateTranscodeScale(probeResult, maxTranscodeSize)
+	args := []string{}
+
+	if startTime != "" {
+		args = append(args, "-ss", startTime)
+	}
+
+	args = append(args,
+		"-i", probeResult.Path,
+		"-an",
+		"-c:v", "libvpx-vp9",
+		"-vf", "scale="+scale,
+		"-deadline", "realtime",
+		"-cpu-used", "5",
+		"-row-mt", "1",
+		"-crf", "30",
+		"-b:v", "0",
+		"-f", "webm",
+		"pipe:",
+	)
+
+	return e.stream(probeResult, args)
+}
+
+//it is very common in MKVs to have just the audio codec unsupported
+//copy the video stream, transcode the audio and serve as Matroska
+func (e *Encoder) StreamMkvTranscodeAudio(probeResult VideoFile, startTime string, maxTranscodeSize models.StreamingResolutionEnum) (io.ReadCloser, *os.Process, error) {
+	args := []string{}
+
+	if startTime != "" {
+		args = append(args, "-ss", startTime)
+	}
+
+	args = append(args,
+		"-i", probeResult.Path,
+		"-c:v", "copy",
+		"-c:a", "libopus",
+		"-b:a", "96k",
+		"-vbr", "on",
+		"-f", "matroska",
+		"pipe:",
+	)
+
+	return e.stream(probeResult, args)
+}