From d469307ca0e04797cb5948d15d03f141f194240a Mon Sep 17 00:00:00 2001 From: bnkai <48220860+bnkai@users.noreply.github.com> Date: Tue, 10 Mar 2020 18:44:02 +0200 Subject: [PATCH] Add detection of container/codec compatibility for live file streaming * take into account container in generate transcodes task * add container info to DB * increment appSchema to 5 --- pkg/api/routes_scene.go | 8 +- pkg/database/database.go | 2 +- .../migrations/5_scenes_format.up.sql | 1 + pkg/ffmpeg/encoder_transcode.go | 12 +++ pkg/ffmpeg/ffprobe.go | 78 +++++++++++++++++++ pkg/ffmpeg/media_detection.go | 66 ++++++++++++++++ pkg/manager/jsonschema/scene.go | 1 + pkg/manager/task_export.go | 3 + pkg/manager/task_import.go | 3 + pkg/manager/task_scan.go | 27 ++++++- pkg/manager/task_transcode.go | 45 +++++++++-- pkg/models/model_scene.go | 1 + pkg/models/querybuilder_scene.go | 17 +++- 13 files changed, 251 insertions(+), 13 deletions(-) create mode 100644 pkg/database/migrations/5_scenes_format.up.sql create mode 100644 pkg/ffmpeg/media_detection.go diff --git a/pkg/api/routes_scene.go b/pkg/api/routes_scene.go index b5cf8827e57..9cc0039350f 100644 --- a/pkg/api/routes_scene.go +++ b/pkg/api/routes_scene.go @@ -42,13 +42,19 @@ func (rs sceneRoutes) Routes() chi.Router { // region Handlers func (rs sceneRoutes) Stream(w http.ResponseWriter, r *http.Request) { + scene := r.Context().Value(sceneKey).(*models.Scene) + container := "" + if scene.Format.Valid { + container = scene.Format.String + } + // detect if not a streamable file and try to transcode it instead filepath := manager.GetInstance().Paths.Scene.GetStreamPath(scene.Path, scene.Checksum) videoCodec := scene.VideoCodec.String hasTranscode, _ := manager.HasTranscode(scene) - if ffmpeg.IsValidCodec(videoCodec) || hasTranscode { + if (ffmpeg.IsValidCodec(videoCodec) && ffmpeg.IsValidCombo(videoCodec, ffmpeg.Container(container))) || hasTranscode { manager.RegisterStream(filepath, &w) http.ServeFile(w, r, filepath) manager.WaitAndDeregisterStream(filepath, &w, r) diff --git a/pkg/database/database.go b/pkg/database/database.go index fb7c4c848d6..86d124c9c9a 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -17,7 +17,7 @@ import ( ) var DB *sqlx.DB -var appSchemaVersion uint = 4 +var appSchemaVersion uint = 5 const sqlite3Driver = "sqlite3_regexp" diff --git a/pkg/database/migrations/5_scenes_format.up.sql b/pkg/database/migrations/5_scenes_format.up.sql new file mode 100644 index 00000000000..93f5c44a907 --- /dev/null +++ b/pkg/database/migrations/5_scenes_format.up.sql @@ -0,0 +1 @@ +ALTER TABLE `scenes` ADD COLUMN `format` varchar(255); diff --git a/pkg/ffmpeg/encoder_transcode.go b/pkg/ffmpeg/encoder_transcode.go index a908f00acda..1fdc12a6125 100644 --- a/pkg/ffmpeg/encoder_transcode.go +++ b/pkg/ffmpeg/encoder_transcode.go @@ -69,6 +69,18 @@ func (e *Encoder) Transcode(probeResult VideoFile, options TranscodeOptions) { _, _ = e.run(probeResult, args) } +//copy the video stream as is +func (e *Encoder) TranscodeAudio(probeResult VideoFile, options TranscodeOptions) { + args := []string{ + "-i", probeResult.Path, + "-c:v", "copy", + "-c:a", "aac", + "-strict", "-2", + options.OutputPath, + } + _, _ = e.run(probeResult, args) +} + func (e *Encoder) StreamTranscode(probeResult VideoFile, startTime string, maxTranscodeSize models.StreamingResolutionEnum) (io.ReadCloser, *os.Process, error) { scale := calculateTranscodeScale(probeResult, maxTranscodeSize) args := []string{} diff --git a/pkg/ffmpeg/ffprobe.go b/pkg/ffmpeg/ffprobe.go index cbddb14d35e..c469f137844 100644 --- a/pkg/ffmpeg/ffprobe.go +++ b/pkg/ffmpeg/ffprobe.go @@ -14,6 +14,60 @@ import ( var ValidCodecs = []string{"h264", "h265", "vp8", "vp9"} +type Container string + +const ( + Mp4 Container = "mp4" + M4v Container = "m4v" + Mov Container = "mov" + Wmv Container = "wmv" + Webm Container = "webm" + Matroska Container = "matroska" + Avi Container = "avi" + Flv Container = "flv" + Mpegts Container = "mpegts" +) + +var validForH264 = []Container{Mp4} +var validForH265 = []Container{Mp4} +var validForVp8 = []Container{Webm} +var validForVp9 = []Container{Webm} + +//maps user readable container strings to ffprobe's format_name +//on some formats ffprobe can't differentiate +var ContainerToFfprobe = map[Container]string{ + Mp4: "mov,mp4,m4a,3gp,3g2,mj2", + M4v: "mov,mp4,m4a,3gp,3g2,mj2", + Mov: "mov,mp4,m4a,3gp,3g2,mj2", + Wmv: "asf", + Webm: "matroska,webm", + Matroska: "matroska,webm", + Avi: "avi", + Flv: "flv", + Mpegts: "mpegts", +} + +var FfprobeToContainer = map[string]Container{ + "mov,mp4,m4a,3gp,3g2,mj2": Mp4, // browsers support all of them so we don't care + "asf": Wmv, + "avi": Avi, + "flv": Flv, + "mpegts": Mpegts, + "matroska,webm": Matroska, +} + +func MatchContainer(format string, filePath string) Container { // match ffprobe string to our Container + + container := FfprobeToContainer[format] + if container == Matroska { + container = MagicContainer(filePath) // use magic number instead of ffprobe for matroska,webm + } + if container == "" { // if format is not in our Container list leave it as ffprobes reported format_name + container = Container(format) + } + return container +} + func IsValidCodec(codecName string) bool { for _, c := range ValidCodecs { if c == codecName { @@ -23,6 +77,30 @@ func IsValidCodec(codecName string) bool { return false } +func IsValidForContainer(format Container, validContainers []Container) bool { + for _, fmt := range validContainers { + if fmt == format { + return true + } + } + return false +} + +//extend stream validation check to take into account container +func IsValidCombo(codecName string, format Container) bool { + switch codecName { + case "h264": + return IsValidForContainer(format, validForH264) + case "h265": + return IsValidForContainer(format, validForH265) + case "vp8": + return IsValidForContainer(format, validForVp8) + case "vp9": + return IsValidForContainer(format, validForVp9) + } + return false +} + type VideoFile struct { JSON FFProbeJSON AudioStream *FFProbeStream diff --git a/pkg/ffmpeg/media_detection.go b/pkg/ffmpeg/media_detection.go new file mode 100644 index 00000000000..4de7e4ba641 --- /dev/null +++ b/pkg/ffmpeg/media_detection.go @@ -0,0 +1,66 @@ +package ffmpeg + +import ( + "bytes" + "github.com/stashapp/stash/pkg/logger" + "os" +) + +// detect file format from magic file number +// https://github.com/lex-r/filetype/blob/73c10ad714e3b8ecf5cd1564c882ed6d440d5c2d/matchers/video.go + +func mkv(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x1A && buf[1] == 0x45 && + buf[2] == 0xDF && buf[3] == 0xA3 && + containsMatroskaSignature(buf, []byte{'m', 'a', 't', 'r', 'o', 's', 'k', 'a'}) +} + +func webm(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x1A && buf[1] == 0x45 && + buf[2] == 0xDF && buf[3] == 0xA3 && + containsMatroskaSignature(buf, []byte{'w', 'e', 'b', 'm'}) +} + +func containsMatroskaSignature(buf, subType []byte) bool { + limit := 4096 + if len(buf) < limit { + limit = len(buf) + } + + index := bytes.Index(buf[:limit], subType) + if index < 3 { + return false + } + + return buf[index-3] == 0x42 && buf[index-2] == 0x82 +} + +//returns container as string ("" on error or no match) +//implements only mkv or webm as ffprobe can't distinguish between them +//and not all browsers support mkv +func MagicContainer(file_path string) Container { + file, err := os.Open(file_path) + if err != nil { + logger.Errorf("[magicfile] %v", err) + return "" + } + + defer file.Close() + + buf := make([]byte, 4096) + _, err = file.Read(buf) + if err != nil { + logger.Errorf("[magicfile] %v", err) + return "" + } + + if webm(buf) { + return Webm + } + if mkv(buf) { + return Matroska + } + return "" +} diff --git a/pkg/manager/jsonschema/scene.go b/pkg/manager/jsonschema/scene.go index 57b16e86018..c69d4466093 100644 --- a/pkg/manager/jsonschema/scene.go +++ b/pkg/manager/jsonschema/scene.go @@ -22,6 +22,7 @@ type SceneFile struct { Duration string `json:"duration"` VideoCodec string `json:"video_codec"` AudioCodec string `json:"audio_codec"` + Format string `json:"format"` Width int `json:"width"` Height int `json:"height"` Framerate string `json:"framerate"` diff --git a/pkg/manager/task_export.go b/pkg/manager/task_export.go index 0599a0eaa2c..dc1edea1e7c 100644 --- a/pkg/manager/task_export.go +++ b/pkg/manager/task_export.go @@ -165,6 +165,9 @@ func (t *ExportTask) ExportScenes(ctx context.Context) { if scene.AudioCodec.Valid { newSceneJSON.File.AudioCodec = scene.AudioCodec.String } + if scene.Format.Valid { + newSceneJSON.File.Format = scene.Format.String + } if scene.Width.Valid { newSceneJSON.File.Width = int(scene.Width.Int64) } diff --git a/pkg/manager/task_import.go b/pkg/manager/task_import.go index 218869f416b..e41f0182d40 100644 --- a/pkg/manager/task_import.go +++ b/pkg/manager/task_import.go @@ -498,6 +498,9 @@ func (t *ImportTask) ImportScenes(ctx context.Context) { if sceneJSON.File.AudioCodec != "" { newScene.AudioCodec = sql.NullString{String: sceneJSON.File.AudioCodec, Valid: true} } + if sceneJSON.File.Format != "" { + newScene.Format = sql.NullString{String: sceneJSON.File.Format, Valid: true} + } if sceneJSON.File.Width != 0 { newScene.Width = sql.NullInt64{Int64: int64(sceneJSON.File.Width), Valid: true} } diff --git a/pkg/manager/task_scan.go b/pkg/manager/task_scan.go index 579a3fc00cd..51a08d0d48d 100644 --- a/pkg/manager/task_scan.go +++ b/pkg/manager/task_scan.go @@ -81,8 +81,31 @@ func (t *ScanTask) scanScene() { qb := models.NewSceneQueryBuilder() scene, _ := qb.FindByPath(t.FilePath) if scene != nil { - // We already have this item in the database, check for thumbnails,screenshots + // We already have this item in the database + //check for thumbnails,screenshots t.makeScreenshots(nil, scene.Checksum) + + //check for container + if !scene.Format.Valid { + videoFile, err := ffmpeg.NewVideoFile(instance.FFProbePath, t.FilePath) + if err != nil { + logger.Error(err.Error()) + return + } + container := ffmpeg.MatchContainer(videoFile.Container, t.FilePath) + logger.Infof("Adding container %s to file %s", container, t.FilePath) + + ctx := context.TODO() + tx := database.DB.MustBeginTx(ctx, nil) + err = qb.UpdateFormat(scene.ID, string(container), tx) + if err != nil { + logger.Error(err.Error()) + _ = tx.Rollback() + } else if err := tx.Commit(); err != nil { + logger.Error(err.Error()) + } + + } return } @@ -91,6 +114,7 @@ func (t *ScanTask) scanScene() { logger.Error(err.Error()) return } + container := ffmpeg.MatchContainer(videoFile.Container, t.FilePath) // Override title to be filename if UseFileMetadata is false if !t.UseFileMetadata { @@ -130,6 +154,7 @@ func (t *ScanTask) scanScene() { Duration: sql.NullFloat64{Float64: videoFile.Duration, Valid: true}, VideoCodec: sql.NullString{String: videoFile.VideoCodec, Valid: true}, AudioCodec: sql.NullString{String: videoFile.AudioCodec, Valid: true}, + Format: sql.NullString{String: string(container), Valid: true}, Width: sql.NullInt64{Int64: int64(videoFile.Width), Valid: true}, Height: sql.NullInt64{Int64: int64(videoFile.Height), Valid: true}, Framerate: sql.NullFloat64{Float64: videoFile.FrameRate, Valid: true}, diff --git a/pkg/manager/task_transcode.go b/pkg/manager/task_transcode.go index a43e50cd3e8..302f938f74c 100644 --- a/pkg/manager/task_transcode.go +++ b/pkg/manager/task_transcode.go @@ -16,17 +16,32 @@ type GenerateTranscodeTask struct { func (t *GenerateTranscodeTask) Start(wg *sync.WaitGroup) { defer wg.Done() - videoCodec := t.Scene.VideoCodec.String - if ffmpeg.IsValidCodec(videoCodec) { - return - } hasTranscode, _ := HasTranscode(&t.Scene) if hasTranscode { return } - logger.Infof("[transcode] <%s> scene has codec %s", t.Scene.Checksum, t.Scene.VideoCodec.String) + var container ffmpeg.Container + + if t.Scene.Format.Valid { + container = ffmpeg.Container(t.Scene.Format.String) + + } else { // container isn't in the DB + // shouldn't happen unless user hasn't scanned after updating to PR#384+ version + tmpVideoFile, err := ffmpeg.NewVideoFile(instance.FFProbePath, t.Scene.Path) + if err != nil { + logger.Errorf("[transcode] error reading video file: %s", err.Error()) + return + } + + container = ffmpeg.MatchContainer(tmpVideoFile.Container, t.Scene.Path) + } + + videoCodec := t.Scene.VideoCodec.String + if ffmpeg.IsValidCodec(videoCodec) && ffmpeg.IsValidCombo(videoCodec, container) { + return + } videoFile, err := ffmpeg.NewVideoFile(instance.FFProbePath, t.Scene.Path) if err != nil { @@ -41,24 +56,38 @@ func (t *GenerateTranscodeTask) Start(wg *sync.WaitGroup) { MaxTranscodeSize: transcodeSize, } encoder := ffmpeg.NewEncoder(instance.FFMPEGPath) - encoder.Transcode(*videoFile, options) + + if videoCodec == "h264" { // for non supported h264 files stream copy the video part + encoder.TranscodeAudio(*videoFile, options) + } else { + encoder.Transcode(*videoFile, options) + } + if err := os.Rename(outputPath, instance.Paths.Scene.GetTranscodePath(t.Scene.Checksum)); err != nil { logger.Errorf("[transcode] error generating transcode: %s", err.Error()) return } + logger.Debugf("[transcode] <%s> created transcode: %s", t.Scene.Checksum, outputPath) return } +// return true if transcode is needed +// used only when counting files to generate, doesn't affect the actual transcode generation +// if container is missing from DB it is treated as non supported in order not to delay the user func (t *GenerateTranscodeTask) isTranscodeNeeded() bool { videoCodec := t.Scene.VideoCodec.String - hasTranscode, _ := HasTranscode(&t.Scene) + container := "" + if t.Scene.Format.Valid { + container = t.Scene.Format.String + } - if ffmpeg.IsValidCodec(videoCodec) { + if ffmpeg.IsValidCodec(videoCodec) && ffmpeg.IsValidCombo(videoCodec, ffmpeg.Container(container)) { return false } + hasTranscode, _ := HasTranscode(&t.Scene) if hasTranscode { return false } diff --git a/pkg/models/model_scene.go b/pkg/models/model_scene.go index f1eb8dd56c5..1623533beb0 100644 --- a/pkg/models/model_scene.go +++ b/pkg/models/model_scene.go @@ -19,6 +19,7 @@ type Scene struct { Size sql.NullString `db:"size" json:"size"` Duration sql.NullFloat64 `db:"duration" json:"duration"` VideoCodec sql.NullString `db:"video_codec" json:"video_codec"` + Format sql.NullString `db:"format" json:"format_name"` AudioCodec sql.NullString `db:"audio_codec" json:"audio_codec"` Width sql.NullInt64 `db:"width" json:"width"` Height sql.NullInt64 `db:"height" json:"height"` diff --git a/pkg/models/querybuilder_scene.go b/pkg/models/querybuilder_scene.go index e353a7c38ee..031396aa084 100644 --- a/pkg/models/querybuilder_scene.go +++ b/pkg/models/querybuilder_scene.go @@ -49,10 +49,10 @@ func (qb *SceneQueryBuilder) Create(newScene Scene, tx *sqlx.Tx) (*Scene, error) ensureTx(tx) result, err := tx.NamedExec( `INSERT INTO scenes (checksum, path, title, details, url, date, rating, size, duration, video_codec, - audio_codec, width, height, framerate, bitrate, studio_id, cover, + audio_codec, format, width, height, framerate, bitrate, studio_id, cover, created_at, updated_at) VALUES (:checksum, :path, :title, :details, :url, :date, :rating, :size, :duration, :video_codec, - :audio_codec, :width, :height, :framerate, :bitrate, :studio_id, :cover, + :audio_codec, :format, :width, :height, :framerate, :bitrate, :studio_id, :cover, :created_at, :updated_at) `, newScene, @@ -521,3 +521,16 @@ func (qb *SceneQueryBuilder) queryScenes(query string, args []interface{}, tx *s return scenes, nil } + +func (qb *SceneQueryBuilder) UpdateFormat(id int, format string, tx *sqlx.Tx) error { + ensureTx(tx) + _, err := tx.Exec( + `UPDATE scenes SET format = ? WHERE scenes.id = ? `, + format, id, + ) + if err != nil { + return err + } + + return nil +}