From 371a8a03551faf272a129f1b77dbdcee2334d5cb Mon Sep 17 00:00:00 2001 From: Infinite Date: Fri, 17 Apr 2020 21:20:58 +0200 Subject: [PATCH 01/16] Optimize performer and scene queries --- pkg/api/resolver_model_scene.go | 10 +-- pkg/api/routes_performer.go | 2 +- pkg/api/routes_scene.go | 2 +- pkg/models/querybuilder_gallery.go | 2 +- pkg/models/querybuilder_movies.go | 3 +- pkg/models/querybuilder_performer.go | 74 +++++++++++++--- pkg/models/querybuilder_scene.go | 111 ++++++++++++++++++------ pkg/models/querybuilder_scene_marker.go | 3 +- pkg/models/querybuilder_tag.go | 3 +- 9 files changed, 156 insertions(+), 54 deletions(-) diff --git a/pkg/api/resolver_model_scene.go b/pkg/api/resolver_model_scene.go index 36e92105272..f960b8c45a9 100644 --- a/pkg/api/resolver_model_scene.go +++ b/pkg/api/resolver_model_scene.go @@ -89,17 +89,17 @@ func (r *sceneResolver) IsStreamable(ctx context.Context, obj *models.Scene) (bo func (r *sceneResolver) SceneMarkers(ctx context.Context, obj *models.Scene) ([]*models.SceneMarker, error) { qb := models.NewSceneMarkerQueryBuilder() - return qb.FindBySceneID(obj.ID, nil) + return qb.FindBySceneID(obj.ID, nil) } func (r *sceneResolver) Gallery(ctx context.Context, obj *models.Scene) (*models.Gallery, error) { qb := models.NewGalleryQueryBuilder() - return qb.FindBySceneID(obj.ID, nil) + return qb.FindBySceneID(obj.ID, nil) } func (r *sceneResolver) Studio(ctx context.Context, obj *models.Scene) (*models.Studio, error) { qb := models.NewStudioQueryBuilder() - return qb.FindBySceneID(obj.ID) + return qb.FindBySceneID(obj.ID) } func (r *sceneResolver) Movies(ctx context.Context, obj *models.Scene) ([]*models.SceneMovie, error) { @@ -136,10 +136,10 @@ func (r *sceneResolver) Movies(ctx context.Context, obj *models.Scene) ([]*model func (r *sceneResolver) Tags(ctx context.Context, obj *models.Scene) ([]*models.Tag, error) { qb := models.NewTagQueryBuilder() - return qb.FindBySceneID(obj.ID, nil) + return qb.FindBySceneID(obj.ID, nil) } func (r *sceneResolver) Performers(ctx context.Context, obj *models.Scene) ([]*models.Performer, error) { qb := models.NewPerformerQueryBuilder() - return qb.FindBySceneID(obj.ID, nil) + return qb.FindBySceneID(obj.ID, nil) } diff --git a/pkg/api/routes_performer.go b/pkg/api/routes_performer.go index 57958998b4d..f6d5cc13ec8 100644 --- a/pkg/api/routes_performer.go +++ b/pkg/api/routes_performer.go @@ -48,7 +48,7 @@ func PerformerCtx(next http.Handler) http.Handler { } qb := models.NewPerformerQueryBuilder() - performer, err := qb.Find(performerID) + performer, err := qb.FindWithAllColumns(performerID) if err != nil { http.Error(w, http.StatusText(404), 404) return diff --git a/pkg/api/routes_scene.go b/pkg/api/routes_scene.go index 171d868b48c..d8c0cfe24b3 100644 --- a/pkg/api/routes_scene.go +++ b/pkg/api/routes_scene.go @@ -280,7 +280,7 @@ func SceneCtx(next http.Handler) http.Handler { if sceneID == 0 { scene, err = qb.FindByChecksum(sceneIdentifierQueryParam) } else { - scene, err = qb.Find(sceneID) + scene, err = qb.FindWithAllColumns(sceneID) } if err != nil { diff --git a/pkg/models/querybuilder_gallery.go b/pkg/models/querybuilder_gallery.go index b27af9b5486..0f2b06e06f9 100644 --- a/pkg/models/querybuilder_gallery.go +++ b/pkg/models/querybuilder_gallery.go @@ -93,7 +93,7 @@ func (qb *GalleryQueryBuilder) FindByPath(path string) (*Gallery, error) { } func (qb *GalleryQueryBuilder) FindBySceneID(sceneID int, tx *sqlx.Tx) (*Gallery, error) { - query := "SELECT galleries.* FROM galleries JOIN scenes ON scenes.id = galleries.scene_id WHERE scenes.id = ? LIMIT 1" + query := "SELECT galleries.* FROM galleries WHERE galleries.scene_id = ? LIMIT 1" args := []interface{}{sceneID} return qb.queryGallery(query, args, tx) } diff --git a/pkg/models/querybuilder_movies.go b/pkg/models/querybuilder_movies.go index 125d7587430..d5a3cdfbe77 100644 --- a/pkg/models/querybuilder_movies.go +++ b/pkg/models/querybuilder_movies.go @@ -76,8 +76,7 @@ func (qb *MovieQueryBuilder) FindBySceneID(sceneID int, tx *sqlx.Tx) ([]*Movie, query := ` SELECT movies.* FROM movies LEFT JOIN movies_scenes as scenes_join on scenes_join.movie_id = movies.id - LEFT JOIN scenes on scenes_join.scene_id = scenes.id - WHERE scenes.id = ? + WHERE scenes_join.scenes.id = ? GROUP BY movies.id ` args := []interface{}{sceneID} diff --git a/pkg/models/querybuilder_performer.go b/pkg/models/querybuilder_performer.go index eea13a3d4da..ed53c43e3c4 100644 --- a/pkg/models/querybuilder_performer.go +++ b/pkg/models/querybuilder_performer.go @@ -3,12 +3,48 @@ package models import ( "database/sql" "strconv" + "strings" "time" "github.com/jmoiron/sqlx" "github.com/stashapp/stash/pkg/database" ) +var metadataColumns = []string { + "id", + "checksum", + "name", + "gender", + "url", + "twitter", + "instagram", + "birthdate", + "ethnicity", + "country", + "eye_color", + "height", + "measurements", + "fake_tits", + "career_length", + "tattoos", + "piercings", + "aliases", + "favorite", + "created_at", + "updated_at", +} + +func selectPerformerMetadata() string { + var str strings.Builder + + for _, colName := range metadataColumns { + str.WriteString("performers." + colName + ", ") + } + + returnVal := str.String() + return "SELECT " + returnVal[:len(returnVal)-2] + " FROM performers " +} + type PerformerQueryBuilder struct{} func NewPerformerQueryBuilder() PerformerQueryBuilder { @@ -67,7 +103,17 @@ func (qb *PerformerQueryBuilder) Destroy(id string, tx *sqlx.Tx) error { } func (qb *PerformerQueryBuilder) Find(id int) (*Performer, error) { - query := "SELECT * FROM performers WHERE id = ? LIMIT 1" + query := selectPerformerMetadata() + "WHERE id = ? LIMIT 1" + args := []interface{}{id} + results, err := qb.queryPerformers(query, args, nil) + if err != nil || len(results) < 1 { + return nil, err + } + return results[0], nil +} + +func (qb *PerformerQueryBuilder) FindWithAllColumns(id int) (*Performer, error) { + query := "SELECT * FROM performers WHERE id = ? LIMIT 1" args := []interface{}{id} results, err := qb.queryPerformers(query, args, nil) if err != nil || len(results) < 1 { @@ -77,11 +123,9 @@ func (qb *PerformerQueryBuilder) Find(id int) (*Performer, error) { } func (qb *PerformerQueryBuilder) FindBySceneID(sceneID int, tx *sqlx.Tx) ([]*Performer, error) { - query := ` - SELECT performers.* FROM performers + query := selectPerformerMetadata() + ` LEFT JOIN performers_scenes as scenes_join on scenes_join.performer_id = performers.id - LEFT JOIN scenes on scenes_join.scene_id = scenes.id - WHERE scenes.id = ? + WHERE scenes_join.scene_id = ? GROUP BY performers.id ` args := []interface{}{sceneID} @@ -133,10 +177,9 @@ func (qb *PerformerQueryBuilder) Query(performerFilter *PerformerFilterType, fin tableName: tableName, } - query.body = selectDistinctIDs(tableName) + query.body = selectPerformerMetadata() query.body += ` left join performers_scenes as scenes_join on scenes_join.performer_id = performers.id - left join scenes on scenes_join.scene_id = scenes.id ` if q := findFilter.Q; q != nil && *q != "" { @@ -196,14 +239,21 @@ func (qb *PerformerQueryBuilder) Query(performerFilter *PerformerFilterType, fin handleStringCriterion(tableName+".aliases", performerFilter.Aliases, &query) query.sortAndPagination = qb.getPerformerSort(findFilter) + getPagination(findFilter) - idsResult, countResult := query.executeFind() - var performers []*Performer - for _, id := range idsResult { - performer, _ := qb.Find(id) - performers = append(performers, performer) + if len(query.whereClauses) > 0 { + query.body += " WHERE " + strings.Join(query.whereClauses, " AND ") // TODO handle AND or OR + } + query.body += " GROUP BY " + tableName + ".id " + if len(query.havingClauses) > 0 { + query.body += " HAVING " + strings.Join(query.havingClauses, " AND ") // TODO handle AND or OR } + countQuery := buildCountQuery(query.body) + countResult, _ := runCountQuery(countQuery, query.args) + + performersQuery := query.body + query.sortAndPagination + performers, _ := qb.queryPerformers(performersQuery, query.args, nil) + return performers, countResult } diff --git a/pkg/models/querybuilder_scene.go b/pkg/models/querybuilder_scene.go index c96fbdec4b2..2f0ce69db35 100644 --- a/pkg/models/querybuilder_scene.go +++ b/pkg/models/querybuilder_scene.go @@ -10,33 +10,67 @@ import ( "github.com/stashapp/stash/pkg/database" ) -const scenesForPerformerQuery = ` -SELECT scenes.* FROM scenes +var sceneMetadataColumns = []string { + "id", + "checksum", + "title", + "details", + "url", + "date", + "rating", + "size", + "duration", + "video_codec", + "audio_codec", + "format", + "width", + "height", + "framerate", + "bitrate", + "studio_id", + "created_at", + "updated_at", +} + +func selectSceneMetadata() string { + var str strings.Builder + + for _, colName := range sceneMetadataColumns { + str.WriteString("scenes." + colName + ", ") + } + + returnVal := str.String() + return "SELECT " + returnVal[:len(returnVal)-2] + " FROM scenes " +} + + + +var scenesForPerformerQuery = selectSceneMetadata() + ` LEFT JOIN performers_scenes as performers_join on performers_join.scene_id = scenes.id -LEFT JOIN performers on performers_join.performer_id = performers.id -WHERE performers.id = ? +WHERE performers_join.performer_id = ? GROUP BY scenes.id ` -const scenesForStudioQuery = ` -SELECT scenes.* FROM scenes +var countScenesForPerformerQuery = ` +SELECT performer_id FROM performers_scenes as performers_join +WHERE performer_id = ? +GROUP BY scene_id +` + +var scenesForStudioQuery = selectSceneMetadata() + ` JOIN studios ON studios.id = scenes.studio_id WHERE studios.id = ? GROUP BY scenes.id ` -const scenesForMovieQuery = ` -SELECT scenes.* FROM scenes +var scenesForMovieQuery = selectSceneMetadata() + ` LEFT JOIN movies_scenes as movies_join on movies_join.scene_id = scenes.id -LEFT JOIN movies on movies_join.movie_id = movies.id -WHERE movies.id = ? +WHERE movies_join.movie_id = ? GROUP BY scenes.id ` -const scenesForTagQuery = ` -SELECT scenes.* FROM scenes +var scenesForTagQuery = selectSceneMetadata() + ` LEFT JOIN scenes_tags as tags_join on tags_join.scene_id = scenes.id -LEFT JOIN tags on tags_join.tag_id = tags.id -WHERE tags.id = ? +WHERE tags_join.tag_id = ? GROUP BY scenes.id ` @@ -149,9 +183,15 @@ func (qb *SceneQueryBuilder) Find(id int) (*Scene, error) { return qb.find(id, nil) } -func (qb *SceneQueryBuilder) find(id int, tx *sqlx.Tx) (*Scene, error) { +func (qb *SceneQueryBuilder) FindWithAllColumns(id int) (*Scene, error) { query := "SELECT * FROM scenes WHERE id = ? LIMIT 1" args := []interface{}{id} + return qb.queryScene(query, args, nil) +} + +func (qb *SceneQueryBuilder) find(id int, tx *sqlx.Tx) (*Scene, error) { + query := selectSceneMetadata() + "WHERE id = ? LIMIT 1" + args := []interface{}{id} return qb.queryScene(query, args, tx) } @@ -162,7 +202,7 @@ func (qb *SceneQueryBuilder) FindByChecksum(checksum string) (*Scene, error) { } func (qb *SceneQueryBuilder) FindByPath(path string) (*Scene, error) { - query := "SELECT * FROM scenes WHERE path = ? LIMIT 1" + query := selectSceneMetadata() + "WHERE path = ? LIMIT 1" args := []interface{}{path} return qb.queryScene(query, args, nil) } @@ -174,7 +214,7 @@ func (qb *SceneQueryBuilder) FindByPerformerID(performerID int) ([]*Scene, error func (qb *SceneQueryBuilder) CountByPerformerID(performerID int) (int, error) { args := []interface{}{performerID} - return runCountQuery(buildCountQuery(scenesForPerformerQuery), args) + return runCountQuery(buildCountQuery(countScenesForPerformerQuery), args) } func (qb *SceneQueryBuilder) FindByStudioID(studioID int) ([]*Scene, error) { @@ -219,12 +259,12 @@ func (qb *SceneQueryBuilder) Wall(q *string) ([]*Scene, error) { if q != nil { s = *q } - query := "SELECT scenes.* FROM scenes WHERE scenes.details LIKE '%" + s + "%' ORDER BY RANDOM() LIMIT 80" + query := selectSceneMetadata() + "WHERE scenes.details LIKE '%" + s + "%' ORDER BY RANDOM() LIMIT 80" return qb.queryScenes(query, nil, nil) } func (qb *SceneQueryBuilder) All() ([]*Scene, error) { - return qb.queryScenes(selectAll("scenes")+qb.getSceneSort(nil), nil, nil) + return qb.queryScenes(selectSceneMetadata() + qb.getSceneSort(nil), nil, nil) } func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *FindFilterType) ([]*Scene, int) { @@ -238,17 +278,14 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin var whereClauses []string var havingClauses []string var args []interface{} - body := selectDistinctIDs("scenes") + body := selectSceneMetadata() body = body + ` left join scene_markers on scene_markers.scene_id = scenes.id left join performers_scenes as performers_join on performers_join.scene_id = scenes.id left join movies_scenes as movies_join on movies_join.scene_id = scenes.id - left join performers on performers_join.performer_id = performers.id - left join movies on movies_join.movie_id = movies.id left join studios as studio on studio.id = scenes.studio_id left join galleries as gallery on gallery.scene_id = scenes.id left join scenes_tags as tags_join on tags_join.scene_id = scenes.id - left join tags on tags_join.tag_id = tags.id ` if q := findFilter.Q; q != nil && *q != "" { @@ -329,6 +366,7 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin args = append(args, tagID) } + body += " LEFT JOIN tags on tags_join.tag_id = tags.id" whereClause, havingClause := getMultiCriterionClause("tags", "scenes_tags", "tag_id", tagsFilter) whereClauses = appendClause(whereClauses, whereClause) havingClauses = appendClause(havingClauses, havingClause) @@ -339,6 +377,7 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin args = append(args, performerID) } + body += " LEFT JOIN performers ON performers_join.performer_id = performers.id" whereClause, havingClause := getMultiCriterionClause("performers", "performers_scenes", "performer_id", performersFilter) whereClauses = appendClause(whereClauses, whereClause) havingClauses = appendClause(havingClauses, havingClause) @@ -359,20 +398,36 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin args = append(args, movieID) } + body += " LEFT JOIN movies ON movies_join.movie_id = movies.id" whereClause, havingClause := getMultiCriterionClause("movies", "movies_scenes", "movie_id", moviesFilter) whereClauses = appendClause(whereClauses, whereClause) havingClauses = appendClause(havingClauses, havingClause) } + //idsResult, countResult := executeFindQuery("scenes", body, args, sortAndPagination, whereClauses, havingClauses) + + //var scenes []*Scene + //for _, id := range idsResult { + //scene, _ := qb.Find(id) + //scenes = append(scenes, scene) + //} + sortAndPagination := qb.getSceneSort(findFilter) + getPagination(findFilter) - idsResult, countResult := executeFindQuery("scenes", body, args, sortAndPagination, whereClauses, havingClauses) - var scenes []*Scene - for _, id := range idsResult { - scene, _ := qb.Find(id) - scenes = append(scenes, scene) + if len(whereClauses) > 0 { + body += " WHERE " + strings.Join(whereClauses, " AND ") // TODO handle AND or OR + } + body += " GROUP BY scenes.id " + if len(havingClauses) > 0 { + body += " HAVING " + strings.Join(havingClauses, " AND ") // TODO handle AND or OR } + countQuery := buildCountQuery(body) + countResult, _ := runCountQuery(countQuery, args) + + scenesQuery := body + sortAndPagination + scenes, _ := qb.queryScenes(scenesQuery, args, nil) + return scenes, countResult } diff --git a/pkg/models/querybuilder_scene_marker.go b/pkg/models/querybuilder_scene_marker.go index b1c8790a5be..07c28d10ec7 100644 --- a/pkg/models/querybuilder_scene_marker.go +++ b/pkg/models/querybuilder_scene_marker.go @@ -77,8 +77,7 @@ func (qb *SceneMarkerQueryBuilder) Find(id int) (*SceneMarker, error) { func (qb *SceneMarkerQueryBuilder) FindBySceneID(sceneID int, tx *sqlx.Tx) ([]*SceneMarker, error) { query := ` SELECT scene_markers.* FROM scene_markers - JOIN scenes ON scenes.id = scene_markers.scene_id - WHERE scenes.id = ? + WHERE scene_markers.scene_id = ? GROUP BY scene_markers.id ORDER BY scene_markers.seconds ASC ` diff --git a/pkg/models/querybuilder_tag.go b/pkg/models/querybuilder_tag.go index 6f7f0e180ba..94cac05cd35 100644 --- a/pkg/models/querybuilder_tag.go +++ b/pkg/models/querybuilder_tag.go @@ -103,8 +103,7 @@ func (qb *TagQueryBuilder) FindBySceneMarkerID(sceneMarkerID int, tx *sqlx.Tx) ( query := ` SELECT tags.* FROM tags LEFT JOIN scene_markers_tags as scene_markers_join on scene_markers_join.tag_id = tags.id - LEFT JOIN scene_markers on scene_markers_join.scene_marker_id = scene_markers.id - WHERE scene_markers.id = ? + WHERE scene_markers_join.scene_marker_id = ? GROUP BY tags.id ` query += qb.getTagSort(nil) From 4c64368d2dc6229bbfdc41ae85f5ab363c6497ec Mon Sep 17 00:00:00 2001 From: Infinite Date: Sat, 18 Apr 2020 18:29:06 +0200 Subject: [PATCH 02/16] Formatting --- pkg/api/resolver_model_scene.go | 10 ++--- pkg/models/querybuilder_performer.go | 62 +++++++++++++-------------- pkg/models/querybuilder_scene.go | 64 ++++++++++++++-------------- 3 files changed, 67 insertions(+), 69 deletions(-) diff --git a/pkg/api/resolver_model_scene.go b/pkg/api/resolver_model_scene.go index f960b8c45a9..36e92105272 100644 --- a/pkg/api/resolver_model_scene.go +++ b/pkg/api/resolver_model_scene.go @@ -89,17 +89,17 @@ func (r *sceneResolver) IsStreamable(ctx context.Context, obj *models.Scene) (bo func (r *sceneResolver) SceneMarkers(ctx context.Context, obj *models.Scene) ([]*models.SceneMarker, error) { qb := models.NewSceneMarkerQueryBuilder() - return qb.FindBySceneID(obj.ID, nil) + return qb.FindBySceneID(obj.ID, nil) } func (r *sceneResolver) Gallery(ctx context.Context, obj *models.Scene) (*models.Gallery, error) { qb := models.NewGalleryQueryBuilder() - return qb.FindBySceneID(obj.ID, nil) + return qb.FindBySceneID(obj.ID, nil) } func (r *sceneResolver) Studio(ctx context.Context, obj *models.Scene) (*models.Studio, error) { qb := models.NewStudioQueryBuilder() - return qb.FindBySceneID(obj.ID) + return qb.FindBySceneID(obj.ID) } func (r *sceneResolver) Movies(ctx context.Context, obj *models.Scene) ([]*models.SceneMovie, error) { @@ -136,10 +136,10 @@ func (r *sceneResolver) Movies(ctx context.Context, obj *models.Scene) ([]*model func (r *sceneResolver) Tags(ctx context.Context, obj *models.Scene) ([]*models.Tag, error) { qb := models.NewTagQueryBuilder() - return qb.FindBySceneID(obj.ID, nil) + return qb.FindBySceneID(obj.ID, nil) } func (r *sceneResolver) Performers(ctx context.Context, obj *models.Scene) ([]*models.Performer, error) { qb := models.NewPerformerQueryBuilder() - return qb.FindBySceneID(obj.ID, nil) + return qb.FindBySceneID(obj.ID, nil) } diff --git a/pkg/models/querybuilder_performer.go b/pkg/models/querybuilder_performer.go index ed53c43e3c4..b5c91f6c0b0 100644 --- a/pkg/models/querybuilder_performer.go +++ b/pkg/models/querybuilder_performer.go @@ -3,46 +3,46 @@ package models import ( "database/sql" "strconv" - "strings" + "strings" "time" "github.com/jmoiron/sqlx" "github.com/stashapp/stash/pkg/database" ) -var metadataColumns = []string { - "id", - "checksum", - "name", - "gender", - "url", - "twitter", - "instagram", - "birthdate", - "ethnicity", - "country", - "eye_color", - "height", - "measurements", - "fake_tits", - "career_length", - "tattoos", - "piercings", - "aliases", - "favorite", - "created_at", - "updated_at", +var metadataColumns = []string{ + "id", + "checksum", + "name", + "gender", + "url", + "twitter", + "instagram", + "birthdate", + "ethnicity", + "country", + "eye_color", + "height", + "measurements", + "fake_tits", + "career_length", + "tattoos", + "piercings", + "aliases", + "favorite", + "created_at", + "updated_at", } func selectPerformerMetadata() string { - var str strings.Builder + var str strings.Builder - for _, colName := range metadataColumns { - str.WriteString("performers." + colName + ", ") - } + for _, colName := range metadataColumns { + str.WriteString("performers." + colName + ", ") + } - returnVal := str.String() - return "SELECT " + returnVal[:len(returnVal)-2] + " FROM performers " + returnVal := str.String() + return "SELECT " + returnVal[:len(returnVal)-2] + " FROM performers " } type PerformerQueryBuilder struct{} @@ -113,7 +113,7 @@ func (qb *PerformerQueryBuilder) Find(id int) (*Performer, error) { } func (qb *PerformerQueryBuilder) FindWithAllColumns(id int) (*Performer, error) { - query := "SELECT * FROM performers WHERE id = ? LIMIT 1" + query := "SELECT * FROM performers WHERE id = ? LIMIT 1" args := []interface{}{id} results, err := qb.queryPerformers(query, args, nil) if err != nil || len(results) < 1 { @@ -252,7 +252,7 @@ func (qb *PerformerQueryBuilder) Query(performerFilter *PerformerFilterType, fin countResult, _ := runCountQuery(countQuery, query.args) performersQuery := query.body + query.sortAndPagination - performers, _ := qb.queryPerformers(performersQuery, query.args, nil) + performers, _ := qb.queryPerformers(performersQuery, query.args, nil) return performers, countResult } diff --git a/pkg/models/querybuilder_scene.go b/pkg/models/querybuilder_scene.go index 2f0ce69db35..3473a66bdc0 100644 --- a/pkg/models/querybuilder_scene.go +++ b/pkg/models/querybuilder_scene.go @@ -10,41 +10,39 @@ import ( "github.com/stashapp/stash/pkg/database" ) -var sceneMetadataColumns = []string { - "id", - "checksum", - "title", - "details", - "url", - "date", - "rating", - "size", - "duration", - "video_codec", - "audio_codec", - "format", - "width", - "height", - "framerate", - "bitrate", - "studio_id", - "created_at", - "updated_at", +var sceneMetadataColumns = []string{ + "id", + "checksum", + "title", + "details", + "url", + "date", + "rating", + "size", + "duration", + "video_codec", + "audio_codec", + "format", + "width", + "height", + "framerate", + "bitrate", + "studio_id", + "created_at", + "updated_at", } func selectSceneMetadata() string { - var str strings.Builder + var str strings.Builder - for _, colName := range sceneMetadataColumns { - str.WriteString("scenes." + colName + ", ") - } + for _, colName := range sceneMetadataColumns { + str.WriteString("scenes." + colName + ", ") + } - returnVal := str.String() - return "SELECT " + returnVal[:len(returnVal)-2] + " FROM scenes " + returnVal := str.String() + return "SELECT " + returnVal[:len(returnVal)-2] + " FROM scenes " } - - var scenesForPerformerQuery = selectSceneMetadata() + ` LEFT JOIN performers_scenes as performers_join on performers_join.scene_id = scenes.id WHERE performers_join.performer_id = ? @@ -264,7 +262,7 @@ func (qb *SceneQueryBuilder) Wall(q *string) ([]*Scene, error) { } func (qb *SceneQueryBuilder) All() ([]*Scene, error) { - return qb.queryScenes(selectSceneMetadata() + qb.getSceneSort(nil), nil, nil) + return qb.queryScenes(selectSceneMetadata()+qb.getSceneSort(nil), nil, nil) } func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *FindFilterType) ([]*Scene, int) { @@ -377,7 +375,7 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin args = append(args, performerID) } - body += " LEFT JOIN performers ON performers_join.performer_id = performers.id" + body += " LEFT JOIN performers ON performers_join.performer_id = performers.id" whereClause, havingClause := getMultiCriterionClause("performers", "performers_scenes", "performer_id", performersFilter) whereClauses = appendClause(whereClauses, whereClause) havingClauses = appendClause(havingClauses, havingClause) @@ -408,8 +406,8 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin //var scenes []*Scene //for _, id := range idsResult { - //scene, _ := qb.Find(id) - //scenes = append(scenes, scene) + //scene, _ := qb.Find(id) + //scenes = append(scenes, scene) //} sortAndPagination := qb.getSceneSort(findFilter) + getPagination(findFilter) @@ -426,7 +424,7 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin countResult, _ := runCountQuery(countQuery, args) scenesQuery := body + sortAndPagination - scenes, _ := qb.queryScenes(scenesQuery, args, nil) + scenes, _ := qb.queryScenes(scenesQuery, args, nil) return scenes, countResult } From 4ac0c241c6d19f2d3a0dc8375d008d37c5d83155 Mon Sep 17 00:00:00 2001 From: Infinite Date: Sat, 18 Apr 2020 18:30:28 +0200 Subject: [PATCH 03/16] Remove old code --- pkg/models/querybuilder_scene.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkg/models/querybuilder_scene.go b/pkg/models/querybuilder_scene.go index 3473a66bdc0..4143dcd93cb 100644 --- a/pkg/models/querybuilder_scene.go +++ b/pkg/models/querybuilder_scene.go @@ -402,14 +402,6 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin havingClauses = appendClause(havingClauses, havingClause) } - //idsResult, countResult := executeFindQuery("scenes", body, args, sortAndPagination, whereClauses, havingClauses) - - //var scenes []*Scene - //for _, id := range idsResult { - //scene, _ := qb.Find(id) - //scenes = append(scenes, scene) - //} - sortAndPagination := qb.getSceneSort(findFilter) + getPagination(findFilter) if len(whereClauses) > 0 { From 8fac44e8e4a77231bc70d18604f8543ea39418f6 Mon Sep 17 00:00:00 2001 From: Infinite Date: Sun, 19 Apr 2020 14:30:57 +0200 Subject: [PATCH 04/16] Add missing column --- pkg/models/querybuilder_scene.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/models/querybuilder_scene.go b/pkg/models/querybuilder_scene.go index 4143dcd93cb..ff6f8e9f6d2 100644 --- a/pkg/models/querybuilder_scene.go +++ b/pkg/models/querybuilder_scene.go @@ -12,6 +12,7 @@ import ( var sceneMetadataColumns = []string{ "id", + "path", "checksum", "title", "details", From a739e0e21e38c8fd7f98e9bc54c73eec90ab83d7 Mon Sep 17 00:00:00 2001 From: Infinite Date: Sun, 19 Apr 2020 14:35:29 +0200 Subject: [PATCH 05/16] Add o_counter column --- pkg/models/querybuilder_scene.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/models/querybuilder_scene.go b/pkg/models/querybuilder_scene.go index ff6f8e9f6d2..049d794bd68 100644 --- a/pkg/models/querybuilder_scene.go +++ b/pkg/models/querybuilder_scene.go @@ -12,7 +12,7 @@ import ( var sceneMetadataColumns = []string{ "id", - "path", + "path", "checksum", "title", "details", @@ -31,6 +31,7 @@ var sceneMetadataColumns = []string{ "studio_id", "created_at", "updated_at", + "o_counter", } func selectSceneMetadata() string { From e3ba50460e10f54ef35e7b3d09fced7e6d9a5ace Mon Sep 17 00:00:00 2001 From: Infinite Date: Mon, 20 Apr 2020 19:12:03 +0200 Subject: [PATCH 06/16] Re-add scenes join to performers query --- pkg/models/querybuilder_performer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/models/querybuilder_performer.go b/pkg/models/querybuilder_performer.go index b5c91f6c0b0..70b5610aaf8 100644 --- a/pkg/models/querybuilder_performer.go +++ b/pkg/models/querybuilder_performer.go @@ -180,6 +180,7 @@ func (qb *PerformerQueryBuilder) Query(performerFilter *PerformerFilterType, fin query.body = selectPerformerMetadata() query.body += ` left join performers_scenes as scenes_join on scenes_join.performer_id = performers.id + left join scenes on scenes_join.scene_id = scenes.id ` if q := findFilter.Q; q != nil && *q != "" { From 680ef01b939102a8a66b9dffd9236132fbaa2a2d Mon Sep 17 00:00:00 2001 From: Infinite Date: Tue, 21 Apr 2020 11:42:13 +0200 Subject: [PATCH 07/16] Revert change to scenes qb.All --- pkg/models/querybuilder_scene.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/models/querybuilder_scene.go b/pkg/models/querybuilder_scene.go index 049d794bd68..58106668a25 100644 --- a/pkg/models/querybuilder_scene.go +++ b/pkg/models/querybuilder_scene.go @@ -264,7 +264,7 @@ func (qb *SceneQueryBuilder) Wall(q *string) ([]*Scene, error) { } func (qb *SceneQueryBuilder) All() ([]*Scene, error) { - return qb.queryScenes(selectSceneMetadata()+qb.getSceneSort(nil), nil, nil) + return qb.queryScenes(selectAll("scenes")+qb.getSceneSort(nil), nil, nil) } func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *FindFilterType) ([]*Scene, int) { From 8587fcc31260176ee879b6c0e00d7773010c72bf Mon Sep 17 00:00:00 2001 From: Infinite Date: Wed, 29 Apr 2020 09:34:43 +0200 Subject: [PATCH 08/16] Revert column changes --- pkg/api/routes_performer.go | 2 +- pkg/api/routes_scene.go | 2 +- pkg/models/querybuilder_performer.go | 49 +----------------------- pkg/models/querybuilder_scene.go | 57 ++++------------------------ 4 files changed, 12 insertions(+), 98 deletions(-) diff --git a/pkg/api/routes_performer.go b/pkg/api/routes_performer.go index f6d5cc13ec8..57958998b4d 100644 --- a/pkg/api/routes_performer.go +++ b/pkg/api/routes_performer.go @@ -48,7 +48,7 @@ func PerformerCtx(next http.Handler) http.Handler { } qb := models.NewPerformerQueryBuilder() - performer, err := qb.FindWithAllColumns(performerID) + performer, err := qb.Find(performerID) if err != nil { http.Error(w, http.StatusText(404), 404) return diff --git a/pkg/api/routes_scene.go b/pkg/api/routes_scene.go index d8c0cfe24b3..171d868b48c 100644 --- a/pkg/api/routes_scene.go +++ b/pkg/api/routes_scene.go @@ -280,7 +280,7 @@ func SceneCtx(next http.Handler) http.Handler { if sceneID == 0 { scene, err = qb.FindByChecksum(sceneIdentifierQueryParam) } else { - scene, err = qb.FindWithAllColumns(sceneID) + scene, err = qb.Find(sceneID) } if err != nil { diff --git a/pkg/models/querybuilder_performer.go b/pkg/models/querybuilder_performer.go index 70b5610aaf8..560ace25975 100644 --- a/pkg/models/querybuilder_performer.go +++ b/pkg/models/querybuilder_performer.go @@ -10,41 +10,6 @@ import ( "github.com/stashapp/stash/pkg/database" ) -var metadataColumns = []string{ - "id", - "checksum", - "name", - "gender", - "url", - "twitter", - "instagram", - "birthdate", - "ethnicity", - "country", - "eye_color", - "height", - "measurements", - "fake_tits", - "career_length", - "tattoos", - "piercings", - "aliases", - "favorite", - "created_at", - "updated_at", -} - -func selectPerformerMetadata() string { - var str strings.Builder - - for _, colName := range metadataColumns { - str.WriteString("performers." + colName + ", ") - } - - returnVal := str.String() - return "SELECT " + returnVal[:len(returnVal)-2] + " FROM performers " -} - type PerformerQueryBuilder struct{} func NewPerformerQueryBuilder() PerformerQueryBuilder { @@ -103,16 +68,6 @@ func (qb *PerformerQueryBuilder) Destroy(id string, tx *sqlx.Tx) error { } func (qb *PerformerQueryBuilder) Find(id int) (*Performer, error) { - query := selectPerformerMetadata() + "WHERE id = ? LIMIT 1" - args := []interface{}{id} - results, err := qb.queryPerformers(query, args, nil) - if err != nil || len(results) < 1 { - return nil, err - } - return results[0], nil -} - -func (qb *PerformerQueryBuilder) FindWithAllColumns(id int) (*Performer, error) { query := "SELECT * FROM performers WHERE id = ? LIMIT 1" args := []interface{}{id} results, err := qb.queryPerformers(query, args, nil) @@ -123,7 +78,7 @@ func (qb *PerformerQueryBuilder) FindWithAllColumns(id int) (*Performer, error) } func (qb *PerformerQueryBuilder) FindBySceneID(sceneID int, tx *sqlx.Tx) ([]*Performer, error) { - query := selectPerformerMetadata() + ` + query := selectAll("performers") + ` LEFT JOIN performers_scenes as scenes_join on scenes_join.performer_id = performers.id WHERE scenes_join.scene_id = ? GROUP BY performers.id @@ -177,7 +132,7 @@ func (qb *PerformerQueryBuilder) Query(performerFilter *PerformerFilterType, fin tableName: tableName, } - query.body = selectPerformerMetadata() + query.body = selectAll("performers") query.body += ` left join performers_scenes as scenes_join on scenes_join.performer_id = performers.id left join scenes on scenes_join.scene_id = scenes.id diff --git a/pkg/models/querybuilder_scene.go b/pkg/models/querybuilder_scene.go index 58106668a25..c17409f95d8 100644 --- a/pkg/models/querybuilder_scene.go +++ b/pkg/models/querybuilder_scene.go @@ -10,42 +10,7 @@ import ( "github.com/stashapp/stash/pkg/database" ) -var sceneMetadataColumns = []string{ - "id", - "path", - "checksum", - "title", - "details", - "url", - "date", - "rating", - "size", - "duration", - "video_codec", - "audio_codec", - "format", - "width", - "height", - "framerate", - "bitrate", - "studio_id", - "created_at", - "updated_at", - "o_counter", -} - -func selectSceneMetadata() string { - var str strings.Builder - - for _, colName := range sceneMetadataColumns { - str.WriteString("scenes." + colName + ", ") - } - - returnVal := str.String() - return "SELECT " + returnVal[:len(returnVal)-2] + " FROM scenes " -} - -var scenesForPerformerQuery = selectSceneMetadata() + ` +var scenesForPerformerQuery = selectAll("scenes") + ` LEFT JOIN performers_scenes as performers_join on performers_join.scene_id = scenes.id WHERE performers_join.performer_id = ? GROUP BY scenes.id @@ -57,18 +22,18 @@ WHERE performer_id = ? GROUP BY scene_id ` -var scenesForStudioQuery = selectSceneMetadata() + ` +var scenesForStudioQuery = selectAll("scenes") + ` JOIN studios ON studios.id = scenes.studio_id WHERE studios.id = ? GROUP BY scenes.id ` -var scenesForMovieQuery = selectSceneMetadata() + ` +var scenesForMovieQuery = selectAll("all") + ` LEFT JOIN movies_scenes as movies_join on movies_join.scene_id = scenes.id WHERE movies_join.movie_id = ? GROUP BY scenes.id ` -var scenesForTagQuery = selectSceneMetadata() + ` +var scenesForTagQuery = selectAll("all") + ` LEFT JOIN scenes_tags as tags_join on tags_join.scene_id = scenes.id WHERE tags_join.tag_id = ? GROUP BY scenes.id @@ -183,14 +148,8 @@ func (qb *SceneQueryBuilder) Find(id int) (*Scene, error) { return qb.find(id, nil) } -func (qb *SceneQueryBuilder) FindWithAllColumns(id int) (*Scene, error) { - query := "SELECT * FROM scenes WHERE id = ? LIMIT 1" - args := []interface{}{id} - return qb.queryScene(query, args, nil) -} - func (qb *SceneQueryBuilder) find(id int, tx *sqlx.Tx) (*Scene, error) { - query := selectSceneMetadata() + "WHERE id = ? LIMIT 1" + query := selectAll("scenes") + "WHERE id = ? LIMIT 1" args := []interface{}{id} return qb.queryScene(query, args, tx) } @@ -202,7 +161,7 @@ func (qb *SceneQueryBuilder) FindByChecksum(checksum string) (*Scene, error) { } func (qb *SceneQueryBuilder) FindByPath(path string) (*Scene, error) { - query := selectSceneMetadata() + "WHERE path = ? LIMIT 1" + query := selectAll("scenes") + "WHERE path = ? LIMIT 1" args := []interface{}{path} return qb.queryScene(query, args, nil) } @@ -259,7 +218,7 @@ func (qb *SceneQueryBuilder) Wall(q *string) ([]*Scene, error) { if q != nil { s = *q } - query := selectSceneMetadata() + "WHERE scenes.details LIKE '%" + s + "%' ORDER BY RANDOM() LIMIT 80" + query := selectAll("scenes") + "WHERE scenes.details LIKE '%" + s + "%' ORDER BY RANDOM() LIMIT 80" return qb.queryScenes(query, nil, nil) } @@ -278,7 +237,7 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin var whereClauses []string var havingClauses []string var args []interface{} - body := selectSceneMetadata() + body := selectAll("scenes") body = body + ` left join scene_markers on scene_markers.scene_id = scenes.id left join performers_scenes as performers_join on performers_join.scene_id = scenes.id From e72ee076d887b9cf6c3abaa02c0aaeb936885e63 Mon Sep 17 00:00:00 2001 From: Infinite Date: Wed, 29 Apr 2020 09:47:32 +0200 Subject: [PATCH 09/16] Revert query changes --- pkg/models/querybuilder_performer.go | 20 ++++++-------------- pkg/models/querybuilder_scene.go | 19 ++++++------------- 2 files changed, 12 insertions(+), 27 deletions(-) diff --git a/pkg/models/querybuilder_performer.go b/pkg/models/querybuilder_performer.go index 560ace25975..cc9a0554cfa 100644 --- a/pkg/models/querybuilder_performer.go +++ b/pkg/models/querybuilder_performer.go @@ -3,7 +3,6 @@ package models import ( "database/sql" "strconv" - "strings" "time" "github.com/jmoiron/sqlx" @@ -132,7 +131,7 @@ func (qb *PerformerQueryBuilder) Query(performerFilter *PerformerFilterType, fin tableName: tableName, } - query.body = selectAll("performers") + query.body = selectDistinctIDs(tableName) query.body += ` left join performers_scenes as scenes_join on scenes_join.performer_id = performers.id left join scenes on scenes_join.scene_id = scenes.id @@ -195,20 +194,13 @@ func (qb *PerformerQueryBuilder) Query(performerFilter *PerformerFilterType, fin handleStringCriterion(tableName+".aliases", performerFilter.Aliases, &query) query.sortAndPagination = qb.getPerformerSort(findFilter) + getPagination(findFilter) + idsResult, countResult := query.executeFind() - if len(query.whereClauses) > 0 { - query.body += " WHERE " + strings.Join(query.whereClauses, " AND ") // TODO handle AND or OR + var performers []*Performer + for _, id := range idsResult { + performer, _ := qb.Find(id) + performers = append(performers, performer) } - query.body += " GROUP BY " + tableName + ".id " - if len(query.havingClauses) > 0 { - query.body += " HAVING " + strings.Join(query.havingClauses, " AND ") // TODO handle AND or OR - } - - countQuery := buildCountQuery(query.body) - countResult, _ := runCountQuery(countQuery, query.args) - - performersQuery := query.body + query.sortAndPagination - performers, _ := qb.queryPerformers(performersQuery, query.args, nil) return performers, countResult } diff --git a/pkg/models/querybuilder_scene.go b/pkg/models/querybuilder_scene.go index c17409f95d8..44f10c33ca5 100644 --- a/pkg/models/querybuilder_scene.go +++ b/pkg/models/querybuilder_scene.go @@ -237,7 +237,7 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin var whereClauses []string var havingClauses []string var args []interface{} - body := selectAll("scenes") + body := selectDistinctIDs("scenes") body = body + ` left join scene_markers on scene_markers.scene_id = scenes.id left join performers_scenes as performers_join on performers_join.scene_id = scenes.id @@ -364,21 +364,14 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin } sortAndPagination := qb.getSceneSort(findFilter) + getPagination(findFilter) + idsResult, countResult := executeFindQuery("scenes", body, args, sortAndPagination, whereClauses, havingClauses) - if len(whereClauses) > 0 { - body += " WHERE " + strings.Join(whereClauses, " AND ") // TODO handle AND or OR - } - body += " GROUP BY scenes.id " - if len(havingClauses) > 0 { - body += " HAVING " + strings.Join(havingClauses, " AND ") // TODO handle AND or OR + var scenes []*Scene + for _, id := range idsResult { + scene, _ := qb.Find(id) + scenes = append(scenes, scene) } - countQuery := buildCountQuery(body) - countResult, _ := runCountQuery(countQuery, args) - - scenesQuery := body + sortAndPagination - scenes, _ := qb.queryScenes(scenesQuery, args, nil) - return scenes, countResult } From bbbe2103b23d355ec9eb98369421baa38bed5509 Mon Sep 17 00:00:00 2001 From: Infinite Date: Wed, 29 Apr 2020 10:41:05 +0200 Subject: [PATCH 10/16] Remove slow and largely pointless groupbys --- pkg/models/querybuilder_performer.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/models/querybuilder_performer.go b/pkg/models/querybuilder_performer.go index cc9a0554cfa..ae140371907 100644 --- a/pkg/models/querybuilder_performer.go +++ b/pkg/models/querybuilder_performer.go @@ -80,7 +80,6 @@ func (qb *PerformerQueryBuilder) FindBySceneID(sceneID int, tx *sqlx.Tx) ([]*Per query := selectAll("performers") + ` LEFT JOIN performers_scenes as scenes_join on scenes_join.performer_id = performers.id WHERE scenes_join.scene_id = ? - GROUP BY performers.id ` args := []interface{}{sceneID} return qb.queryPerformers(query, args, tx) @@ -91,7 +90,6 @@ func (qb *PerformerQueryBuilder) FindNameBySceneID(sceneID int, tx *sqlx.Tx) ([] SELECT performers.name FROM performers LEFT JOIN performers_scenes as scenes_join on scenes_join.performer_id = performers.id WHERE scenes_join.scene_id = ? - GROUP BY performers.name ` args := []interface{}{sceneID} return qb.queryPerformers(query, args, tx) From 2079dc5b6230f61985ed63c4c2d30f569e215e6e Mon Sep 17 00:00:00 2001 From: Infinite Date: Wed, 29 Apr 2020 11:47:00 +0200 Subject: [PATCH 11/16] Change scene.query to use querybuilder --- pkg/models/querybuilder_performer.go | 2 +- pkg/models/querybuilder_scene.go | 90 ++++++++++++++-------------- pkg/models/querybuilder_sql.go | 12 +++- 3 files changed, 57 insertions(+), 47 deletions(-) diff --git a/pkg/models/querybuilder_performer.go b/pkg/models/querybuilder_performer.go index ae140371907..ffe3e5d3a10 100644 --- a/pkg/models/querybuilder_performer.go +++ b/pkg/models/querybuilder_performer.go @@ -192,7 +192,7 @@ func (qb *PerformerQueryBuilder) Query(performerFilter *PerformerFilterType, fin handleStringCriterion(tableName+".aliases", performerFilter.Aliases, &query) query.sortAndPagination = qb.getPerformerSort(findFilter) + getPagination(findFilter) - idsResult, countResult := query.executeFind() + idsResult, countResult := query.executeFind() var performers []*Performer for _, id := range idsResult { diff --git a/pkg/models/querybuilder_scene.go b/pkg/models/querybuilder_scene.go index 44f10c33ca5..8a035c4f7cb 100644 --- a/pkg/models/querybuilder_scene.go +++ b/pkg/models/querybuilder_scene.go @@ -234,11 +234,13 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin findFilter = &FindFilterType{} } - var whereClauses []string - var havingClauses []string - var args []interface{} - body := selectDistinctIDs("scenes") - body = body + ` + tableName := "scenes" + query := queryBuilder{ + tableName: tableName, + } + + query.body = selectDistinctIDs(tableName) + query.body += ` left join scene_markers on scene_markers.scene_id = scenes.id left join performers_scenes as performers_join on performers_join.scene_id = scenes.id left join movies_scenes as movies_join on movies_join.scene_id = scenes.id @@ -250,121 +252,121 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin if q := findFilter.Q; q != nil && *q != "" { searchColumns := []string{"scenes.title", "scenes.details", "scenes.path", "scenes.checksum", "scene_markers.title"} clause, thisArgs := getSearchBinding(searchColumns, *q, false) - whereClauses = append(whereClauses, clause) - args = append(args, thisArgs...) + query.addWhere(clause) + query.addArg(thisArgs...) } if rating := sceneFilter.Rating; rating != nil { clause, count := getIntCriterionWhereClause("scenes.rating", *sceneFilter.Rating) - whereClauses = append(whereClauses, clause) + query.addWhere(clause) if count == 1 { - args = append(args, sceneFilter.Rating.Value) + query.addArg(sceneFilter.Rating.Value) } } if oCounter := sceneFilter.OCounter; oCounter != nil { clause, count := getIntCriterionWhereClause("scenes.o_counter", *sceneFilter.OCounter) - whereClauses = append(whereClauses, clause) + query.addWhere(clause) if count == 1 { - args = append(args, sceneFilter.OCounter.Value) + query.addArg(sceneFilter.OCounter.Value) } } if durationFilter := sceneFilter.Duration; durationFilter != nil { clause, thisArgs := getDurationWhereClause(*durationFilter) - whereClauses = append(whereClauses, clause) - args = append(args, thisArgs...) + query.addWhere(clause) + query.addArg(thisArgs...) } if resolutionFilter := sceneFilter.Resolution; resolutionFilter != nil { if resolution := resolutionFilter.String(); resolutionFilter.IsValid() { switch resolution { case "LOW": - whereClauses = append(whereClauses, "scenes.height < 480") + query.addWhere("scenes.height < 480") case "STANDARD": - whereClauses = append(whereClauses, "(scenes.height >= 480 AND scenes.height < 720)") + query.addWhere("(scenes.height >= 480 AND scenes.height < 720)") case "STANDARD_HD": - whereClauses = append(whereClauses, "(scenes.height >= 720 AND scenes.height < 1080)") + query.addWhere("(scenes.height >= 720 AND scenes.height < 1080)") case "FULL_HD": - whereClauses = append(whereClauses, "(scenes.height >= 1080 AND scenes.height < 2160)") + query.addWhere("(scenes.height >= 1080 AND scenes.height < 2160)") case "FOUR_K": - whereClauses = append(whereClauses, "scenes.height >= 2160") + query.addWhere("scenes.height >= 2160") } } } if hasMarkersFilter := sceneFilter.HasMarkers; hasMarkersFilter != nil { if strings.Compare(*hasMarkersFilter, "true") == 0 { - havingClauses = append(havingClauses, "count(scene_markers.scene_id) > 0") + query.addHaving("count(scene_markers.scene_id) > 0") } else { - whereClauses = append(whereClauses, "scene_markers.id IS NULL") + query.addWhere("scene_markers.id IS NULL") } } if isMissingFilter := sceneFilter.IsMissing; isMissingFilter != nil && *isMissingFilter != "" { switch *isMissingFilter { case "gallery": - whereClauses = append(whereClauses, "gallery.scene_id IS NULL") + query.addWhere("gallery.scene_id IS NULL") case "studio": - whereClauses = append(whereClauses, "scenes.studio_id IS NULL") + query.addWhere("scenes.studio_id IS NULL") case "movie": - whereClauses = append(whereClauses, "movies_join.scene_id IS NULL") + query.addWhere("movies_join.scene_id IS NULL") case "performers": - whereClauses = append(whereClauses, "performers_join.scene_id IS NULL") + query.addWhere("performers_join.scene_id IS NULL") case "date": - whereClauses = append(whereClauses, "scenes.date IS \"\" OR scenes.date IS \"0001-01-01\"") + query.addWhere("scenes.date IS \"\" OR scenes.date IS \"0001-01-01\"") case "tags": - whereClauses = append(whereClauses, "tags_join.scene_id IS NULL") + query.addWhere("tags_join.scene_id IS NULL") default: - whereClauses = append(whereClauses, "scenes."+*isMissingFilter+" IS NULL") + query.addWhere("scenes." + *isMissingFilter + " IS NULL") } } if tagsFilter := sceneFilter.Tags; tagsFilter != nil && len(tagsFilter.Value) > 0 { for _, tagID := range tagsFilter.Value { - args = append(args, tagID) + query.addArg(tagID) } - body += " LEFT JOIN tags on tags_join.tag_id = tags.id" + query.body += " LEFT JOIN tags on tags_join.tag_id = tags.id" whereClause, havingClause := getMultiCriterionClause("tags", "scenes_tags", "tag_id", tagsFilter) - whereClauses = appendClause(whereClauses, whereClause) - havingClauses = appendClause(havingClauses, havingClause) + query.addWhere(whereClause) + query.addHaving(havingClause) } if performersFilter := sceneFilter.Performers; performersFilter != nil && len(performersFilter.Value) > 0 { for _, performerID := range performersFilter.Value { - args = append(args, performerID) + query.addArg(performerID) } - body += " LEFT JOIN performers ON performers_join.performer_id = performers.id" + query.body += " LEFT JOIN performers ON performers_join.performer_id = performers.id" whereClause, havingClause := getMultiCriterionClause("performers", "performers_scenes", "performer_id", performersFilter) - whereClauses = appendClause(whereClauses, whereClause) - havingClauses = appendClause(havingClauses, havingClause) + query.addWhere(whereClause) + query.addHaving(havingClause) } if studiosFilter := sceneFilter.Studios; studiosFilter != nil && len(studiosFilter.Value) > 0 { for _, studioID := range studiosFilter.Value { - args = append(args, studioID) + query.addArg(studioID) } whereClause, havingClause := getMultiCriterionClause("studio", "", "studio_id", studiosFilter) - whereClauses = appendClause(whereClauses, whereClause) - havingClauses = appendClause(havingClauses, havingClause) + query.addWhere(whereClause) + query.addHaving(havingClause) } if moviesFilter := sceneFilter.Movies; moviesFilter != nil && len(moviesFilter.Value) > 0 { for _, movieID := range moviesFilter.Value { - args = append(args, movieID) + query.addArg(movieID) } - body += " LEFT JOIN movies ON movies_join.movie_id = movies.id" + query.body += " LEFT JOIN movies ON movies_join.movie_id = movies.id" whereClause, havingClause := getMultiCriterionClause("movies", "movies_scenes", "movie_id", moviesFilter) - whereClauses = appendClause(whereClauses, whereClause) - havingClauses = appendClause(havingClauses, havingClause) + query.addWhere(whereClause) + query.addHaving(havingClause) } - sortAndPagination := qb.getSceneSort(findFilter) + getPagination(findFilter) - idsResult, countResult := executeFindQuery("scenes", body, args, sortAndPagination, whereClauses, havingClauses) + query.sortAndPagination = qb.getSceneSort(findFilter) + getPagination(findFilter) + idsResult, countResult := query.executeFind() var scenes []*Scene for _, id := range idsResult { diff --git a/pkg/models/querybuilder_sql.go b/pkg/models/querybuilder_sql.go index 3acbd7d441f..2944462c21a 100644 --- a/pkg/models/querybuilder_sql.go +++ b/pkg/models/querybuilder_sql.go @@ -29,11 +29,19 @@ func (qb queryBuilder) executeFind() ([]int, int) { } func (qb *queryBuilder) addWhere(clauses ...string) { - qb.whereClauses = append(qb.whereClauses, clauses...) + for _, clause := range clauses { + if len(clause) > 0 { + qb.whereClauses = append(qb.whereClauses, clauses...) + } + } } func (qb *queryBuilder) addHaving(clauses ...string) { - qb.havingClauses = append(qb.havingClauses, clauses...) + for _, clause := range clauses { + if len(clause) > 0 { + qb.havingClauses = append(qb.havingClauses, clause) + } + } } func (qb *queryBuilder) addArg(args ...interface{}) { From 68d834f8c5c94db8ca4379a71ee8791e5817c665 Mon Sep 17 00:00:00 2001 From: Infinite Date: Mon, 4 May 2020 17:20:29 +0200 Subject: [PATCH 12/16] Fix Movie.FindBySceneId --- pkg/models/querybuilder_movies.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/models/querybuilder_movies.go b/pkg/models/querybuilder_movies.go index d5a3cdfbe77..9c35017a17e 100644 --- a/pkg/models/querybuilder_movies.go +++ b/pkg/models/querybuilder_movies.go @@ -76,7 +76,7 @@ func (qb *MovieQueryBuilder) FindBySceneID(sceneID int, tx *sqlx.Tx) ([]*Movie, query := ` SELECT movies.* FROM movies LEFT JOIN movies_scenes as scenes_join on scenes_join.movie_id = movies.id - WHERE scenes_join.scenes.id = ? + WHERE scenes_join.scene_id = ? GROUP BY movies.id ` args := []interface{}{sceneID} From f962e4c2b71032e63edfde1a53de175b03f855a7 Mon Sep 17 00:00:00 2001 From: Infinite Date: Fri, 8 May 2020 15:45:27 +0200 Subject: [PATCH 13/16] Add additional tests from @WithoutPants and fix bug --- pkg/models/querybuilder_scene.go | 23 ++--- pkg/models/querybuilder_scene_test.go | 129 ++++++++++++++++++++++++-- 2 files changed, 135 insertions(+), 17 deletions(-) diff --git a/pkg/models/querybuilder_scene.go b/pkg/models/querybuilder_scene.go index 8a035c4f7cb..f683649e245 100644 --- a/pkg/models/querybuilder_scene.go +++ b/pkg/models/querybuilder_scene.go @@ -10,7 +10,9 @@ import ( "github.com/stashapp/stash/pkg/database" ) -var scenesForPerformerQuery = selectAll("scenes") + ` +const TABLE_NAME = "scenes" + +var scenesForPerformerQuery = selectAll(TABLE_NAME) + ` LEFT JOIN performers_scenes as performers_join on performers_join.scene_id = scenes.id WHERE performers_join.performer_id = ? GROUP BY scenes.id @@ -22,18 +24,18 @@ WHERE performer_id = ? GROUP BY scene_id ` -var scenesForStudioQuery = selectAll("scenes") + ` +var scenesForStudioQuery = selectAll(TABLE_NAME) + ` JOIN studios ON studios.id = scenes.studio_id WHERE studios.id = ? GROUP BY scenes.id ` -var scenesForMovieQuery = selectAll("all") + ` +var scenesForMovieQuery = selectAll(TABLE_NAME) + ` LEFT JOIN movies_scenes as movies_join on movies_join.scene_id = scenes.id WHERE movies_join.movie_id = ? GROUP BY scenes.id ` -var scenesForTagQuery = selectAll("all") + ` +var scenesForTagQuery = selectAll(TABLE_NAME) + ` LEFT JOIN scenes_tags as tags_join on tags_join.scene_id = scenes.id WHERE tags_join.tag_id = ? GROUP BY scenes.id @@ -149,7 +151,7 @@ func (qb *SceneQueryBuilder) Find(id int) (*Scene, error) { } func (qb *SceneQueryBuilder) find(id int, tx *sqlx.Tx) (*Scene, error) { - query := selectAll("scenes") + "WHERE id = ? LIMIT 1" + query := selectAll(TABLE_NAME) + "WHERE id = ? LIMIT 1" args := []interface{}{id} return qb.queryScene(query, args, tx) } @@ -161,7 +163,7 @@ func (qb *SceneQueryBuilder) FindByChecksum(checksum string) (*Scene, error) { } func (qb *SceneQueryBuilder) FindByPath(path string) (*Scene, error) { - query := selectAll("scenes") + "WHERE path = ? LIMIT 1" + query := selectAll(TABLE_NAME) + "WHERE path = ? LIMIT 1" args := []interface{}{path} return qb.queryScene(query, args, nil) } @@ -218,12 +220,12 @@ func (qb *SceneQueryBuilder) Wall(q *string) ([]*Scene, error) { if q != nil { s = *q } - query := selectAll("scenes") + "WHERE scenes.details LIKE '%" + s + "%' ORDER BY RANDOM() LIMIT 80" + query := selectAll(TABLE_NAME) + "WHERE scenes.details LIKE '%" + s + "%' ORDER BY RANDOM() LIMIT 80" return qb.queryScenes(query, nil, nil) } func (qb *SceneQueryBuilder) All() ([]*Scene, error) { - return qb.queryScenes(selectAll("scenes")+qb.getSceneSort(nil), nil, nil) + return qb.queryScenes(selectAll(TABLE_NAME)+qb.getSceneSort(nil), nil, nil) } func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *FindFilterType) ([]*Scene, int) { @@ -234,12 +236,11 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin findFilter = &FindFilterType{} } - tableName := "scenes" query := queryBuilder{ - tableName: tableName, + tableName: TABLE_NAME, } - query.body = selectDistinctIDs(tableName) + query.body = selectDistinctIDs(TABLE_NAME) query.body += ` left join scene_markers on scene_markers.scene_id = scenes.id left join performers_scenes as performers_join on performers_join.scene_id = scenes.id diff --git a/pkg/models/querybuilder_scene_test.go b/pkg/models/querybuilder_scene_test.go index 06a03a26ff0..df6b0e8177f 100644 --- a/pkg/models/querybuilder_scene_test.go +++ b/pkg/models/querybuilder_scene_test.go @@ -735,18 +735,135 @@ func TestSceneQueryPagination(t *testing.T) { assert.Equal(t, secondID, scenes[1].ID) } +func TestSceneCountByTagID(t *testing.T) { + sqb := models.NewSceneQueryBuilder() + + sceneCount, err := sqb.CountByTagID(tagIDs[tagIdxWithScene]) + + if err != nil { + t.Fatalf("error calling CountByTagID: %s", err.Error()) + } + + assert.Equal(t, 1, sceneCount) + + sceneCount, err = sqb.CountByTagID(0) + + if err != nil { + t.Fatalf("error calling CountByTagID: %s", err.Error()) + } + + assert.Equal(t, 0, sceneCount) +} + +func TestSceneCountByMovieID(t *testing.T) { + sqb := models.NewSceneQueryBuilder() + + sceneCount, err := sqb.CountByMovieID(movieIDs[movieIdxWithScene]) + + if err != nil { + t.Fatalf("error calling CountByMovieID: %s", err.Error()) + } + + assert.Equal(t, 1, sceneCount) + + sceneCount, err = sqb.CountByMovieID(0) + + if err != nil { + t.Fatalf("error calling CountByMovieID: %s", err.Error()) + } + + assert.Equal(t, 0, sceneCount) +} + +func TestSceneCountByStudioID(t *testing.T) { + sqb := models.NewSceneQueryBuilder() + + sceneCount, err := sqb.CountByStudioID(studioIDs[studioIdxWithScene]) + + if err != nil { + t.Fatalf("error calling CountByStudioID: %s", err.Error()) + } + + assert.Equal(t, 1, sceneCount) + + sceneCount, err = sqb.CountByStudioID(0) + + if err != nil { + t.Fatalf("error calling CountByStudioID: %s", err.Error()) + } + + assert.Equal(t, 0, sceneCount) +} + +func TestFindByMovieID(t *testing.T) { + sqb := models.NewSceneQueryBuilder() + + scenes, err := sqb.FindByMovieID(movieIDs[movieIdxWithScene]) + + if err != nil { + t.Fatalf("error calling FindByMovieID: %s", err.Error()) + } + + assert.Len(t, scenes, 1) + assert.Equal(t, sceneIDs[sceneIdxWithMovie], scenes[0].ID) + + scenes, err = sqb.FindByMovieID(0) + + if err != nil { + t.Fatalf("error calling FindByMovieID: %s", err.Error()) + } + + assert.Len(t, scenes, 0) +} + +func TestFindByPerformerID(t *testing.T) { + sqb := models.NewSceneQueryBuilder() + + scenes, err := sqb.FindByPerformerID(performerIDs[performerIdxWithScene]) + + if err != nil { + t.Fatalf("error calling FindByPerformerID: %s", err.Error()) + } + + assert.Len(t, scenes, 1) + assert.Equal(t, sceneIDs[sceneIdxWithPerformer], scenes[0].ID) + + scenes, err = sqb.FindByPerformerID(0) + + if err != nil { + t.Fatalf("error calling FindByPerformerID: %s", err.Error()) + } + + assert.Len(t, scenes, 0) +} + +func TestFindByStudioID(t *testing.T) { + sqb := models.NewSceneQueryBuilder() + + scenes, err := sqb.FindByStudioID(performerIDs[studioIdxWithScene]) + + if err != nil { + t.Fatalf("error calling FindByStudioID: %s", err.Error()) + } + + assert.Len(t, scenes, 1) + assert.Equal(t, sceneIDs[sceneIdxWithStudio], scenes[0].ID) + + scenes, err = sqb.FindByStudioID(0) + + if err != nil { + t.Fatalf("error calling FindByStudioID: %s", err.Error()) + } + + assert.Len(t, scenes, 0) +} + // TODO Update // TODO IncrementOCounter // TODO DecrementOCounter // TODO ResetOCounter // TODO Destroy // TODO FindByChecksum -// TODO FindByPerformerID -// TODO FindByStudioID -// TODO FindByMovieID -// TODO CountByMovieID // TODO Count // TODO SizeCount -// TODO CountByStudioID -// TODO CountByTagID // TODO All From 7681b2a1c3e8d8ce07beb5b34f1d54524dc45890 Mon Sep 17 00:00:00 2001 From: Infinite Date: Sat, 9 May 2020 08:42:21 +0200 Subject: [PATCH 14/16] Rename sceneTable constZ --- go.mod | 1 - pkg/models/querybuilder_scene.go | 22 +++++++++++----------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index d92f27d203c..45ecb910277 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,6 @@ require ( github.com/gorilla/sessions v1.2.0 github.com/gorilla/websocket v1.4.0 github.com/h2non/filetype v1.0.8 - github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/jmoiron/sqlx v1.2.0 github.com/json-iterator/go v1.1.9 diff --git a/pkg/models/querybuilder_scene.go b/pkg/models/querybuilder_scene.go index f683649e245..2c2aa8a392d 100644 --- a/pkg/models/querybuilder_scene.go +++ b/pkg/models/querybuilder_scene.go @@ -10,9 +10,9 @@ import ( "github.com/stashapp/stash/pkg/database" ) -const TABLE_NAME = "scenes" +const sceneTable = "scenes" -var scenesForPerformerQuery = selectAll(TABLE_NAME) + ` +var scenesForPerformerQuery = selectAll(sceneTable) + ` LEFT JOIN performers_scenes as performers_join on performers_join.scene_id = scenes.id WHERE performers_join.performer_id = ? GROUP BY scenes.id @@ -24,18 +24,18 @@ WHERE performer_id = ? GROUP BY scene_id ` -var scenesForStudioQuery = selectAll(TABLE_NAME) + ` +var scenesForStudioQuery = selectAll(sceneTable) + ` JOIN studios ON studios.id = scenes.studio_id WHERE studios.id = ? GROUP BY scenes.id ` -var scenesForMovieQuery = selectAll(TABLE_NAME) + ` +var scenesForMovieQuery = selectAll(sceneTable) + ` LEFT JOIN movies_scenes as movies_join on movies_join.scene_id = scenes.id WHERE movies_join.movie_id = ? GROUP BY scenes.id ` -var scenesForTagQuery = selectAll(TABLE_NAME) + ` +var scenesForTagQuery = selectAll(sceneTable) + ` LEFT JOIN scenes_tags as tags_join on tags_join.scene_id = scenes.id WHERE tags_join.tag_id = ? GROUP BY scenes.id @@ -151,7 +151,7 @@ func (qb *SceneQueryBuilder) Find(id int) (*Scene, error) { } func (qb *SceneQueryBuilder) find(id int, tx *sqlx.Tx) (*Scene, error) { - query := selectAll(TABLE_NAME) + "WHERE id = ? LIMIT 1" + query := selectAll(sceneTable) + "WHERE id = ? LIMIT 1" args := []interface{}{id} return qb.queryScene(query, args, tx) } @@ -163,7 +163,7 @@ func (qb *SceneQueryBuilder) FindByChecksum(checksum string) (*Scene, error) { } func (qb *SceneQueryBuilder) FindByPath(path string) (*Scene, error) { - query := selectAll(TABLE_NAME) + "WHERE path = ? LIMIT 1" + query := selectAll(sceneTable) + "WHERE path = ? LIMIT 1" args := []interface{}{path} return qb.queryScene(query, args, nil) } @@ -220,12 +220,12 @@ func (qb *SceneQueryBuilder) Wall(q *string) ([]*Scene, error) { if q != nil { s = *q } - query := selectAll(TABLE_NAME) + "WHERE scenes.details LIKE '%" + s + "%' ORDER BY RANDOM() LIMIT 80" + query := selectAll(sceneTable) + "WHERE scenes.details LIKE '%" + s + "%' ORDER BY RANDOM() LIMIT 80" return qb.queryScenes(query, nil, nil) } func (qb *SceneQueryBuilder) All() ([]*Scene, error) { - return qb.queryScenes(selectAll(TABLE_NAME)+qb.getSceneSort(nil), nil, nil) + return qb.queryScenes(selectAll(sceneTable)+qb.getSceneSort(nil), nil, nil) } func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *FindFilterType) ([]*Scene, int) { @@ -237,10 +237,10 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin } query := queryBuilder{ - tableName: TABLE_NAME, + tableName: sceneTable, } - query.body = selectDistinctIDs(TABLE_NAME) + query.body = selectDistinctIDs(sceneTable) query.body += ` left join scene_markers on scene_markers.scene_id = scenes.id left join performers_scenes as performers_join on performers_join.scene_id = scenes.id From 832cda53dfbe4393d5e7e25e65b1a59dac34adcb Mon Sep 17 00:00:00 2001 From: Infinite Date: Sat, 9 May 2020 08:53:28 +0200 Subject: [PATCH 15/16] Revert go.mod --- go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/go.mod b/go.mod index 45ecb910277..d92f27d203c 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/gorilla/sessions v1.2.0 github.com/gorilla/websocket v1.4.0 github.com/h2non/filetype v1.0.8 + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/jmoiron/sqlx v1.2.0 github.com/json-iterator/go v1.1.9 From 50921cdbee688093b4b599f07025a1513c986938 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Mon, 11 May 2020 14:52:23 +1000 Subject: [PATCH 16/16] Add test for FindNameBySceneID --- pkg/models/querybuilder_performer_test.go | 25 ++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/pkg/models/querybuilder_performer_test.go b/pkg/models/querybuilder_performer_test.go index 32f6111a7c5..a371e3c57cb 100644 --- a/pkg/models/querybuilder_performer_test.go +++ b/pkg/models/querybuilder_performer_test.go @@ -34,10 +34,33 @@ func TestPerformerFindBySceneID(t *testing.T) { assert.Equal(t, 0, len(performers)) } +func TestPerformerFindNameBySceneID(t *testing.T) { + pqb := models.NewPerformerQueryBuilder() + sceneID := sceneIDs[sceneIdxWithPerformer] + + performers, err := pqb.FindNameBySceneID(sceneID, nil) + + if err != nil { + t.Fatalf("Error finding performer: %s", err.Error()) + } + + assert.Equal(t, 1, len(performers)) + performer := performers[0] + + assert.Equal(t, getPerformerStringValue(performerIdxWithScene, "Name"), performer.Name.String) + + performers, err = pqb.FindBySceneID(0, nil) + + if err != nil { + t.Fatalf("Error finding performer: %s", err.Error()) + } + + assert.Equal(t, 0, len(performers)) +} + // TODO Update // TODO Destroy // TODO Find -// TODO FindNameBySceneID // TODO FindByNames // TODO Count // TODO All