From d26ecbfc2fd883043758f9c5389a14f9082f6020 Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Tue, 20 Aug 2024 02:11:43 -0400 Subject: [PATCH 1/4] Parse more metadata badges for SearchVideos --- src/invidious/helpers/serialized_yt_data.cr | 14 ++++++++ src/invidious/routes/feeds.cr | 7 ++++ src/invidious/yt_backend/extractors.cr | 40 +++++++++++++++++++-- 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index 463d5557b..c0ca789e0 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -13,6 +13,13 @@ struct SearchVideo property premium : Bool property premiere_timestamp : Time? property author_verified : Bool + property is_new : Bool + property is_4k : Bool + property is_8k : Bool + property is_vr180 : Bool + property is_vr360 : Bool + property is_3d : Bool + property has_captions : Bool def to_xml(auto_generated, query_params, xml : XML::Builder) query_params["v"] = self.id @@ -95,6 +102,13 @@ struct SearchVideo if self.premiere_timestamp json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix end + json.field "isNew", self.is_new + json.field "is4k", self.is_4k + json.field "is8k", self.is_8k + json.field "isVR180", is_vr180 + json.field "isVR360", is_vr360 + json.field "is3d", is_3d + json.field "hasCaptions", self.has_captions end end diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index e20a7139e..b76aeb467 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -197,6 +197,13 @@ module Invidious::Routes::Feeds premium: false, premiere_timestamp: nil, author_verified: false, + is_new: false, + is_4k: false, + is_8k: false, + is_vr180: false, + is_vr360: false, + is_3d: false, + has_captions: false, }) end diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 38dc2c04b..48c2155be 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -110,6 +110,13 @@ private module Parsers live_now = false premium = false + is_new = false + is_4k = false + is_8k = false + is_vr180 = false + is_vr360 = false + is_3d = false + has_captions = false premiere_timestamp = item_contents.dig?("upcomingEventData", "startTime").try { |t| Time.unix(t.as_s.to_i64) } @@ -118,12 +125,25 @@ private module Parsers case b["label"].as_s when "LIVE NOW" live_now = true - when "New", "4K", "CC" - # TODO + when "New" + is_new = true + when "4K" + is_4k = true + when "8K" + is_8k = true + when "VR180" + is_vr180 = true + when "360°" + is_vr360 = true + when "3D" + is_3d = true + when "CC" + has_captions = true when "Premium" # TODO: Potentially available as item_contents["topStandaloneBadge"]["metadataBadgeRenderer"] premium = true - else nil # Ignore + else # Ignore + puts b["label"].as_s end end @@ -140,6 +160,13 @@ private module Parsers premium: premium, premiere_timestamp: premiere_timestamp, author_verified: author_verified, + is_new: is_new, + is_4k: is_4k, + is_8k: is_8k, + is_vr180: is_vr180, + is_vr360: is_vr360, + is_3d: is_3d, + has_captions: has_captions, }) end @@ -567,6 +594,13 @@ private module Parsers premium: false, premiere_timestamp: Time.unix(0), author_verified: false, + is_new: false, + is_4k: false, + is_8k: false, + is_vr180: false, + is_vr360: false, + is_3d: false, + has_captions: false, }) end From a9cc0125427aa2030a050d60218845d0791a015c Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Sat, 24 Aug 2024 18:00:59 -0400 Subject: [PATCH 2/4] switch to enum flag instead of adding lots of properties to SearchVideo --- src/invidious/channels/channels.cr | 4 +- src/invidious/helpers/serialized_yt_data.cr | 41 +++++++++------- src/invidious/routes/feeds.cr | 11 +---- src/invidious/yt_backend/extractors.cr | 53 +++++---------------- 4 files changed, 39 insertions(+), 70 deletions(-) diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 29546e38b..1478c8fca 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -223,7 +223,7 @@ def fetch_channel(ucid, pull_all_videos : Bool) length_seconds = channel_video.try &.length_seconds length_seconds ||= 0 - live_now = channel_video.try &.live_now + live_now = channel_video.try &.badges.live_now? live_now ||= false premiere_timestamp = channel_video.try &.premiere_timestamp @@ -275,7 +275,7 @@ def fetch_channel(ucid, pull_all_videos : Bool) ucid: video.ucid, author: video.author, length_seconds: video.length_seconds, - live_now: video.live_now, + live_now: video.badges.live_now?, premiere_timestamp: video.premiere_timestamp, views: video.views, }) diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index c0ca789e0..8f57954a4 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -1,3 +1,16 @@ +@[Flags] +enum VideoBadges + LiveNow + Premium + ThreeD + FourK + New + EightK + VR180 + VR360 + CCommons +end + struct SearchVideo include DB::Serializable @@ -9,17 +22,9 @@ struct SearchVideo property views : Int64 property description_html : String property length_seconds : Int32 - property live_now : Bool - property premium : Bool property premiere_timestamp : Time? property author_verified : Bool - property is_new : Bool - property is_4k : Bool - property is_8k : Bool - property is_vr180 : Bool - property is_vr360 : Bool - property is_3d : Bool - property has_captions : Bool + property badges : VideoBadges def to_xml(auto_generated, query_params, xml : XML::Builder) query_params["v"] = self.id @@ -95,20 +100,20 @@ struct SearchVideo json.field "published", self.published.to_unix json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) json.field "lengthSeconds", self.length_seconds - json.field "liveNow", self.live_now - json.field "premium", self.premium + json.field "liveNow", self.badges.live_now? + json.field "premium", self.badges.premium? json.field "isUpcoming", self.upcoming? if self.premiere_timestamp json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix end - json.field "isNew", self.is_new - json.field "is4k", self.is_4k - json.field "is8k", self.is_8k - json.field "isVR180", is_vr180 - json.field "isVR360", is_vr360 - json.field "is3d", is_3d - json.field "hasCaptions", self.has_captions + json.field "isNew", self.badges.new? + json.field "is4k", self.badges.four_k? + json.field "is8k", self.badges.eight_k? + json.field "isVr180", self.badges.vr180? + json.field "isVr360", self.badges.vr360? + json.field "is3d", self.badges.three_d? + json.field "hasCaptions", self.badges.c_commons? end end diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index b76aeb467..ea7fb3965 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -192,18 +192,9 @@ module Invidious::Routes::Feeds views: views, description_html: description_html, length_seconds: 0, - live_now: false, - paid: false, - premium: false, premiere_timestamp: nil, author_verified: false, - is_new: false, - is_4k: false, - is_8k: false, - is_vr180: false, - is_vr360: false, - is_3d: false, - has_captions: false, + badges: VideoBadges::None, }) end diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 48c2155be..36f2dc4a9 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -108,42 +108,31 @@ private module Parsers length_seconds = 0 end - live_now = false - premium = false - is_new = false - is_4k = false - is_8k = false - is_vr180 = false - is_vr360 = false - is_3d = false - has_captions = false - premiere_timestamp = item_contents.dig?("upcomingEventData", "startTime").try { |t| Time.unix(t.as_s.to_i64) } - + badges = VideoBadges::None item_contents["badges"]?.try &.as_a.each do |badge| b = badge["metadataBadgeRenderer"] case b["label"].as_s when "LIVE NOW" - live_now = true + badges |= VideoBadges::LiveNow when "New" - is_new = true + badges |= VideoBadges::New when "4K" - is_4k = true + badges |= VideoBadges::FourK when "8K" - is_8k = true + badges |= VideoBadges::EightK when "VR180" - is_vr180 = true + badges |= VideoBadges::VR180 when "360°" - is_vr360 = true + badges |= VideoBadges::VR360 when "3D" - is_3d = true + badges |= VideoBadges::ThreeD when "CC" - has_captions = true + badges |= VideoBadges::CCommons when "Premium" # TODO: Potentially available as item_contents["topStandaloneBadge"]["metadataBadgeRenderer"] - premium = true - else # Ignore - puts b["label"].as_s + badges |= VideoBadges::Premium + else nil # Ignore end end @@ -156,17 +145,9 @@ private module Parsers views: view_count, description_html: description_html, length_seconds: length_seconds, - live_now: live_now, - premium: premium, premiere_timestamp: premiere_timestamp, author_verified: author_verified, - is_new: is_new, - is_4k: is_4k, - is_8k: is_8k, - is_vr180: is_vr180, - is_vr360: is_vr360, - is_3d: is_3d, - has_captions: has_captions, + badges: badges, }) end @@ -590,17 +571,9 @@ private module Parsers views: view_count, description_html: "", length_seconds: duration, - live_now: false, - premium: false, premiere_timestamp: Time.unix(0), author_verified: false, - is_new: false, - is_4k: false, - is_8k: false, - is_vr180: false, - is_vr360: false, - is_3d: false, - has_captions: false, + badges: VideoBadges::None, }) end From 60de5e91c05dc495b54b09620f4517536ec37d69 Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Sat, 24 Aug 2024 18:08:48 -0400 Subject: [PATCH 3/4] Fix tests --- spec/invidious/hashtag_spec.cr | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/spec/invidious/hashtag_spec.cr b/spec/invidious/hashtag_spec.cr index 266ec57bc..abc81225c 100644 --- a/spec/invidious/hashtag_spec.cr +++ b/spec/invidious/hashtag_spec.cr @@ -27,8 +27,8 @@ Spectator.describe Invidious::Hashtag do expect(video_11.length_seconds).to eq((56.minutes + 41.seconds).total_seconds.to_i32) expect(video_11.views).to eq(40_504_893) - expect(video_11.live_now).to be_false - expect(video_11.premium).to be_false + expect(video_11.badges.live_now?).to be_false + expect(video_11.badges.premium?).to be_false expect(video_11.premiere_timestamp).to be_nil # @@ -49,8 +49,8 @@ Spectator.describe Invidious::Hashtag do expect(video_35.length_seconds).to eq((3.minutes + 14.seconds).total_seconds.to_i32) expect(video_35.views).to eq(30_790_049) - expect(video_35.live_now).to be_false - expect(video_35.premium).to be_false + expect(video_35.badges.live_now?).to be_false + expect(video_35.badges.premium?).to be_false expect(video_35.premiere_timestamp).to be_nil end @@ -80,8 +80,8 @@ Spectator.describe Invidious::Hashtag do expect(video_41.length_seconds).to eq((1.hour).total_seconds.to_i32) expect(video_41.views).to eq(63_240) - expect(video_41.live_now).to be_false - expect(video_41.premium).to be_false + expect(video_41.badges.live_now?).to be_false + expect(video_41.badges.premium?).to be_false expect(video_41.premiere_timestamp).to be_nil # @@ -102,8 +102,8 @@ Spectator.describe Invidious::Hashtag do expect(video_48.length_seconds).to eq((35.minutes + 46.seconds).total_seconds.to_i32) expect(video_48.views).to eq(68_704) - expect(video_48.live_now).to be_false - expect(video_48.premium).to be_false + expect(video_48.badges.live_now?).to be_false + expect(video_48.badges.premium?).to be_false expect(video_48.premiere_timestamp).to be_nil end end From e48577392885d46cb376572a57cc97f3516809b3 Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Sat, 24 Aug 2024 21:04:22 -0400 Subject: [PATCH 4/4] Rename CCommons to ClosedCaptions --- src/invidious/helpers/serialized_yt_data.cr | 4 ++-- src/invidious/yt_backend/extractors.cr | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index 8f57954a4..1fef5f934 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -8,7 +8,7 @@ enum VideoBadges EightK VR180 VR360 - CCommons + ClosedCaptions end struct SearchVideo @@ -113,7 +113,7 @@ struct SearchVideo json.field "isVr180", self.badges.vr180? json.field "isVr360", self.badges.vr360? json.field "is3d", self.badges.three_d? - json.field "hasCaptions", self.badges.c_commons? + json.field "hasCaptions", self.badges.closed_captions? end end diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 36f2dc4a9..7c89f6c42 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -128,7 +128,7 @@ private module Parsers when "3D" badges |= VideoBadges::ThreeD when "CC" - badges |= VideoBadges::CCommons + badges |= VideoBadges::ClosedCaptions when "Premium" # TODO: Potentially available as item_contents["topStandaloneBadge"]["metadataBadgeRenderer"] badges |= VideoBadges::Premium