Merge pull request #944 from Theta-Dev/fix-rich-grid-renderer

[YouTube] Support richGridRenderer on channel page to fix extraction of videos in channels
TeamNewPipe · Nov 2, 2022 · 31bf704 · 31bf704
2 parents 4cae66f + e4c24d4
commit 31bf704
Show file tree

Hide file tree

Showing 30 changed files with 798 additions and 831 deletions.
diff --git a/...ava/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/...ava/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
@@ -332,16 +332,20 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, Extrac
         Page nextPage = null;
 
         if (getVideoTab() != null) {
-            final JsonObject gridRenderer = getVideoTab().getObject("content")
+            final JsonObject tabContent = getVideoTab().getObject("content");
+            JsonArray items = tabContent
                     .getObject("sectionListRenderer")
                     .getArray("contents").getObject(0).getObject("itemSectionRenderer")
-                    .getArray("contents").getObject(0).getObject("gridRenderer");
+                    .getArray("contents").getObject(0).getObject("gridRenderer").getArray("items");
+
+            if (items.isEmpty()) {
+                items = tabContent.getObject("richGridRenderer").getArray("contents");
+            }
 
             final List<String> channelIds = new ArrayList<>();
             channelIds.add(getName());
             channelIds.add(getUrl());
-            final JsonObject continuation = collectStreamsFrom(collector, gridRenderer
-                    .getArray("items"), channelIds);
+            final JsonObject continuation = collectStreamsFrom(collector, items, channelIds);
 
             nextPage = getNextPageFrom(continuation, channelIds);
         }
@@ -433,6 +437,21 @@ public String getUploaderUrl() {
                         return uploaderUrl;
                     }
                 });
+            } else if (video.has("richItemRenderer")) {
+                collector.commit(new YoutubeStreamInfoItemExtractor(
+                        video.getObject("richItemRenderer")
+                                .getObject("content").getObject("videoRenderer"), timeAgoParser) {
+                    @Override
+                    public String getUploaderName() {
+                        return uploaderName;
+                    }
+
+                    @Override
+                    public String getUploaderUrl() {
+                        return uploaderUrl;
+                    }
+                });
+
             } else if (video.has("continuationItemRenderer")) {
                 continuation = video.getObject("continuationItemRenderer");
             }

diff --git a/.../schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_0.json b/.../schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_0.json
@@ -37,23 +37,20 @@
       "content-type": [
         "text/javascript; charset\u003dutf-8"
       ],
-      "critical-ch": [
-        "Sec-CH-UA-Arch, Sec-CH-UA-Bitness, Sec-CH-UA-Full-Version, Sec-CH-UA-Full-Version-List, Sec-CH-UA-Model, Sec-CH-UA-Platform, Sec-CH-UA-Platform-Version"
-      ],
       "cross-origin-opener-policy-report-only": [
         "same-origin; report-to\u003d\"youtube_main\""
       ],
       "date": [
-        "Fri, 12 Aug 2022 17:16:25 GMT"
+        "Wed, 02 Nov 2022 17:40:55 GMT"
       ],
       "expires": [
-        "Fri, 12 Aug 2022 17:16:25 GMT"
+        "Wed, 02 Nov 2022 17:40:55 GMT"
       ],
       "p3p": [
         "CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
       ],
       "permissions-policy": [
-        "ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
+        "ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
       ],
       "report-to": [
         "{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
@@ -62,9 +59,9 @@
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003dH65DWcDwcL0; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSat, 16-Nov-2019 17:16:25 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "CONSENT\u003dPENDING+212; expires\u003dSun, 11-Aug-2024 17:16:25 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
+        "YSC\u003d6aCd9lqn2z8; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dThu, 06-Feb-2020 17:40:55 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "CONSENT\u003dPENDING+174; expires\u003dFri, 01-Nov-2024 17:40:54 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"

diff --git a/.../schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_1.json b/.../schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_1.json
diff --git a/.../schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_2.json b/.../schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_2.json
@@ -13,7 +13,7 @@
         "https://www.youtube.com"
       ],
       "X-YouTube-Client-Version": [
-        "2.20220809.02.00"
+        "2.20221101.00.00"
       ],
       "Content-Type": [
         "application/json"
@@ -205,13 +205,13 @@
       48,
       50,
       50,
+      49,
+      49,
       48,
-      56,
-      48,
-      57,
+      49,
       46,
       48,
-      50,
+      48,
       46,
       48,
       48,
@@ -338,10 +338,10 @@
         "application/json; charset\u003dUTF-8"
       ],
       "date": [
-        "Fri, 12 Aug 2022 17:16:25 GMT"
+        "Wed, 02 Nov 2022 17:40:55 GMT"
       ],
       "expires": [
-        "Fri, 12 Aug 2022 17:16:25 GMT"
+        "Wed, 02 Nov 2022 17:40:55 GMT"
       ],
       "p3p": [
         "CP\u003d\"This is not a P3P policy! See g.co/p3phelp for more info.\""
@@ -350,7 +350,7 @@
         "scaffolding on HTTPServer2"
       ],
       "set-cookie": [
-        "CONSENT\u003dPENDING+061; expires\u003dSun, 11-Aug-2024 17:16:25 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
+        "CONSENT\u003dPENDING+984; expires\u003dFri, 01-Nov-2024 17:40:55 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
       ],
       "vary": [
         "Origin",
@@ -367,7 +367,7 @@
         "0"
       ]
     },
-    "responseBody": "{\"responseContext\":{\"visitorData\":\"Cgs0d1FhenR6RXVOayjpldqXBg%3D%3D\",\"serviceTrackingParams\":[{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20220809.02.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"ResolveUrl_rid\",\"value\":\"0xc781391fa55a10e4\"}]},{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"24139339,24209350,23940248,24187377,1714242,24002025,24164186,24216166,24199724,24199710,24135943,24226335,24220088,24085811,24077241,23998056,24245746,24253963,24235348,24036948,24007246,24225483,39321934,24246427,24222379,24004644,24244808,24241090,24114244,24219713,24166867,24080738,24135310,24238983,24245799,23918597,24260692,23744176,24241936,24199774,24254429,24219033,23983296,24186125,24230625,24140247,24169501,24248231,24227844,24198739,24239147,24077266,24181174,24252017,24034168,23986031,23966208,23885487,24223124,24250883,24211177,23946420,23804281,24214092,24241165,24248385,24161116,24246938,24191629,24001373,24152442,24215196,24187043,24145390,24037231,24229161,24002022,24185614,23882503,24120819,24227532,23934970\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20220805\"},{\"key\":\"client.name\",\"value\":\"WEB\"},{\"key\":\"client.fexp\",\"value\":\"24139339,24209350,23940248,24187377,1714242,24002025,24164186,24216166,24199724,24199710,24135943,24226335,24220088,24085811,24077241,23998056,24245746,24253963,24235348,24036948,24007246,24225483,39321934,24246427,24222379,24004644,24244808,24241090,24114244,24219713,24166867,24080738,24135310,24238983,24245799,23918597,24260692,23744176,24241936,24199774,24254429,24219033,23983296,24186125,24230625,24140247,24169501,24248231,24227844,24198739,24239147,24077266,24181174,24252017,24034168,23986031,23966208,23885487,24223124,24250883,24211177,23946420,23804281,24214092,24241165,24248385,24161116,24246938,24191629,24001373,24152442,24215196,24187043,24145390,24037231,24229161,24002022,24185614,23882503,24120819,24227532,23934970\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"endpoint\":{\"clickTrackingParams\":\"IhMI5ZeWiefB-QIVGF6yCh0ZxgVhMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/youtubei/v1/navigation/resolve_url\",\"webPageType\":\"WEB_PAGE_TYPE_CHANNEL\",\"rootVe\":3611,\"apiUrl\":\"/youtubei/v1/browse\"},\"resolveUrlCommandMetadata\":{\"isVanityUrl\":true}},\"browseEndpoint\":{\"browseId\":\"UC6nSFpj9HTCZ5t-N3Rm3-HA\",\"params\":\"EgC4AQDyBgQKAjIA\"}}}",
+    "responseBody": "{\"responseContext\":{\"visitorData\":\"CgtJUUxxcnRoM214WSin14qbBg%3D%3D\",\"serviceTrackingParams\":[{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20221101.00.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"ResolveUrl_rid\",\"value\":\"0x7b414305caad8bf8\"}]},{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"1714257,23804281,23882502,23885487,23918597,23934970,23946420,23966208,23983296,23986016,23998056,24001373,24002022,24002025,24004644,24007246,24034168,24036947,24077241,24080738,24120820,24135310,24140247,24152443,24161116,24162920,24164186,24166867,24169501,24181174,24184446,24185614,24187043,24187377,24191629,24199724,24206234,24209350,24211178,24219713,24229161,24230618,24241378,24246502,24248092,24254502,24255165,24255543,24255545,24260783,24262346,24263796,24265820,24267564,24267570,24268142,24269410,24278596,24279196,24279540,24279628,24282828,24283093,24283556,24286003,24286019,24287169,24287327,24287795,24288047,24288489,24288912,24290289,24290971,24291857,24292739,24292955,24299747,24390674,24391539,24392268,24392399,24392496,24393382,24394548,24395541,24396645,24396818,24398124,24398996,24400009,24400945,24401557,24406381,24406604,24406983,24407199,24407452,24408951,24411026,39322399,39322504,39322574\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20221101\"},{\"key\":\"client.name\",\"value\":\"WEB\"},{\"key\":\"client.fexp\",\"value\":\"24185614,24036947,24199724,24391539,24268142,24396645,24406604,24164186,24282828,24299747,24187043,24392399,24291857,24219713,24394548,24288489,24152443,24080738,24406983,39322399,24191629,39322504,24140247,24286003,24209350,24265820,23983296,24255545,24254502,24287327,24287169,24400009,24286019,23882502,24279196,24279540,24255543,24396818,23946420,23986016,23918597,24411026,23885487,24279628,24002022,24034168,23966208,24269410,24181174,24169501,24267570,24290971,24161116,24398124,24001373,24292955,24120820,1714257,23804281,24278596,24077241,24262346,24287795,24002025,24248092,24392496,24398996,24400945,24283093,24241378,24406381,24263796,24267564,24392268,24288912,24162920,24260783,24166867,24407452,24184446,24407199,24229161,24230618,24290289,24408951,39322574,23998056,24288047,24211178,24007246,24135310,24401557,24206234,23934970,24393382,24283556,24395541,24004644,24255165,24292739,24246502,24390674,24187377\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"endpoint\":{\"clickTrackingParams\":\"IhMI157_vYWQ-wIVjs5VCh2k1A1NMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/youtubei/v1/navigation/resolve_url\",\"webPageType\":\"WEB_PAGE_TYPE_CHANNEL\",\"rootVe\":3611,\"apiUrl\":\"/youtubei/v1/browse\"},\"resolveUrlCommandMetadata\":{\"isVanityUrl\":true}},\"browseEndpoint\":{\"browseId\":\"UC6nSFpj9HTCZ5t-N3Rm3-HA\",\"params\":\"EgC4AQDyBgQKAjIA\"}}}",
     "latestUrl": "https://www.youtube.com/youtubei/v1/navigation/resolve_url?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8\u0026prettyPrint\u003dfalse"
   }
 }
diff --git a/.../schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_3.json b/.../schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_3.json