Merge pull request #665 from B0pol/agerestricted

[youtube] Fix extraction of age restricted videos

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -16,15 +16,8 @@
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
-import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException;
-import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
-import org.schabi.newpipe.extractor.exceptions.ExtractionException;
-import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException;
-import org.schabi.newpipe.extractor.exceptions.PaidContentException;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.exceptions.PrivateContentException;
-import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
-import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException;
+import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.exceptions.*;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.localization.Localization;
@@ -792,11 +785,9 @@ public void onFetchPage(@Nonnull final Downloader downloader)
     }
 
     private void fetchVideoInfoPage() throws ParsingException, ReCaptchaException, IOException {
-        final String sts = getEmbeddedInfoStsAndStorePlayerJsUrl();
-        final String videoInfoUrl = getVideoInfoUrl(getId(), sts);
-        final String infoPageResponse = NewPipe.getDownloader()
-                .get(videoInfoUrl, getExtractorLocalization()).responseBody();
-        videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
+        final String videoInfoUrl = getVideoInfoUrl(getId());
+        final Response videoInfoResponse = NewPipe.getDownloader().get(videoInfoUrl, getExtractorLocalization());
+        videoInfoPage.putAll(Parser.compatParseMap(videoInfoResponse.responseBody()));
 
         try {
             playerResponse = JsonParser.object().from(videoInfoPage.get("player_response"));
@@ -975,11 +966,11 @@ private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException {
     }
 
     @Nonnull
-    private static String getVideoInfoUrl(final String id, final String sts) {
+    private static String getVideoInfoUrl(final String id) {
         // TODO: Try parsing embedded_player_response first
-        return "https://www.youtube.com/get_video_info?" + "video_id=" + id +
-                "&html5=1&eurl=https://youtube.googleapis.com/v/" + id +
-                "&sts=" + sts + "&ps=default&gl=US&hl=en";
+        return "https://www.youtube.com/get_video_info?video_id=" + id +
+                "&eurl=https://youtube.googleapis.com/v/" + id +
+                "&html5=1&c=TVHTML5&cver=6.20180913&gl=US&hl=en";
     }
 
     private Map<String, ItagItem> getItags(final String streamingDataKey,
@@ -1121,7 +1112,7 @@ public Privacy getPrivacy() {
     public String getCategory() {
         return playerResponse.getObject("microformat")
                 .getObject("playerMicroformatRenderer")
-                .getString("category");
+                .getString("category", EMPTY_STRING);
     }
 
     @Nonnull

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java

@@ -57,7 +57,7 @@ public static void setUp() throws Exception {
     @Nullable @Override public String expectedErrorMessage() { return "Sign in to confirm your age"; }
     @Override public boolean expectedHasSubtitles() { return false; }
 
-    @Override public String expectedCategory() {return "Entertainment"; }
+    @Override public String expectedCategory() { return ""; } // Unavailable on age restricted videos
     @Override public String expectedLicence() { return "YouTube licence"; }
     @Override
     public List<String> expectedTags() {

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/stream/notAvailable/generated_mock_1.json

@@ -26,7 +26,7 @@
     "responseMessage": "",
     "responseHeaders": {
       "alt-svc": [
-        "h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
+        "h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
       ],
       "cache-control": [
         "no-cache, no-store, max-age\u003d0, must-revalidate"
@@ -38,7 +38,7 @@
         "application/json; charset\u003dutf-8"
       ],
       "date": [
-        "Sat, 22 May 2021 18:30:38 GMT"
+        "Wed, 23 Jun 2021 10:12:06 GMT"
       ],
       "expires": [
         "Mon, 01 Jan 1990 00:00:00 GMT"
@@ -56,8 +56,8 @@
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003dbWHlSChZfJE; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "CONSENT\u003dPENDING+805; expires\u003dFri, 01-Jan-2038 00:00:00 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
+        "YSC\u003d-LT8a9maoCU; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "CONSENT\u003dPENDING+142; expires\u003dFri, 01-Jan-2038 00:00:00 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"
@@ -75,7 +75,7 @@
         "0"
       ]
     },
-    "responseBody": "[\r\n{\"page\": \"watch\",\"rootVe\": \"3832\"},\r\n{\"page\": \"watch\",\"preconnect\": [\"https:\\/\\/r5---sn-4g5ednly.googlevideo.com\\/generate_204\",\"https:\\/\\/r5---sn-4g5ednly.googlevideo.com\\/generate_204?conn2\"]},\r\n{\"page\": \"watch\",\"playerResponse\": {\"responseContext\":{\"serviceTrackingParams\":[{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"is_viewed_live\",\"value\":\"False\"},{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"24037794,24040786,24044575,23884386,23944779,23983296,23882685,24014268,24027649,23995927,23735347,24043867,24032296,24011649,23975059,23891344,23968099,23970529,23966208,24005207,24042868,24005646,24019954,23986021,23940237,23891346,24037806,24003105,23996830,1714251,24005802,23804281,24030040,23946420,23744176,24035514,23890959,23918597,24007246,23974595,24045350,23934970,24001373,24036086,23968386,23997375,24042653,24027133,23857949,24004644,24035275,24024949,24036899,24042802\"}]},{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20200214.04.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"GetPlayer_rid\",\"value\":\"0xd8a06d19cc578a00\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20210324\"},{\"key\":\"client.name\",\"value\":\"WEB\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"playabilityStatus\":{\"status\":\"ERROR\",\"reason\":\"Video unavailable\",\"errorScreen\":{\"playerErrorMessageRenderer\":{\"reason\":{\"simpleText\":\"Video unavailable\"},\"thumbnail\":{\"thumbnails\":[{\"url\":\"//s.ytimg.com/yts/img/meh7-vflGevej7.png\",\"width\":140,\"height\":100}]},\"icon\":{\"iconType\":\"ERROR_OUTLINE\"}}},\"contextParams\":\"Q0FBU0FnZ0E\u003d\"},\"trackingParams\":\"CAAQu2kiEwi_h9z79d3wAhXw1xEIHUqjCfI\u003d\"}},\r\n{\"page\": \"watch\",\"response\": {\"responseContext\":{\"webResponseContextExtensionData\":{\"ytConfigData\":{\"visitorData\":\"CgswTjM5MHFXN0hlayjOm6WFBg%3D%3D\",\"rootVisualElementType\":3832}}}},\"xsrf_token\": \"QUFFLUhqbmQyWVlOb0RMYzZ2YXVzdmhjWDZpRjdLQUJMQXxBQ3Jtc0trS2FPZGRQUzJZNFhoTU5EcnZjQjZFV0dMYnE0ZkNxYUNkeVlLd3N4MU9fRGpLYjRzdkliTVE4N3prNVByTVl2Mi04aFpZcHdvckhGSDI2U1Ewa24tbWFHNjNMa1pBb2lET0JSQW0xTUY4WklUOEtTcw\\u003d\\u003d\",\"url\": \"/watch?v\\u003dINVALID_ID_\",\"endpoint\": {\"clickTrackingParams\":\"IhMIobfb-_Xd8AIVzWjgCh0aDgYWMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/watch?v\u003dINVALID_ID_\",\"webPageType\":\"WEB_PAGE_TYPE_WATCH\",\"rootVe\":3832}},\"watchEndpoint\":{\"videoId\":\"INVALID_ID_\"}}},\r\n{\"page\": \"watch\",\"timing\": {\"info\": {\"st\":  0.0 }}}]\r\n",
+    "responseBody": "[\r\n{\"page\": \"watch\",\"rootVe\": \"3832\"},\r\n{\"page\": \"watch\",\"preconnect\": [\"https:\\/\\/r5---sn-hgn7yn7z.googlevideo.com\\/generate_204\",\"https:\\/\\/r5---sn-hgn7yn7z.googlevideo.com\\/generate_204?conn2\"]},\r\n{\"page\": \"watch\",\"playerResponse\": {\"responseContext\":{\"serviceTrackingParams\":[{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"is_viewed_live\",\"value\":\"False\"},{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"24001373,23946420,23970529,24049577,24042868,23966208,23804281,24027649,23998056,24049575,24036947,24004644,24017848,23891344,24007246,24037794,24052246,23963745,23934970,23996830,24030040,23983296,23891346,23884386,24037806,24042870,24012513,24058813,23857948,24053867,24049569,23968386,24049820,1714249,23973490,24049567,23918597,24003105,24049573,23744176,23974595,24058380,24003103,23882685,23944779,24038425,23890959,23999405\"}]},{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20200214.04.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"GetPlayer_rid\",\"value\":\"0x58da0be5a82f9e65\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20210526\"},{\"key\":\"client.name\",\"value\":\"WEB\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"playabilityStatus\":{\"status\":\"ERROR\",\"reason\":\"Video unavailable\",\"errorScreen\":{\"playerErrorMessageRenderer\":{\"reason\":{\"simpleText\":\"Video unavailable\"},\"thumbnail\":{\"thumbnails\":[{\"url\":\"//s.ytimg.com/yts/img/meh7-vflGevej7.png\",\"width\":140,\"height\":100}]},\"icon\":{\"iconType\":\"ERROR_OUTLINE\"}}},\"contextParams\":\"Q0FBU0FnZ0E\u003d\"},\"trackingParams\":\"CAAQu2kiEwiG75Ocwq3xAhU_NPEFHUM9A70\u003d\",\"frameworkUpdates\":{\"entityBatchUpdate\":{\"mutations\":[{\"entityKey\":\"Eg0KC0lOVkFMSURfSURfIPYBKAE%3D\",\"type\":\"ENTITY_MUTATION_TYPE_REPLACE\",\"payload\":{\"offlineabilityEntity\":{\"key\":\"Eg0KC0lOVkFMSURfSURfIPYBKAE%3D\",\"accessState\":\"OFFLINEABILITY_FEATURE_ACCESS_STATE_UNKNOWN\"}}}],\"timestamp\":{\"seconds\":\"1624443126\",\"nanos\":616071922}}}}},\r\n{\"page\": \"watch\",\"response\": {\"responseContext\":{\"webResponseContextExtensionData\":{\"ytConfigData\":{\"visitorData\":\"Cgs1SUlhQUVDenVSUSj2kcyGBg%3D%3D\",\"rootVisualElementType\":3832}}}},\"xsrf_token\": \"QUFFLUhqbkdDQldjVWFhaHN4ZUQzTjMtY01fU1FpVElFd3xBQ3Jtc0tseTZPT0J2ZzA2eDNuYXRMREtXaGN0RWJsUWJ5eW1vWDQ4aGg5TDJHTjZvX3lSM2tRamh2M3lDdDNXc3haZE00TUJveWNZa2JBbWdYX21URUI2dTM2MTlfX0pIcWU1RVNXRHpreExHeHNfaW5LQjdvQQ\\u003d\\u003d\",\"url\": \"/watch?v\\u003dINVALID_ID_\",\"endpoint\": {\"clickTrackingParams\":\"IhMIpImTnMKt8QIV9TvxBR0TrgLLMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/watch?v\u003dINVALID_ID_\",\"webPageType\":\"WEB_PAGE_TYPE_WATCH\",\"rootVe\":3832}},\"watchEndpoint\":{\"videoId\":\"INVALID_ID_\"}}},\r\n{\"page\": \"watch\",\"timing\": {\"info\": {\"st\":  0.0 }}}]\r\n",
     "latestUrl": "https://www.youtube.com/watch?v\u003dINVALID_ID_\u0026pbj\u003d1"
   }
 }