Merge pull request #267 from mauriciocolli/hotfixes

Hotfixes for YouTube and improve unavailable cases

extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java

@@ -15,12 +15,16 @@ public class Response {
     private final Map<String, List<String>> responseHeaders;
     private final String responseBody;
 
-    public Response(int responseCode, String responseMessage, Map<String, List<String>> responseHeaders, @Nullable String responseBody) {
+    private final String latestUrl;
+
+    public Response(int responseCode, String responseMessage, Map<String, List<String>> responseHeaders,
+                    @Nullable String responseBody, @Nullable String latestUrl) {
         this.responseCode = responseCode;
         this.responseMessage = responseMessage;
         this.responseHeaders = responseHeaders != null ? responseHeaders : Collections.<String, List<String>>emptyMap();
 
         this.responseBody = responseBody == null ? "" : responseBody;
+        this.latestUrl = latestUrl;
     }
 
     public int responseCode() {
@@ -40,6 +44,16 @@ public String responseBody() {
         return responseBody;
     }
 
+    /**
+     * Used for detecting a possible redirection, limited to the latest one.
+     *
+     * @return latest url known right before this response object was created
+     */
+    @Nonnull
+    public String latestUrl() {
+        return latestUrl;
+    }
+
     /*//////////////////////////////////////////////////////////////////////////
     // Utils
     //////////////////////////////////////////////////////////////////////////*/
@@ -54,7 +68,8 @@ public String responseBody() {
     @Nullable
     public String getHeader(String name) {
         for (Map.Entry<String, List<String>> headerEntry : responseHeaders.entrySet()) {
-            if (headerEntry.getKey().equalsIgnoreCase(name)) {
+            final String key = headerEntry.getKey();
+            if (key != null && key.equalsIgnoreCase(name)) {
                 if (headerEntry.getValue().size() > 0) {
                     return headerEntry.getValue().get(0);
                 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -58,8 +58,8 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
         final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid";
 
         final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
-
         initialData = ajaxJson.getObject(1).getObject("response");
+        YoutubeParsingHelper.defaultAlertsCheck(initialData);
     }
 
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -10,6 +10,7 @@
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.Utils;
@@ -39,6 +40,8 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
         final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
 
         initialData = ajaxJson.getObject(1).getObject("response");
+        YoutubeParsingHelper.defaultAlertsCheck(initialData);
+
         playlistInfo = getPlaylistInfo();
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java

@@ -63,7 +63,7 @@ public String getUrl() throws ParsingException {
     }
 
     @Override
-    public String getSearchSuggestion() {
+    public String getSearchSuggestion() throws ParsingException {
         JsonObject showingResultsForRenderer = initialData.getObject("contents")
                 .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents")
                 .getObject("sectionListRenderer").getArray("contents").getObject(0)
@@ -114,7 +114,7 @@ public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, Extra
         return new InfoItemsPage<>(collector, getNextPageUrlFrom(itemSectionRenderer.getArray("continuations")));
     }
 
-    private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException {
+    private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException, ParsingException {
         collector.reset();
 
         final TimeAgoParser timeAgoParser = getTimeAgoParser();

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -11,6 +11,7 @@
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
+import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
@@ -32,6 +33,7 @@
 import org.schabi.newpipe.extractor.stream.StreamType;
 import org.schabi.newpipe.extractor.stream.SubtitlesStream;
 import org.schabi.newpipe.extractor.stream.VideoStream;
+import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
@@ -564,8 +566,12 @@ public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException {
      */
     @Override
     public String getErrorMessage() {
-        return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse").getObject("playabilityStatus")
-                .getObject("errorScreen").getObject("playerErrorMessageRenderer").getObject("reason"));
+        try {
+            return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse").getObject("playabilityStatus")
+                    .getObject("errorScreen").getObject("playerErrorMessageRenderer").getObject("reason"));
+        } catch (ParsingException e) {
+            return null;
+        }
     }
 
     /*//////////////////////////////////////////////////////////////////////////
@@ -615,6 +621,13 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
 
         playerResponse = getPlayerResponse();
 
+        final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus", JsonUtils.DEFAULT_EMPTY);
+        final String status = playabilityStatus.getString("status");
+        if (status != null && status.toLowerCase().equals("error")) {
+            final String reason = playabilityStatus.getString("reason");
+            throw new ContentNotAvailableException("Got error: \"" + reason + "\"");
+        }
+
         if (decryptionCode.isEmpty()) {
             decryptionCode = loadDecryptionCode(playerUrl);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java

@@ -2,7 +2,6 @@
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
-
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
@@ -13,10 +12,11 @@
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import javax.annotation.Nullable;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
 
-import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
-import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
-import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.*;
 
 /*
  * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
@@ -86,7 +86,9 @@ public String getName() throws ParsingException {
 
     @Override
     public long getDuration() throws ParsingException {
-        if (getStreamType() == StreamType.LIVE_STREAM) return -1;
+        if (getStreamType() == StreamType.LIVE_STREAM || isPremiere()) {
+            return -1;
+        }
 
         String duration = null;
 
@@ -165,7 +167,16 @@ public String getUploaderUrl() throws ParsingException {
 
     @Nullable
     @Override
-    public String getTextualUploadDate() {
+    public String getTextualUploadDate() throws ParsingException {
+        if (getStreamType().equals(StreamType.LIVE_STREAM)) {
+            return null;
+        }
+
+        if (isPremiere()) {
+            final Date date = getDateFromPremiere().getTime();
+            return new SimpleDateFormat("yyyy-MM-dd HH:mm").format(date);
+        }
+
         try {
             return getTextFromObject(videoInfo.getObject("publishedTimeText"));
         } catch (Exception e) {
@@ -177,7 +188,15 @@ public String getTextualUploadDate() {
     @Nullable
     @Override
     public DateWrapper getUploadDate() throws ParsingException {
-        String textualUploadDate = getTextualUploadDate();
+        if (getStreamType().equals(StreamType.LIVE_STREAM)) {
+            return null;
+        }
+
+        if (isPremiere()) {
+            return new DateWrapper(getDateFromPremiere());
+        }
+
+        final String textualUploadDate = getTextualUploadDate();
         if (timeAgoParser != null && textualUploadDate != null && !textualUploadDate.isEmpty()) {
             try {
                 return timeAgoParser.parse(textualUploadDate);
@@ -194,11 +213,22 @@ public long getViewCount() throws ParsingException {
             if (videoInfo.getObject("topStandaloneBadge") != null || isPremium()) {
                 return -1;
             }
-            String viewCount = getTextFromObject(videoInfo.getObject("viewCountText"));
+
+            final JsonObject viewCountObject = videoInfo.getObject("viewCountText");
+            if (viewCountObject == null) {
+                // This object is null when a video has its views hidden.
+                return -1;
+            }
+
+            final String viewCount = getTextFromObject(viewCountObject);
+
+            if (viewCount.toLowerCase().contains("no views")) {
+                return 0;
+            } else if (viewCount.toLowerCase().contains("recommended")) {
+                return -1;
+            }
 
             return Long.parseLong(Utils.removeNonDigitCharacters(viewCount));
-        } catch (NumberFormatException e) {
-            return -1;
         } catch (Exception e) {
             throw new ParsingException("Could not get view count", e);
         }
@@ -225,7 +255,26 @@ private boolean isPremium() {
                     return true;
                 }
             }
-        } catch (Exception ignored) {}
+        } catch (Exception ignored) {
+        }
         return false;
     }
+
+    private boolean isPremiere() {
+        return videoInfo.has("upcomingEventData");
+    }
+
+    private Calendar getDateFromPremiere() throws ParsingException {
+        final JsonObject upcomingEventData = videoInfo.getObject("upcomingEventData");
+        final String startTime = upcomingEventData.getString("startTime");
+
+        try {
+            final long startTimeTimestamp = Long.parseLong(startTime);
+            final Calendar calendar = Calendar.getInstance();
+            calendar.setTime(new Date(startTimeTimestamp * 1000L));
+            return calendar;
+        } catch (Exception e) {
+            throw new ParsingException("Could not parse date from premiere:  \"" + startTime + "\"");
+        }
+    }
 }