Release 0.21.12

README.md

@@ -11,7 +11,7 @@ NewPipe Extractor is available at JitPack's Maven repo.
 If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps:
 
 1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`.
-2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.21.11'`the `dependencies` in your `build.gradle`. Replace `v0.21.11` with the latest release.
+2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:INSERT_VERSION_HERE'` to the `dependencies` in your `build.gradle`. Replace `INSERT_VERSION_HERE` with the [latest release](https://github.com/TeamNewPipe/NewPipeExtractor/releases/latest).
 
 **Note:** To use NewPipe Extractor in projects with a `minSdkVersion` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required.
 

build.gradle

@@ -8,7 +8,7 @@ allprojects {
     sourceCompatibility = 1.8
     targetCompatibility = 1.8
 
-    version 'v0.21.11'
+    version 'v0.21.12'
     group 'com.github.TeamNewPipe'
 
     repositories {
@@ -28,7 +28,7 @@ allprojects {
 
     ext {
         nanojsonVersion = "1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751"
-        spotbugsVersion = "4.4.1"
+        spotbugsVersion = "4.5.0"
         junitVersion = "4.13.2"
     }
 }

extractor/build.gradle

@@ -16,5 +16,5 @@ dependencies {
 
     testImplementation "junit:junit:$junitVersion"
     testImplementation "com.squareup.okhttp3:okhttp:3.12.13"
-    testImplementation 'com.google.code.gson:gson:2.8.8'
+    testImplementation 'com.google.code.gson:gson:2.8.9'
 }

extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java

@@ -24,9 +24,9 @@ public static CommentsInfo getInfo(final String url) throws IOException, Extract
         return getInfo(NewPipe.getServiceByUrl(url), url);
     }
 
-    public static CommentsInfo getInfo(final StreamingService serviceByUrl, final String url)
+    public static CommentsInfo getInfo(final StreamingService service, final String url)
             throws ExtractionException, IOException {
-        return getInfo(serviceByUrl.getCommentsExtractor(url));
+        return getInfo(service.getCommentsExtractor(url));
     }
 
     public static CommentsInfo getInfo(final CommentsExtractor commentsExtractor)
@@ -63,7 +63,7 @@ public static InfoItemsPage<CommentsInfoItem> getMoreItems(
             final StreamingService service,
             final CommentsInfo commentsInfo,
             final Page page) throws IOException, ExtractionException {
-        if (null == commentsInfo.getCommentsExtractor()) {
+        if (commentsInfo.getCommentsExtractor() == null) {
             commentsInfo.setCommentsExtractor(service.getCommentsExtractor(commentsInfo.getUrl()));
             commentsInfo.getCommentsExtractor().fetchPage();
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java

@@ -10,7 +10,7 @@
 public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
 
     private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory();
-    private static final String ID_PATTERN = "(accounts|video-channels)/([^/?&#]*)";
+    private static final String ID_PATTERN = "((accounts|a)|(video-channels|c))/([^/?&#]*)";
     public static final String API_ENDPOINT = "/api/v1/";
 
     public static PeertubeChannelLinkHandlerFactory getInstance() {
@@ -19,7 +19,7 @@ public static PeertubeChannelLinkHandlerFactory getInstance() {
 
     @Override
     public String getId(String url) throws ParsingException {
-        return Parser.matchGroup(ID_PATTERN, url, 0);
+        return fixId(Parser.matchGroup(ID_PATTERN, url, 0));
     }
 
     @Override
@@ -31,7 +31,7 @@ public String getUrl(String id, List<String> contentFilters, String searchFilter
     public String getUrl(String id, List<String> contentFilter, String sortFilter, String baseUrl)
             throws ParsingException {
         if (id.matches(ID_PATTERN)) {
-            return baseUrl + "/" + id;
+            return baseUrl + "/" + fixId(id);
         } else {
             // This is needed for compatibility with older versions were we didn't support video channels yet
             return baseUrl + "/accounts/" + id;
@@ -40,6 +40,28 @@ public String getUrl(String id, List<String> contentFilter, String sortFilter, S
 
     @Override
     public boolean onAcceptUrl(String url) {
-        return url.contains("/accounts/") || url.contains("/video-channels/");
+        return url.contains("/accounts/") || url.contains("/a/")
+                || url.contains("/video-channels/") || url.contains("/c/");
+    }
+
+    /**
+     * Fix id
+     *
+     * <p>
+     * a/:accountName and c/:channelName ids are supported
+     * by the PeerTube web client (>= v3.3.0)
+     * but not by the API.
+     * </p>
+     *
+     * @param id the id to fix
+     * @return the fixed id
+     */
+    private String fixId(String id) {
+        if (id.startsWith("a/")) {
+            id = "accounts" + id.substring(1);
+        } else if (id.startsWith("c/")) {
+            id = "video-channels" + id.substring(1);
+        }
+        return id;
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java

@@ -11,7 +11,6 @@
 public class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
 
     private static final PeertubeCommentsLinkHandlerFactory instance = new PeertubeCommentsLinkHandlerFactory();
-    private static final String ID_PATTERN = "/videos/(watch/)?([^/?&#]*)";
     private static final String COMMENTS_ENDPOINT = "/api/v1/videos/%s/comment-threads";
 
     public static PeertubeCommentsLinkHandlerFactory getInstance() {
@@ -20,12 +19,12 @@ public static PeertubeCommentsLinkHandlerFactory getInstance() {
 
     @Override
     public String getId(String url) throws ParsingException, IllegalArgumentException {
-        return Parser.matchGroup(ID_PATTERN, url, 2);
+        return PeertubeStreamLinkHandlerFactory.getInstance().getId(url); // the same id is needed
     }
 
     @Override
     public boolean onAcceptUrl(final String url) throws FoundAdException {
-        return url.contains("/videos/");
+        return url.contains("/videos/") || url.contains("/w/");
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java

@@ -11,7 +11,7 @@
 public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
 
     private static final PeertubePlaylistLinkHandlerFactory instance = new PeertubePlaylistLinkHandlerFactory();
-    private static final String ID_PATTERN = "/videos/watch/playlist/([^/?&#]*)";
+    private static final String ID_PATTERN = "(/videos/watch/playlist/|/w/p/)([^/?&#]*)";
 
     public static PeertubePlaylistLinkHandlerFactory getInstance() {
         return instance;
@@ -30,7 +30,7 @@ public String getUrl(String id, List<String> contentFilters, String sortFilter,
 
     @Override
     public String getId(String url) throws ParsingException {
-        return Parser.matchGroup1(ID_PATTERN, url);
+        return Parser.matchGroup(ID_PATTERN, url, 2);
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java

@@ -9,8 +9,13 @@
 public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory {
 
     private static final PeertubeStreamLinkHandlerFactory instance = new PeertubeStreamLinkHandlerFactory();
-    private static final String ID_PATTERN = "/videos/(watch/|embed/)?([^/?&#]*)";
+    private static final String ID_PATTERN = "(/w/|(/videos/(watch/|embed/)?))(?!p/)([^/?&#]*)";
+    // we exclude p/ because /w/p/ is playlist, not video
     public static final String VIDEO_API_ENDPOINT = "/api/v1/videos/";
+
+    // From PeerTube 3.3.0, the default path is /w/.
+    // We still use /videos/watch/ for compatibility reasons:
+    // /videos/watch/ is still accepted by >=3.3.0 but /w/ isn't by <3.3.0
     private static final String VIDEO_PATH = "/videos/watch/";
 
     private PeertubeStreamLinkHandlerFactory() {
@@ -32,7 +37,7 @@ public String getUrl(String id, String baseUrl) {
 
     @Override
     public String getId(String url) throws ParsingException, IllegalArgumentException {
-        return Parser.matchGroup(ID_PATTERN, url, 2);
+        return Parser.matchGroup(ID_PATTERN, url, 4);
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -40,8 +40,8 @@
 import static org.schabi.newpipe.extractor.utils.Utils.*;
 
 public class SoundcloudParsingHelper {
-    private static final String HARDCODED_CLIENT_ID =
-            "yoxLvaFlJ3V5LbNCt53Cwvw5KXKKxWfn"; // Updated on 01/10/21
+    static final String HARDCODED_CLIENT_ID =
+            "1NKODbzHzEpoowFHxTAmS7oB08DObPuK"; // Updated on 08/12/21
     private static String clientId;
     public static final String SOUNDCLOUD_API_V2_URL = "https://api-v2.soundcloud.com/";
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -162,6 +162,10 @@ public static boolean isInvidioURL(@Nonnull final URL url) {
                 || host.equalsIgnoreCase("y.com.cm");
     }
 
+    public static boolean isY2ubeURL(@Nonnull final URL url) {
+        return url.getHost().equalsIgnoreCase("y2u.be");
+    }
+
     /**
      * Parses the duration string of the video expecting ":" or "." as separators
      *

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -337,19 +337,33 @@ public long getViewCount() throws ParsingException {
     @Override
     public long getLikeCount() throws ParsingException {
         assertPageFetched();
-        String likesString = "";
+        String likesString = null;
         try {
-            try {
-                likesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar")
-                        .getObject("sentimentBarRenderer").getString("tooltip").split("/")[0];
-            } catch (final NullPointerException e) {
+            likesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar")
+                    .getObject("sentimentBarRenderer").getString("tooltip");
+            if (likesString != null && likesString.contains("/")) {
+                likesString = likesString.split("/")[0];
+            } else {
+                likesString = getVideoPrimaryInfoRenderer()
+                        .getObject("videoActions")
+                        .getObject("menuRenderer")
+                        .getArray("topLevelButtons")
+                        .getObject(0)
+                        .getObject("toggleButtonRenderer")
+                        .getObject("defaultText")
+                        .getObject("accessibility")
+                        .getObject("accessibilityData")
+                        .getString("label");
+            }
+
+            if (likesString == null) {
                 // If this kicks in our button has no content and therefore ratings must be disabled
                 if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
-                    throw new ParsingException(
-                            "Ratings are enabled even though the like button is missing", e);
+                    throw new ParsingException("Ratings are enabled even though the like button is missing");
                 }
                 return -1;
             }
+
             return Integer.parseInt(Utils.removeNonDigitCharacters(likesString));
         } catch (final NumberFormatException nfe) {
             throw new ParsingException("Could not parse \"" + likesString + "\" as an Integer",
@@ -366,29 +380,28 @@ public long getLikeCount() throws ParsingException {
     public long getDislikeCount() throws ParsingException {
         assertPageFetched();
 
-        String dislikesString = "";
         try {
-            try {
-                dislikesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar")
-                        .getObject("sentimentBarRenderer").getString("tooltip").split("/")[1];
-            } catch (final NullPointerException e) {
-                // If this kicks in our button has no content and therefore ratings must be disabled
-                if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
-                    throw new ParsingException(
-                            "Ratings are enabled even though the dislike button is missing", e);
+            String dislikesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar")
+                    .getObject("sentimentBarRenderer").getString("tooltip");
+            if (dislikesString != null && dislikesString.contains("/")) {
+                dislikesString = dislikesString.split("/")[1];
+                return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
+            } else {
+                // Calculate dislike with average rating and like count
+                long likes = getLikeCount();
+                double averageRating = playerResponse.getObject("videoDetails").getDouble("averageRating");
+
+                if (likes != -1 && averageRating > 1) {
+                    // If averageRating can't be gathered, it will be 0,
+                    // but we also can't divide by 0 so we need > 1
+                    return Math.round(likes * ((5 - averageRating) / (averageRating - 1)));
                 }
-                return -1;
             }
-            return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
-        } catch (final NumberFormatException nfe) {
-            throw new ParsingException("Could not parse \"" + dislikesString + "\" as an Integer",
-                    nfe);
         } catch (final Exception e) {
-            if (getAgeLimit() == NO_AGE_LIMIT) {
-                throw new ParsingException("Could not get dislike count", e);
-            }
-            return -1;
         }
+        // Silently fail as YouTube is "gradually rolling out" removing dislike count
+        // https://blog.youtube/news-and-events/update-to-youtube/
+        return -1;
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java

@@ -288,4 +288,19 @@ private OffsetDateTime getDateFromPremiere() throws ParsingException {
             throw new ParsingException("Could not parse date from premiere:  \"" + startTime + "\"");
         }
     }
+
+    @Nullable
+    @Override
+    public String getShortDescription() throws ParsingException {
+
+        if (videoInfo.has("detailedMetadataSnippets")) {
+            return getTextFromObject(videoInfo.getArray("detailedMetadataSnippets").getObject(0).getObject("snippetText"));
+        }
+
+        if (videoInfo.has("descriptionSnippet")) {
+            return getTextFromObject(videoInfo.getObject("descriptionSnippet"));
+        }
+
+        return null;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java

@@ -110,7 +110,7 @@ public String getId(String urlString) throws ParsingException, IllegalArgumentEx
 
         if (!Utils.isHTTP(url) || !(YoutubeParsingHelper.isYoutubeURL(url) ||
                 YoutubeParsingHelper.isYoutubeServiceURL(url) || YoutubeParsingHelper.isHooktubeURL(url) ||
-                YoutubeParsingHelper.isInvidioURL(url))) {
+                YoutubeParsingHelper.isInvidioURL(url) || YoutubeParsingHelper.isY2ubeURL(url))) {
             if (host.equalsIgnoreCase("googleads.g.doubleclick.net")) {
                 throw new FoundAdException("Error found ad: " + urlString);
             }
@@ -160,6 +160,7 @@ public String getId(String urlString) throws ParsingException, IllegalArgumentEx
                 return assertIsId(viewQueryValue);
             }
 
+            case "Y2U.BE":
             case "YOUTU.BE": {
                 String viewQueryValue = Utils.getQueryValue(url, "v");
                 if (viewQueryValue != null) {

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java

@@ -32,6 +32,7 @@ public class StreamInfoItem extends InfoItem {
     private final StreamType streamType;
 
     private String uploaderName;
+    private String shortDescription;
     private String textualUploadDate;
     @Nullable
     private DateWrapper uploadDate;
@@ -92,6 +93,14 @@ public void setUploaderAvatarUrl(final String uploaderAvatarUrl) {
         this.uploaderAvatarUrl = uploaderAvatarUrl;
     }
 
+    public String getShortDescription() {
+        return shortDescription;
+    }
+
+    public void setShortDescription(final String shortDescription) {
+        this.shortDescription = shortDescription;
+    }
+
     @Nullable
     public String getTextualUploadDate() {
         return textualUploadDate;

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java

@@ -116,4 +116,14 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor {
     @Nullable
     DateWrapper getUploadDate() throws ParsingException;
 
+
+    /**
+     * Get the video's short description.
+     *
+     * @return The video's short description or {@code null} if not provided by the service.
+     * @throws ParsingException if there is an error in the extraction
+     */
+    @Nullable
+    default String getShortDescription() throws ParsingException { return null; }
+
 }

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java

@@ -101,6 +101,11 @@ public StreamInfoItem extract(StreamInfoItemExtractor extractor) throws ParsingE
         } catch (Exception e) {
             addError(e);
         }
+        try {
+            resultItem.setShortDescription(extractor.getShortDescription());
+        } catch (Exception e) {
+            addError(e);
+        }
 
         return resultItem;
     }