diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Instance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Instance.java index 5f2ab9ecee..615b852b4c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Instance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Instance.java @@ -2,6 +2,7 @@ import org.schabi.newpipe.extractor.exceptions.InvalidInstanceException; +import javax.annotation.Nonnull; import javax.annotation.Nullable; /* @@ -27,6 +28,7 @@ public interface Instance { @Nullable String getName(); + @Nonnull String getUrl(); boolean isValid(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java index 0482f6515b..6ff53d42fb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java @@ -13,13 +13,14 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; import java.io.IOException; public class PeertubeInstance implements Instance { private final String url; private String name; - public static final PeertubeInstance defaultInstance = new PeertubeInstance("https://framatube.org", "FramaTube"); + private static final PeertubeInstance defaultInstance = new PeertubeInstance("https://framatube.org", "FramaTube"); public PeertubeInstance(String url) { this(url, "PeerTube"); @@ -30,6 +31,7 @@ public PeertubeInstance(String url, String name) { this.name = name; } + @Nonnull public String getUrl() { return url; } @@ -66,6 +68,10 @@ public void fetchInstanceMetaData() throws InvalidInstanceException { } } + public static Instance getDefaultInstance() { + return defaultInstance; + } + public String getName() { return name; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index 8df2d3414e..3fdc83ce43 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.peertube; +import org.schabi.newpipe.extractor.Instance; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; @@ -59,11 +60,12 @@ * PeertubeService, uses documented API: https://docs.joinpeertube.org/api-rest-reference.html */ public class PeertubeService extends StreamingService { + public PeertubeService(final int id) { - this(id, PeertubeInstance.defaultInstance); + this(id, PeertubeInstance.getDefaultInstance()); } - public PeertubeService(final int id, final PeertubeInstance instance) { + public PeertubeService(final int id, final Instance instance) { super(id, "PeerTube", asList(VIDEO, COMMENTS, INSTANCES)); setInstance(instance); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstance.java index f61cc1e5a9..0f754f083c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstance.java @@ -3,7 +3,6 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; - import org.schabi.newpipe.extractor.Instance; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -14,12 +13,12 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import javax.annotation.Nullable; - import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isHooktubeURL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL; @@ -46,8 +45,7 @@ public class InvidiousInstance implements Instance { private final String url; private String name; - public static final InvidiousInstance defaultInstance = new InvidiousInstance("http://192.168.1.11:3000", "invidious"); - // todo: change this. local instance because invidio.us is semi-blocked (no metadata) + private static final InvidiousInstance defaultInstance = new InvidiousInstance("https://invidio.us", "invidious"); public InvidiousInstance(String url, String name) { this.url = url; @@ -64,6 +62,7 @@ public String getName() { return name; } + @Nonnull @Override public String getUrl() { return url; @@ -112,4 +111,9 @@ public void fetchInstanceMetaData() throws InvalidInstanceException { throw new InvalidInstanceException("unable to parse instance config", e); } } + + public static Instance getDefaultInstance() { + return defaultInstance; + } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java index 54f93354ad..97beed99f1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java @@ -18,6 +18,7 @@ * along with NewPipe Extractor. If not, see . */ +import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; @@ -50,6 +51,26 @@ public static JsonObject getValidJsonObjectFromResponse(final Response response, } } + /** + * Used to check HTTP code and handle Json parsing. + * + * @param response the response got from the service + * @param apiUrl the url used to call the service + * @return a valid JsonArray + * @throws ExtractionException if the HTTP code indicate an error or the json parsing went wrong. + */ + public static JsonArray getValidJsonArrayFromResponse(final Response response, final String apiUrl) throws ExtractionException { + if (response.responseCode() >= 400) { + throw new ExtractionException("Could not get page " + apiUrl + " (" + response.responseCode() + " : " + response.responseMessage()); + } + + try { + return JsonParser.array().from(response.responseBody()); + } catch (JsonParserException e) { + throw new ExtractionException("Could not parse json", e); + } + } + public static DateWrapper getUploadDateFromEpochTime(final long epochTime) { Calendar calendar = Calendar.getInstance(); calendar.setTime(new Date(epochTime * 1000)); // * 1000 because it's second-based, not millisecond based diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousService.java index 164842e731..88f7b0aa58 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousService.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube.invidious; +import org.schabi.newpipe.extractor.Instance; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; @@ -16,6 +17,7 @@ import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.invidious.extractors.InvidiousChannelExtractor; import org.schabi.newpipe.extractor.services.youtube.invidious.extractors.InvidiousCommentsExtractor; import org.schabi.newpipe.extractor.services.youtube.invidious.extractors.InvidiousStreamExtractor; import org.schabi.newpipe.extractor.services.youtube.invidious.extractors.InvidiousSuggestionExtractor; @@ -62,10 +64,10 @@ public class InvidiousService extends StreamingService { public InvidiousService(int id) { - this(id, InvidiousInstance.defaultInstance); + this(id, InvidiousInstance.getDefaultInstance()); } - public InvidiousService(final int id, final InvidiousInstance instance) { + public InvidiousService(final int id, final Instance instance) { super(id, "Invidious", asList(AUDIO, VIDEO, LIVE, COMMENTS, INSTANCES)); setInstance(instance); } @@ -102,7 +104,7 @@ public StreamExtractor getStreamExtractor(LinkHandler linkHandler) { @Override public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) { - return null; + return new InvidiousChannelExtractor(this, linkHandler); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java new file mode 100644 index 0000000000..0cd4387c63 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java @@ -0,0 +1,137 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +import javax.annotation.Nonnull; +import java.io.IOException; + +/* + * Copyright (C) 2020 Team NewPipe + * InvidiousChannelExtractor.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + +public class InvidiousChannelExtractor extends ChannelExtractor { + + private final String baseUrl; + private JsonObject json; + + public InvidiousChannelExtractor(StreamingService service, ListLinkHandler linkHandler) { + super(service, linkHandler); + this.baseUrl = service.getInstance().getUrl(); + } + + @Override + public String getAvatarUrl() { + return json.getArray("authorThumbnails").getObject(0).getString("url"); + } + + @Override + public String getBannerUrl() { + return json.getArray("authorBanners").getObject(0).getString("url"); + } + + @Override + public String getFeedUrl() { + return baseUrl + "/feed/channel/" + json.getString("authorId"); + } + + @Override + public long getSubscriberCount() { + return json.getNumber("subCount").longValue(); + } + + @Override + public String getDescription() throws ParsingException { + return json.getString("description"); + } + + @Override + public String getParentChannelName() throws ParsingException { + return null; + } + + @Override + public String getParentChannelUrl() throws ParsingException { + return null; + } + + @Override + public String getParentChannelAvatarUrl() throws ParsingException { + return null; + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + final Downloader dl = NewPipe.getDownloader(); + final String apiUrl = getPageUrl(1); + final Response rp = dl.get(apiUrl); + final JsonArray array = InvidiousParsingHelper.getValidJsonArrayFromResponse(rp, apiUrl); + + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + collectStreamsFrom(collector, array); + return new InfoItemsPage<>(collector, getNextPageUrl()); + } + + @Override + public String getNextPageUrl() { + return getPageUrl(2); + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + return null; + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + final String apiUrl = baseUrl + "/api/v1/channels/" + getId() + + "?fields=author,description,subCount,authorThumbnails,authorBanners,authorId" + + "®ion=" + getExtractorContentCountry().getCountryCode(); + + final Response response = downloader.get(apiUrl); + + json = InvidiousParsingHelper.getValidJsonObjectFromResponse(response, apiUrl); + } + + @Nonnull + @Override + public String getName() { + return json.getString("author"); + } + + private String getPageUrl(int page) { + return baseUrl + "/api/v1/channels/videos/" + json.getString("authorId") + "?page=" + page; + } + + private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) { + for (Object o : videos) { + collector.commit(new InvidiousStreamInfoItemExtractor((JsonObject) o, baseUrl)); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java index f219df8af8..cde8c44e04 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java @@ -27,13 +27,13 @@ import javax.annotation.Nullable; import java.io.IOException; import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import static org.schabi.newpipe.extractor.utils.Utils.isBlank; + /* * Copyright (C) 2020 Team NewPipe * InvidiousStreamExtractor.java is part of NewPipe Extractor. @@ -85,7 +85,12 @@ public String getThumbnailUrl() { @Nonnull @Override public Description getDescription() { - return new Description(json.getString("descriptionHtml"), Description.HTML); + final String descriptionHtml = json.getString("descriptionHtml"); + if (!isBlank(descriptionHtml) || descriptionHtml.equals("

")) { + return new Description(descriptionHtml, Description.HTML); + } + + return new Description(json.getString("description"), Description.PLAIN_TEXT); } @Override @@ -263,19 +268,16 @@ public StreamInfoItem getNextStream() { @Override public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException { - /* try { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final JsonArray relatedStreams = json.getArray("recommendedVideos"); for (Object o : relatedStreams) { - collector.commit(new InvidiousStreamInfoItemExtractor((JsonObject) o, new InvidiousInstance(baseUrl))); + collector.commit(new InvidiousStreamInfoItemExtractor((JsonObject) o, baseUrl)); } return collector; } catch (Exception e) { throw new ParsingException("Could not get related videos", e); } - */ - return null; } @Override @@ -304,7 +306,7 @@ public String getCategory() { @Nonnull @Override public String getLicence() { - return null; + return ""; } @Nullable @@ -322,21 +324,17 @@ public List getTags() { @Nonnull @Override public String getSupportInfo() { - return null; + return ""; } @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String apiUrl = baseUrl + "/api/v1/videos/" + getId() + - "?fields=title,descriptionHtml,viewCount,likeCount,dislikeCount,genre,authorUrl,author," + - "authorThumbnails,lengthSeconds,authorThumbnails,hlsUrl,captions,isListed,dashUrl," + - "publishedText,published,isFamilyFriendly,keywords,adaptiveFormats,formatStreams,recommendedVideos" + - "®ion=" + getExtractorContentCountry().getCountryCode(); + "?region=" + getExtractorContentCountry().getCountryCode(); final Response response = downloader.get(apiUrl); json = InvidiousParsingHelper.getValidJsonObjectFromResponse(response, apiUrl); - } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamInfoItemExtractor.java index 270af8b98f..bc415a955b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamInfoItemExtractor.java @@ -3,12 +3,14 @@ import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousInstance; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper.getUploadDateFromEpochTime; + /* * Copyright (C) 2020 Team NewPipe * InvidiousStreamInfoItemExtractor.java is part of NewPipe Extractor. @@ -29,22 +31,26 @@ public class InvidiousStreamInfoItemExtractor implements StreamInfoItemExtractor { - private JsonObject json; - private InvidiousInstance instance; + private final JsonObject json; + private final String baseUrl; - public InvidiousStreamInfoItemExtractor(JsonObject json, InvidiousInstance instance) { + public InvidiousStreamInfoItemExtractor(JsonObject json, String baseUrl) { this.json = json; - this.instance = instance; + this.baseUrl = baseUrl; } @Override - public StreamType getStreamType() throws ParsingException { - return null; + public StreamType getStreamType() { + if (json.getBoolean("liveNow")) { + return StreamType.LIVE_STREAM; + } else { + return StreamType.VIDEO_STREAM; + } } @Override - public boolean isAd() throws ParsingException { - return false; + public boolean isAd() { + return json.getBoolean("premium") /*|| json.getBoolean("paid")*/; // not sure about this one } @Override @@ -54,7 +60,17 @@ public long getDuration() { @Override public long getViewCount() { - return json.getNumber("viewCountText").longValue(); + final Number viewCount = json.getNumber("viewCountText"); + if (viewCount != null) { + return viewCount.longValue(); + } + + final String viewCountText = json.getString("viewCountText"); + try { + return Utils.mixedNumberWordToLong(viewCountText); + } catch (ParsingException e) { + return -1; + } } @Override @@ -65,18 +81,28 @@ public String getUploaderName() { @Override public String getUploaderUrl() { final String url = json.getString("authorUrl"); - return url != null ? url : instance.getUrl() + "/channel/" + json.getString("authorId"); + return url != null ? url : baseUrl + "/channel/" + json.getString("authorId"); } @Nullable @Override public String getTextualUploadDate() { - return null; + return json.getString("publishedText"); } @Nullable @Override public DateWrapper getUploadDate() { + final Number epochTime = json.getNumber("published"); + if (epochTime != null) { + return getUploadDateFromEpochTime(epochTime.longValue()); + } + + // maybe use getTextualUploadDate() BUT is unstable because it depends on instance localization + // (or configuration? I mean servers' OS language. That's something to investigate). + // then it won't always be English, and there is no way to know the language from the API + // therefore we should check if the string contains "ago" -> english + return null; } @@ -87,11 +113,11 @@ public String getName() { @Override public String getUrl() throws ParsingException { - return instance.getUrl() + "/watch?v=" + json.getString("videoId"); + return baseUrl + "/watch?v=" + json.getString("videoId"); } @Override public String getThumbnailUrl() throws ParsingException { - return null; + return json.getArray("videoThumbnails").getObject(0).getString("url"); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousStreamExtractorTest.java index 74df7d21fa..270bbb8108 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousStreamExtractorTest.java @@ -1,7 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube.invidious; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.MediaFormat; @@ -161,7 +160,6 @@ public void testGetDashMpd() { assertTrue(extractor.getDashMpdUrl().endsWith("api/manifest/dash/id/YQHsXMglC9A")); } - @Ignore @Test public void testGetRelatedVideos() throws ExtractionException { final StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();