Skip to content

Commit

Permalink
InvidiousChannelExtractor added
Browse files Browse the repository at this point in the history
  • Loading branch information
B0pol committed Jun 12, 2020
1 parent ca0d95e commit bac5f47
Show file tree
Hide file tree
Showing 10 changed files with 237 additions and 41 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import org.schabi.newpipe.extractor.exceptions.InvalidInstanceException;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;

/*
Expand All @@ -27,6 +28,7 @@ public interface Instance {
@Nullable
String getName();

@Nonnull
String getUrl();

boolean isValid();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,14 @@
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;

import javax.annotation.Nonnull;
import java.io.IOException;

public class PeertubeInstance implements Instance {

private final String url;
private String name;
public static final PeertubeInstance defaultInstance = new PeertubeInstance("https://framatube.org", "FramaTube");
private static final PeertubeInstance defaultInstance = new PeertubeInstance("https://framatube.org", "FramaTube");

public PeertubeInstance(String url) {
this(url, "PeerTube");
Expand All @@ -30,6 +31,7 @@ public PeertubeInstance(String url, String name) {
this.name = name;
}

@Nonnull
public String getUrl() {
return url;
}
Expand Down Expand Up @@ -66,6 +68,10 @@ public void fetchInstanceMetaData() throws InvalidInstanceException {
}
}

public static Instance getDefaultInstance() {
return defaultInstance;
}

public String getName() {
return name;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.peertube;

import org.schabi.newpipe.extractor.Instance;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
Expand Down Expand Up @@ -59,11 +60,12 @@
* PeertubeService, uses documented API: https://docs.joinpeertube.org/api-rest-reference.html
*/
public class PeertubeService extends StreamingService {

public PeertubeService(final int id) {
this(id, PeertubeInstance.defaultInstance);
this(id, PeertubeInstance.getDefaultInstance());
}

public PeertubeService(final int id, final PeertubeInstance instance) {
public PeertubeService(final int id, final Instance instance) {
super(id, "PeerTube", asList(VIDEO, COMMENTS, INSTANCES));
setInstance(instance);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;

import org.schabi.newpipe.extractor.Instance;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.downloader.Downloader;
Expand All @@ -14,12 +13,12 @@
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;

import javax.annotation.Nullable;

import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isHooktubeURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL;

Expand All @@ -46,8 +45,7 @@ public class InvidiousInstance implements Instance {
private final String url;
private String name;

public static final InvidiousInstance defaultInstance = new InvidiousInstance("http://192.168.1.11:3000", "invidious");
// todo: change this. local instance because invidio.us is semi-blocked (no metadata)
private static final InvidiousInstance defaultInstance = new InvidiousInstance("https://invidio.us", "invidious");

public InvidiousInstance(String url, String name) {
this.url = url;
Expand All @@ -64,6 +62,7 @@ public String getName() {
return name;
}

@Nonnull
@Override
public String getUrl() {
return url;
Expand Down Expand Up @@ -112,4 +111,9 @@ public void fetchInstanceMetaData() throws InvalidInstanceException {
throw new InvalidInstanceException("unable to parse instance config", e);
}
}

public static Instance getDefaultInstance() {
return defaultInstance;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
* along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
*/

import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
Expand Down Expand Up @@ -50,6 +51,26 @@ public static JsonObject getValidJsonObjectFromResponse(final Response response,
}
}

/**
* Used to check HTTP code and handle Json parsing.
*
* @param response the response got from the service
* @param apiUrl the url used to call the service
* @return a valid JsonArray
* @throws ExtractionException if the HTTP code indicate an error or the json parsing went wrong.
*/
public static JsonArray getValidJsonArrayFromResponse(final Response response, final String apiUrl) throws ExtractionException {
if (response.responseCode() >= 400) {
throw new ExtractionException("Could not get page " + apiUrl + " (" + response.responseCode() + " : " + response.responseMessage());
}

try {
return JsonParser.array().from(response.responseBody());
} catch (JsonParserException e) {
throw new ExtractionException("Could not parse json", e);
}
}

public static DateWrapper getUploadDateFromEpochTime(final long epochTime) {
Calendar calendar = Calendar.getInstance();
calendar.setTime(new Date(epochTime * 1000)); // * 1000 because it's second-based, not millisecond based
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube.invidious;

import org.schabi.newpipe.extractor.Instance;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
Expand All @@ -16,6 +17,7 @@
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.invidious.extractors.InvidiousChannelExtractor;
import org.schabi.newpipe.extractor.services.youtube.invidious.extractors.InvidiousCommentsExtractor;
import org.schabi.newpipe.extractor.services.youtube.invidious.extractors.InvidiousStreamExtractor;
import org.schabi.newpipe.extractor.services.youtube.invidious.extractors.InvidiousSuggestionExtractor;
Expand Down Expand Up @@ -62,10 +64,10 @@
public class InvidiousService extends StreamingService {

public InvidiousService(int id) {
this(id, InvidiousInstance.defaultInstance);
this(id, InvidiousInstance.getDefaultInstance());
}

public InvidiousService(final int id, final InvidiousInstance instance) {
public InvidiousService(final int id, final Instance instance) {
super(id, "Invidious", asList(AUDIO, VIDEO, LIVE, COMMENTS, INSTANCES));
setInstance(instance);
}
Expand Down Expand Up @@ -102,7 +104,7 @@ public StreamExtractor getStreamExtractor(LinkHandler linkHandler) {

@Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) {
return null;
return new InvidiousChannelExtractor(this, linkHandler);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
package org.schabi.newpipe.extractor.services.youtube.invidious.extractors;

import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;

import javax.annotation.Nonnull;
import java.io.IOException;

/*
* Copyright (C) 2020 Team NewPipe <tnp@newpipe.schabi.org>
* InvidiousChannelExtractor.java is part of NewPipe Extractor.
*
* NewPipe Extractor is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
*/

public class InvidiousChannelExtractor extends ChannelExtractor {

private final String baseUrl;
private JsonObject json;

public InvidiousChannelExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
this.baseUrl = service.getInstance().getUrl();
}

@Override
public String getAvatarUrl() {
return json.getArray("authorThumbnails").getObject(0).getString("url");
}

@Override
public String getBannerUrl() {
return json.getArray("authorBanners").getObject(0).getString("url");
}

@Override
public String getFeedUrl() {
return baseUrl + "/feed/channel/" + json.getString("authorId");
}

@Override
public long getSubscriberCount() {
return json.getNumber("subCount").longValue();
}

@Override
public String getDescription() throws ParsingException {
return json.getString("description");
}

@Override
public String getParentChannelName() throws ParsingException {
return null;
}

@Override
public String getParentChannelUrl() throws ParsingException {
return null;
}

@Override
public String getParentChannelAvatarUrl() throws ParsingException {
return null;
}

@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
final Downloader dl = NewPipe.getDownloader();
final String apiUrl = getPageUrl(1);
final Response rp = dl.get(apiUrl);
final JsonArray array = InvidiousParsingHelper.getValidJsonArrayFromResponse(rp, apiUrl);

final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, array);
return new InfoItemsPage<>(collector, getNextPageUrl());
}

@Override
public String getNextPageUrl() {
return getPageUrl(2);
}

@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
return null;
}

@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String apiUrl = baseUrl + "/api/v1/channels/" + getId()
+ "?fields=author,description,subCount,authorThumbnails,authorBanners,authorId"
+ "&region=" + getExtractorContentCountry().getCountryCode();

final Response response = downloader.get(apiUrl);

json = InvidiousParsingHelper.getValidJsonObjectFromResponse(response, apiUrl);
}

@Nonnull
@Override
public String getName() {
return json.getString("author");
}

private String getPageUrl(int page) {
return baseUrl + "/api/v1/channels/videos/" + json.getString("authorId") + "?page=" + page;
}

private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) {
for (Object o : videos) {
collector.commit(new InvidiousStreamInfoItemExtractor((JsonObject) o, baseUrl));
}
}
}
Loading

0 comments on commit bac5f47

Please sign in to comment.