Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SoundCloud] Detect whether there are any more search results #1081

Merged
merged 4 commits into from
Aug 6, 2023
Merged

[SoundCloud] Detect whether there are any more search results #1081

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2fb9922
[SoundCloud] Detect whether there are any more search results
TobiGr Jul 14, 2023
aa6c17d
[SoundCloud] Deduplicate some code
TobiGr Aug 3, 2023
485bfbc
[SoundCloud] Move try-catch inside getOffsetFromUrl
Stypox Aug 6, 2023
2947257
[SoundCloud] Properly calculate if results have finished
Stypox Aug 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 15 additions & 22 deletions ...c/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,9 @@
package org.schabi.newpipe.extractor.services.soundcloud;

import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;

import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
Expand All @@ -28,6 +23,7 @@
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import org.schabi.newpipe.extractor.utils.Utils;

import javax.annotation.Nonnull;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
Expand All @@ -38,7 +34,9 @@
import java.util.List;
import java.util.Map;

import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;

public final class SoundcloudParsingHelper {
private static String clientId;
Expand Down Expand Up @@ -200,6 +198,7 @@ public static String getUsersFromApiMinItems(final int minItems,
*
* @return the next streams url, empty if don't have
*/
@Nonnull
public static String getUsersFromApi(final ChannelInfoItemsCollector collector,
final String apiUrl) throws IOException,
ReCaptchaException, ParsingException {
Expand All @@ -221,17 +220,7 @@ public static String getUsersFromApi(final ChannelInfoItemsCollector collector,
}
}

String nextPageUrl;
try {
nextPageUrl = responseObject.getString("next_href");
if (!nextPageUrl.contains("client_id=")) {
nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
}
} catch (final Exception ignored) {
nextPageUrl = "";
}

return nextPageUrl;
return getNextPageUrl(responseObject);
}

/**
Expand Down Expand Up @@ -261,6 +250,7 @@ public static String getStreamsFromApiMinItems(final int minItems,
*
* @return the next streams url, empty if don't have
*/
@Nonnull
public static String getStreamsFromApi(final StreamInfoItemsCollector collector,
final String apiUrl,
final boolean charts) throws IOException,
Expand Down Expand Up @@ -288,17 +278,20 @@ public static String getStreamsFromApi(final StreamInfoItemsCollector collector,
}
}

String nextPageUrl;
return getNextPageUrl(responseObject);
}

@Nonnull
private static String getNextPageUrl(@Nonnull final JsonObject response) {
try {
nextPageUrl = responseObject.getString("next_href");
String nextPageUrl = response.getString("next_href");
if (!nextPageUrl.contains("client_id=")) {
nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
}
return nextPageUrl;
} catch (final Exception ignored) {
nextPageUrl = "";
return "";
}

return nextPageUrl;
}

public static String getStreamsFromApi(final StreamInfoItemsCollector collector,
Expand Down
28 changes: 9 additions & 19 deletions .../schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;

import java.io.IOException;
Expand All @@ -33,22 +34,7 @@ public SoundcloudCommentsExtractor(final StreamingService service,
@Override
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws ExtractionException,
IOException {
final Downloader downloader = NewPipe.getDownloader();
final Response response = downloader.get(getUrl());

final JsonObject json;
try {
json = JsonParser.object().from(response.responseBody());
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json", e);
}

final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
getServiceId());

collectStreamsFrom(collector, json.getArray("collection"));

return new InfoItemsPage<>(collector, new Page(json.getString("next_href")));
return getPage(getUrl());
}

@Override
Expand All @@ -57,9 +43,14 @@ public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws Extractio
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
return getPage(page.getUrl());
}

@Nonnull
private InfoItemsPage<CommentsInfoItem> getPage(@Nonnull final String url)
throws ParsingException, IOException, ReCaptchaException {
final Downloader downloader = NewPipe.getDownloader();
final Response response = downloader.get(page.getUrl());
final Response response = downloader.get(url);

final JsonObject json;
try {
Expand All @@ -72,8 +63,7 @@ public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws Extractio
getServiceId());

collectStreamsFrom(collector, json.getArray("collection"));

return new InfoItemsPage<>(collector, new Page(json.getString("next_href")));
return new InfoItemsPage<>(collector, new Page(json.getString("next_href", null)));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You do not needed to specify a null fallback, as that's already the default value.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's right, however this is fine, too

}

@Override
Expand Down
47 changes: 34 additions & 13 deletions ...rg/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@
import javax.annotation.Nonnull;

public class SoundcloudSearchExtractor extends SearchExtractor {
private JsonArray initialSearchCollection;
private JsonObject initialSearchObject;
private static final String COLLECTION = "collection";
private static final String TOTAL_RESULTS = "total_results";

public SoundcloudSearchExtractor(final StreamingService service,
final SearchQueryHandler linkHandler) {
Expand All @@ -60,9 +62,15 @@ public List<MetaInfo> getMetaInfo() {
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
return new InfoItemsPage<>(
collectItems(initialSearchCollection),
getNextPageFromCurrentUrl(getUrl(), currentOffset -> ITEMS_PER_PAGE));
if (initialSearchObject.getInt(TOTAL_RESULTS) > ITEMS_PER_PAGE) {
return new InfoItemsPage<>(
collectItems(initialSearchObject.getArray(COLLECTION)),
getNextPageFromCurrentUrl(getUrl(), currentOffset -> ITEMS_PER_PAGE));
} else {
return new InfoItemsPage<>(
collectItems(initialSearchObject.getArray(COLLECTION)), null);
}

}

@Override
Expand All @@ -74,17 +82,23 @@ public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException,

final Downloader dl = getDownloader();
final JsonArray searchCollection;
final int totalResults;
try {
final String response = dl.get(page.getUrl(), getExtractorLocalization())
.responseBody();
searchCollection = JsonParser.object().from(response).getArray("collection");
final JsonObject result = JsonParser.object().from(response);
searchCollection = result.getArray(COLLECTION);
totalResults = result.getInt(TOTAL_RESULTS);
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}

return new InfoItemsPage<>(collectItems(searchCollection),
getNextPageFromCurrentUrl(page.getUrl(),
currentOffset -> currentOffset + ITEMS_PER_PAGE));
if (getOffsetFromUrl(page.getUrl()) + ITEMS_PER_PAGE < totalResults) {
return new InfoItemsPage<>(collectItems(searchCollection),
getNextPageFromCurrentUrl(page.getUrl(),
currentOffset -> currentOffset + ITEMS_PER_PAGE));
}
return new InfoItemsPage<>(collectItems(searchCollection), null);
}

@Override
Expand All @@ -94,12 +108,12 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
final String url = getUrl();
try {
final String response = dl.get(url, getExtractorLocalization()).responseBody();
initialSearchCollection = JsonParser.object().from(response).getArray("collection");
initialSearchObject = JsonParser.object().from(response);
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}

if (initialSearchCollection.isEmpty()) {
if (initialSearchObject.getArray(COLLECTION).isEmpty()) {
throw new SearchExtractor.NothingFoundException("Nothing found");
}
}
Expand Down Expand Up @@ -133,13 +147,20 @@ private InfoItemsCollector<InfoItem, InfoItemExtractor> collectItems(

private Page getNextPageFromCurrentUrl(final String currentUrl,
final IntUnaryOperator newPageOffsetCalculator)
throws MalformedURLException, UnsupportedEncodingException {
final int currentPageOffset = Integer.parseInt(
Parser.compatParseMap(new URL(currentUrl).getQuery()).get("offset"));
throws ParsingException {
final int currentPageOffset = getOffsetFromUrl(currentUrl);

return new Page(
currentUrl.replace(
"&offset=" + currentPageOffset,
"&offset=" + newPageOffsetCalculator.applyAsInt(currentPageOffset)));
}

private int getOffsetFromUrl(final String url) throws ParsingException {
try {
return Integer.parseInt(Parser.compatParseMap(new URL(url).getQuery()).get("offset"));
} catch (MalformedURLException | UnsupportedEncodingException e) {
throw new ParsingException("Could not get offset from page URL", e);
}
}
}
24 changes: 24 additions & 0 deletions ...rg/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.soundcloud.search;

import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems;
Expand Down Expand Up @@ -181,4 +182,27 @@ void testIsVerified() throws IOException, ExtractionException {
assertTrue(verified);
}
}

public static class NoNextPage extends DefaultSearchExtractorTest {

private static SearchExtractor extractor;
private static final String QUERY = "Dan at hor#berlgbd";

@BeforeAll
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = SoundCloud.getSearchExtractor(QUERY);
extractor.fetchPage();
}

@Override public boolean expectedHasMoreItems() { return false; }
@Override public SearchExtractor extractor() throws Exception { return extractor; }
@Override public StreamingService expectedService() throws Exception { return SoundCloud; }
@Override public String expectedName() throws Exception { return QUERY; }
@Override public String expectedId() throws Exception { return QUERY; }
@Override public String expectedUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
}
}