Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[YouTube] Now music mixes can be treated as normal mixes #1013

Merged
merged 2 commits into from
Jan 29, 2023
Merged

[YouTube] Now music mixes can be treated as normal mixes #1013

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7293991
[YouTube] Now music mixes can be treated as normal mixes
Stypox Jan 15, 2023
5945057
[YouTube] Add music mix test
Stypox Jan 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions ...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -388,8 +388,7 @@ public static OffsetDateTime parseDateFrom(final String textualUploadDate)
* @return Whether given id belongs to a YouTube Mix
*/
public static boolean isYoutubeMixId(@Nonnull final String playlistId) {
return playlistId.startsWith("RD")
&& !isYoutubeMusicMixId(playlistId);
return playlistId.startsWith("RD");
}

/**
Expand Down
3 changes: 1 addition & 2 deletions extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,7 @@ public ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler) {

@Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())
&& !YoutubeParsingHelper.isYoutubeMusicMixId(linkHandler.getId())) {
if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) {
return new YoutubeMixPlaylistExtractor(this, linkHandler);
} else {
return new YoutubePlaylistExtractor(this, linkHandler);
Expand Down
104 changes: 92 additions & 12 deletions ...t/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,22 +23,22 @@
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

@SuppressWarnings({"MismatchedQueryAndUpdateOfCollection", "NewClassNamingConvention"})
public class YoutubeMixPlaylistExtractorTest {

private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/mix/";
private static final Map<String, String> dummyCookie = new HashMap<>();

private static final Map<String, String> dummyCookie = Map.of(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
private static YoutubeMixPlaylistExtractor extractor;

public static class Mix {
Expand All @@ -50,7 +50,6 @@ public static void setUp() throws Exception {
YoutubeTestsUtils.ensureStateless();
YoutubeParsingHelper.setConsentAccepted(true);
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mix"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID
+ "&list=RD" + VIDEO_ID);
Expand Down Expand Up @@ -135,7 +134,6 @@ void getPlaylistType() throws ParsingException {
}

public static class MixWithIndex {

private static final String VIDEO_ID = "FAqYW76GLPA";
private static final String VIDEO_TITLE = "Mix – ";
private static final int INDEX = 7; // YT starts the index with 1...
Expand All @@ -146,7 +144,6 @@ public static void setUp() throws Exception {
YoutubeTestsUtils.ensureStateless();
YoutubeParsingHelper.setConsentAccepted(true);
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mixWithIndex"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID_AT_INDEX
+ "&list=RD" + VIDEO_ID + "&index=" + INDEX);
Expand Down Expand Up @@ -233,7 +230,6 @@ public static void setUp() throws Exception {
YoutubeTestsUtils.ensureStateless();
YoutubeParsingHelper.setConsentAccepted(true);
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "myMix"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID
+ "&list=RDMM" + VIDEO_ID);
Expand Down Expand Up @@ -316,15 +312,13 @@ void getPlaylistType() throws ParsingException {
}

public static class Invalid {

private static final String VIDEO_ID = "QMVCAPd5cwBcg";

@BeforeAll
public static void setUp() throws IOException {
YoutubeTestsUtils.ensureStateless();
YoutubeParsingHelper.setConsentAccepted(true);
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "invalid"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
}

@Test
Expand All @@ -348,7 +342,6 @@ void invalidVideoId() throws Exception {
}

public static class ChannelMix {

private static final String CHANNEL_ID = "UCXuqSBlHAE6Xw-yeJA0Tunw";
private static final String VIDEO_ID_OF_CHANNEL = "mnk6gnOBYIo";
private static final String CHANNEL_TITLE = "Linus Tech Tips";
Expand All @@ -359,7 +352,6 @@ public static void setUp() throws Exception {
YoutubeTestsUtils.ensureStateless();
YoutubeParsingHelper.setConsentAccepted(true);
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "channelMix"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+ "&list=RDCM" + CHANNEL_ID);
Expand Down Expand Up @@ -424,7 +416,6 @@ public static void setUp() throws Exception {
YoutubeTestsUtils.ensureStateless();
YoutubeParsingHelper.setConsentAccepted(true);
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "genreMix"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID
+ "&list=RDGMEMYH9CUrFO7CfLJpaD7UR85w");
Expand Down Expand Up @@ -504,4 +495,93 @@ void getPlaylistType() throws ParsingException {
assertEquals(PlaylistInfo.PlaylistType.MIX_GENRE, extractor.getPlaylistType());
}
}

public static class Music {
private static final String VIDEO_ID = "dQw4w9WgXcQ";
private static final String MIX_TITLE = "Mix – Rick Astley - Never Gonna Give You Up (Official Music Video)";

@BeforeAll
public static void setUp() throws Exception {
YoutubeTestsUtils.ensureStateless();
YoutubeParsingHelper.setConsentAccepted(true);
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "musicMix"));
extractor = (YoutubeMixPlaylistExtractor)
YouTube.getPlaylistExtractor("https://m.youtube.com/watch?v=" + VIDEO_ID
+ "&list=RDAMVM" + VIDEO_ID);
extractor.fetchPage();
}

@Test
void getServiceId() {
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
}

@Test
void getName() throws Exception {
assertEquals(MIX_TITLE, extractor.getName());
}

@Test
void getThumbnailUrl() throws Exception {
final String thumbnailUrl = extractor.getThumbnailUrl();
assertIsSecureUrl(thumbnailUrl);
ExtractorAsserts.assertContains("yt", thumbnailUrl);
ExtractorAsserts.assertContains(VIDEO_ID, thumbnailUrl);
}

@Test
void getInitialPage() throws Exception {
final InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}

@Test
void getPage() throws Exception {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
.value("videoId", VIDEO_ID)
.value("playlistId", "RD" + VIDEO_ID)
.value("params", "OAE%3D")
.done())
.getBytes(StandardCharsets.UTF_8);

final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}

@Test
void getContinuations() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
final Set<String> urls = new HashSet<>();

// Should work infinitely, but for testing purposes only 3 times
for (int i = 0; i < 3; i++) {
assertTrue(streams.hasNextPage());
assertFalse(streams.getItems().isEmpty());

for (final StreamInfoItem item : streams.getItems()) {
// TODO Duplicates are appearing
// assertFalse(urls.contains(item.getUrl()));
urls.add(item.getUrl());
}

streams = extractor.getPage(streams.getNextPage());
}
assertTrue(streams.hasNextPage());
assertFalse(streams.getItems().isEmpty());
}

@Test
void getStreamCount() throws ParsingException {
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
}

@Test
void getPlaylistType() throws ParsingException {
assertEquals(PlaylistInfo.PlaylistType.MIX_MUSIC, extractor.getPlaylistType());
}
}
}
82 changes: 82 additions & 0 deletions ...rg/schabi/newpipe/extractor/services/youtube/extractor/mix/musicMix/generated_mock_0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
{
"request": {
"httpMethod": "GET",
"url": "https://www.youtube.com/sw.js",
"headers": {
"Origin": [
"https://www.youtube.com"
],
"Referer": [
"https://www.youtube.com"
],
"Accept-Language": [
"en-GB, en;q\u003d0.9"
]
},
"localization": {
"languageCode": "en",
"countryCode": "GB"
}
},
"response": {
"responseCode": 200,
"responseMessage": "",
"responseHeaders": {
"access-control-allow-credentials": [
"true"
],
"access-control-allow-origin": [
"https://www.youtube.com"
],
"alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
],
"cache-control": [
"private, max-age\u003d0"
],
"content-type": [
"text/javascript; charset\u003dutf-8"
],
"cross-origin-opener-policy-report-only": [
"same-origin; report-to\u003d\"youtube_main\""
],
"date": [
"Sun, 15 Jan 2023 22:30:06 GMT"
],
"expires": [
"Sun, 15 Jan 2023 22:30:06 GMT"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
],
"permissions-policy": [
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
],
"report-to": [
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
],
"server": [
"ESF"
],
"set-cookie": [
"YSC\u003d8TYEubKDjFA; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 20-Apr-2020 22:30:06 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+835; expires\u003dTue, 14-Jan-2025 22:30:06 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
],
"strict-transport-security": [
"max-age\u003d31536000"
],
"x-content-type-options": [
"nosniff"
],
"x-frame-options": [
"SAMEORIGIN"
],
"x-xss-protection": [
"0"
]
},
"responseBody": "\n self.addEventListener(\u0027install\u0027, event \u003d\u003e {\n event.waitUntil(self.skipWaiting());\n });\n self.addEventListener(\u0027activate\u0027, event \u003d\u003e {\n event.waitUntil(\n self.clients.claim().then(() \u003d\u003e self.registration.unregister()));\n });\n ",
"latestUrl": "https://www.youtube.com/sw.js"
}
}
78 changes: 78 additions & 0 deletions ...rg/schabi/newpipe/extractor/services/youtube/extractor/mix/musicMix/generated_mock_1.json
View file
Open in desktop

Large diffs are not rendered by default.

Loading