Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Soundcloud] Remove DRM-protected and downloadable formats extraction #1269

Merged
merged 2 commits into from
Feb 3, 2025
Merged

[Soundcloud] Remove DRM-protected and downloadable formats extraction #1269

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7607688
[Soundcloud] Remove DRM-protected and downloadable formats extraction
AudricV Jan 18, 2025
a336a8c
[Soundcloud] Update SoundcloudStreamExtractorTest with latest changes
AudricV Jan 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 18 additions & 77 deletions ...rg/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromTrackObject;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAvatarUrl;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.parseDateFrom;
import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;

Expand Down Expand Up @@ -170,34 +169,29 @@ public List<AudioStream> getAudioStreams() throws ExtractionException {
}

try {
final JsonArray transcodings = track.getObject("media").getArray("transcodings");
final JsonArray transcodings = track.getObject("media")
.getArray("transcodings");
if (!isNullOrEmpty(transcodings)) {
// Get information about what stream formats are available
extractAudioStreams(transcodings, checkMp3ProgressivePresence(transcodings),
audioStreams);
extractAudioStreams(transcodings, audioStreams);
}

extractDownloadableFileIfAvailable(audioStreams);
} catch (final NullPointerException e) {
throw new ExtractionException("Could not get audio streams", e);
}

return audioStreams;
}

private static boolean checkMp3ProgressivePresence(@Nonnull final JsonArray transcodings) {
return transcodings.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.anyMatch(transcodingJsonObject -> transcodingJsonObject.getString("preset")
.contains("mp3") && transcodingJsonObject.getObject("format")
.getString("protocol").equals("progressive"));
}

@Nonnull
private String getTranscodingUrl(final String endpointUrl)
throws IOException, ExtractionException {
final String apiStreamUrl = endpointUrl + "?client_id=" + clientId();
String apiStreamUrl = endpointUrl + "?client_id=" + clientId();

final String trackAuthorization = track.getString("track_authorization");
if (!isNullOrEmpty(trackAuthorization)) {
apiStreamUrl += "&track_authorization=" + trackAuthorization;
}

final String response = NewPipe.getDownloader().get(apiStreamUrl).responseBody();
final JsonObject urlObject;
try {
Expand All @@ -209,27 +203,7 @@ private String getTranscodingUrl(final String endpointUrl)
return urlObject.getString("url");
}

@Nullable
private String getDownloadUrl(@Nonnull final String trackId)
throws IOException, ExtractionException {
final String response = NewPipe.getDownloader().get(SOUNDCLOUD_API_V2_URL + "tracks/"
+ trackId + "/download" + "?client_id=" + clientId()).responseBody();

final JsonObject downloadJsonObject;
try {
downloadJsonObject = JsonParser.object().from(response);
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse download URL", e);
}
final String redirectUri = downloadJsonObject.getString("redirectUri");
if (!isNullOrEmpty(redirectUri)) {
return redirectUri;
}
return null;
}

private void extractAudioStreams(@Nonnull final JsonArray transcodings,
final boolean mp3ProgressiveInStreams,
final List<AudioStream> audioStreams) {
transcodings.stream()
.filter(JsonObject.class::isInstance)
Expand All @@ -244,23 +218,23 @@ private void extractAudioStreams(@Nonnull final JsonArray transcodings,
final String preset = transcoding.getString("preset", ID_UNKNOWN);
final String protocol = transcoding.getObject("format")
.getString("protocol");

if (protocol.contains("encrypted")) {
// Skip DRM-protected streams, which have encrypted in their protocol
// name
return;
}

final AudioStream.Builder builder = new AudioStream.Builder()
.setId(preset);

final boolean isHls = protocol.equals("hls");
if (isHls) {
if (protocol.equals("hls")) {
builder.setDeliveryMethod(DeliveryMethod.HLS);
}

builder.setContent(getTranscodingUrl(url), true);

if (preset.contains("mp3")) {
// Don't add the MP3 HLS stream if there is a progressive stream
// present because both have the same bitrate
if (mp3ProgressiveInStreams && isHls) {
return;
}

builder.setMediaFormat(MediaFormat.MP3);
builder.setAverageBitrate(128);
} else if (preset.contains("opus")) {
Expand All @@ -283,39 +257,6 @@ private void extractAudioStreams(@Nonnull final JsonArray transcodings,
});
}

/**
* Add the downloadable format if it is available.
*
* <p>
* A track can have the {@code downloadable} boolean set to {@code true}, but it doesn't mean
* we can download it.
* </p>
*
* <p>
* If the value of the {@code has_download_left} boolean is {@code true}, the track can be
* downloaded, and not otherwise.
* </p>
*
* @param audioStreams the audio streams to which the downloadable file is added
*/
public void extractDownloadableFileIfAvailable(final List<AudioStream> audioStreams) {
if (track.getBoolean("downloadable") && track.getBoolean("has_downloads_left")) {
try {
final String downloadUrl = getDownloadUrl(getId());
if (!isNullOrEmpty(downloadUrl)) {
audioStreams.add(new AudioStream.Builder()
.setId("original-format")
.setContent(downloadUrl, true)
.setAverageBitrate(UNKNOWN_BITRATE)
.build());
}
} catch (final Exception ignored) {
// If something went wrong when trying to get the download URL, ignore the
// exception throw because this "stream" is not necessary to play the track
}
}
}

@Override
public List<VideoStream> getVideoStreams() {
return Collections.emptyList();
Expand Down
32 changes: 20 additions & 12 deletions .../java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.fail;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;

public class SoundcloudStreamExtractorTest {
Expand Down Expand Up @@ -188,26 +189,33 @@ public static void setUp() throws Exception {
public void testAudioStreams() throws Exception {
super.testAudioStreams();
final List<AudioStream> audioStreams = extractor.getAudioStreams();
assertEquals(2, audioStreams.size());
assertEquals(3, audioStreams.size()); // 2 MP3 streams (1 progressive, 1 HLS) and 1 OPUS
audioStreams.forEach(audioStream -> {
final DeliveryMethod deliveryMethod = audioStream.getDeliveryMethod();
final String mediaUrl = audioStream.getContent();
if (audioStream.getFormat() == MediaFormat.OPUS) {
// Assert that it's an OPUS 64 kbps media URL with a single range which comes
// from an HLS SoundCloud CDN
ExtractorAsserts.assertContains("-hls-opus-media.sndcdn.com", mediaUrl);
ExtractorAsserts.assertContains(".64.opus", mediaUrl);
assertSame(DeliveryMethod.HLS, deliveryMethod,
"Wrong delivery method for stream " + audioStream.getId() + ": "
+ deliveryMethod);
} else if (audioStream.getFormat() == MediaFormat.MP3) {
// Assert that it's a MP3 128 kbps media URL which comes from a progressive
// Assert it's an OPUS 64 kbps media playlist URL which comes from an HLS
// SoundCloud CDN
ExtractorAsserts.assertContains("-media.sndcdn.com/bKOA7Pwbut93.128.mp3",
mediaUrl);
assertSame(DeliveryMethod.PROGRESSIVE_HTTP, deliveryMethod,
"Wrong delivery method for stream " + audioStream.getId() + ": "
+ deliveryMethod);
ExtractorAsserts.assertContains("-hls-opus-media.sndcdn.com", mediaUrl);
ExtractorAsserts.assertContains(".64.opus", mediaUrl);
} else if (audioStream.getFormat() == MediaFormat.MP3) {
if (deliveryMethod == DeliveryMethod.PROGRESSIVE_HTTP) {
// Assert it's a MP3 128 kbps media URL which comes from a progressive
// SoundCloud CDN
ExtractorAsserts.assertContains("-media.sndcdn.com/bKOA7Pwbut93.128.mp3",
mediaUrl);
} else if (deliveryMethod == DeliveryMethod.HLS) {
// Assert it's a MP3 128 kbps media HLS playlist URL which comes from an HLS
// SoundCloud CDN
ExtractorAsserts.assertContains("-hls-media.sndcdn.com", mediaUrl);
ExtractorAsserts.assertContains(".128.mp3", mediaUrl);
} else {
fail("Wrong delivery method for stream " + audioStream.getId() + ": "
+ deliveryMethod);
}
}
});
}
Expand Down
Loading