Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Extract Video Short Description in YouTube. #731

Merged
merged 3 commits into from
Oct 15, 2021
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,12 @@ public DateWrapper getUploadDate() throws ParsingException {
return BandcampExtractorHelper.parseDate(getTextualUploadDate());
}

@Nullable
@Override
public String getShortDescription() {
return null;
}

@Override
public String getName() throws ParsingException {
return show.getString("subtitle");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,12 @@ public String getUploaderAvatarUrl() {
return null;
}

@Nullable
@Override
public String getShortDescription() {
return null;
}

@Override
public String getName() {
return discograph.getString("title");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,12 @@ public String getUploaderAvatarUrl() {
return null;
}

@Nullable
@Override
public String getShortDescription() {
return null;
}

/**
* Each track can have its own cover art. Therefore, unless a substitute is provided,
* the thumbnail is extracted using a stream extractor.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,12 @@ public String getUploaderAvatarUrl() {
return null;
}

@Nullable
@Override
public String getShortDescription() {
return null;
}

@Override
public String getName() throws ParsingException {
return resultInfo.getElementsByClass("heading").text();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,4 +95,10 @@ public String getTextualUploadDate() throws ParsingException {
public DateWrapper getUploadDate() throws ParsingException {
return null;
}

@Nullable
@Override
public String getShortDescription() {
return null;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -92,4 +92,10 @@ public DateWrapper getUploadDate() throws ParsingException {
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSzzzz"));
return new DateWrapper(zonedDateTime.toOffsetDateTime(), false);
}

@Nullable
@Override
public String getShortDescription() {
return null;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,12 @@ public DateWrapper getUploadDate() throws ParsingException {
return new DateWrapper(MediaCCCParsingHelper.parseDateFrom(date));
}

@Nullable
@Override
public String getShortDescription() {
return null;
}

@Override
public String getName() throws ParsingException {
return event.getString("title");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,12 @@ public DateWrapper getUploadDate() throws ParsingException {
return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
}

@Nullable
@Override
public String getShortDescription() {
return null;
}

@Override
public StreamType getStreamType() {
return item.getBoolean("isLive") ? StreamType.LIVE_STREAM : StreamType.VIDEO_STREAM;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,12 @@ public DateWrapper getUploadDate() throws ParsingException {
return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualUploadDate()));
}

@Nullable
@Override
public String getShortDescription() {
return null;
}

@Override
public long getViewCount() {
return itemObject.getLong("playback_count");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,12 @@ public DateWrapper getUploadDate() throws ParsingException {
}
}

@Nullable
@Override
public String getShortDescription() {
return null;
}

@Override
public String getName() {
return entryElement.getElementsByTag("title").first().text();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -288,4 +288,17 @@ private OffsetDateTime getDateFromPremiere() throws ParsingException {
throw new ParsingException("Could not parse date from premiere: \"" + startTime + "\"");
}
}

@Nullable
@Override
public String getShortDescription() throws ParsingException {

if(videoInfo.has("detailedMetadataSnippets"))
return getTextFromObject(videoInfo.getArray("detailedMetadataSnippets").getObject(0).getObject("snippetText"));

if(videoInfo.has("descriptionSnippet"))
return getTextFromObject(videoInfo.getObject("descriptionSnippet"));
FireMasterK marked this conversation as resolved.
Show resolved Hide resolved

return null;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ public class StreamInfoItem extends InfoItem {
private final StreamType streamType;

private String uploaderName;
private String shortDescription;
private String textualUploadDate;
@Nullable
private DateWrapper uploadDate;
Expand Down Expand Up @@ -92,6 +93,14 @@ public void setUploaderAvatarUrl(final String uploaderAvatarUrl) {
this.uploaderAvatarUrl = uploaderAvatarUrl;
}

public String getShortDescription() {
return shortDescription;
}

public void setShortDescription(String shortDescription) {
FireMasterK marked this conversation as resolved.
Show resolved Hide resolved
this.shortDescription = shortDescription;
}

@Nullable
public String getTextualUploadDate() {
return textualUploadDate;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,4 +116,14 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor {
@Nullable
DateWrapper getUploadDate() throws ParsingException;


/**
* Get the video's short description.
*
* @return The video's short description or {@code null} if not provided by the service.
* @throws ParsingException if there is an error in the extraction
*/
@Nullable
String getShortDescription() throws ParsingException;
FireMasterK marked this conversation as resolved.
Show resolved Hide resolved

}
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,11 @@ public StreamInfoItem extract(StreamInfoItemExtractor extractor) throws ParsingE
} catch (Exception e) {
addError(e);
}
try {
resultItem.setShortDescription(extractor.getShortDescription());
} catch (Exception e) {
addError(e);
}

return resultItem;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -343,4 +343,34 @@ public void testUploaderAvatar() throws IOException, ExtractionException {
}
}
}

public static class VideoDescription extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "44wLAzydRFU";

@BeforeClass
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
YoutubeParsingHelper.setNumberGenerator(new Random(1));
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "video_description"));
extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
extractor.fetchPage();
}

@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; }
@Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }

@Test
public void testVideoDescription() throws IOException, ExtractionException {
final List<InfoItem> items = extractor.getInitialPage().getItems();
assertNotNull(((StreamInfoItem) items.get(0)).getShortDescription());
}
}
}
Loading