-
-
Notifications
You must be signed in to change notification settings - Fork 26
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* add support of yt playlists * fix playlist link and add yt lang param * exclude failed entries from keep part of processing * fix processed count
- Loading branch information
Showing
19 changed files
with
380 additions
and
237 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
2 changes: 1 addition & 1 deletion
2
app/youtube/channel/downloader.go → app/youtube/feed/downloader.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,4 @@ | ||
| package channel | ||
| package feed | ||
|
|
||
| import ( | ||
| "bytes" | ||
|
|
||
2 changes: 1 addition & 1 deletion
2
app/youtube/channel/downloader_test.go → app/youtube/feed/downloader_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,4 @@ | ||
| package channel | ||
| package feed | ||
|
|
||
| import ( | ||
| "bytes" | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| // Package feed provided parser and downloader for youtube feeds and entries. | ||
| package feed | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/xml" | ||
| "html/template" | ||
| "net/http" | ||
| "sort" | ||
| "time" | ||
|
|
||
| "github.com/pkg/errors" | ||
| ) | ||
|
|
||
| // Feed represents a YouTube channel feed. | ||
| type Feed struct { | ||
| Client *http.Client | ||
| ChannelBaseURL string | ||
| PlaylistBaseURL string | ||
| } | ||
|
|
||
| // Type represents the type of YouTube feed. | ||
| type Type string | ||
|
|
||
| // enum for the different YouTube feed types. | ||
| const ( | ||
| FTDefault = Type("") | ||
| FTChannel = Type("channel") | ||
| FTPlaylist = Type("playlist") | ||
| ) | ||
|
|
||
| // Get xml/rss feed for channel | ||
| // https://www.youtube.com/feeds/videos.xml?channel_id=UCPU28A9z_ka_R5dQfecHJlA | ||
| func (c *Feed) Get(ctx context.Context, id string, feedType Type) ([]Entry, error) { | ||
|
|
||
| feedURL, err := c.url(id, feedType) | ||
| if err != nil { | ||
| return nil, errors.Wrap(err, "failed to get feed url") | ||
| } | ||
|
|
||
| req, err := http.NewRequest("GET", feedURL, http.NoBody) | ||
| if err != nil { | ||
| return nil, errors.Wrapf(err, "failed to create request for %s", id) | ||
| } | ||
| resp, err := c.Client.Do(req.WithContext(ctx)) | ||
| if err != nil { | ||
| return nil, errors.Wrapf(err, "failed to get channel %s", id) | ||
| } | ||
| defer resp.Body.Close() | ||
| if resp.StatusCode != http.StatusOK { | ||
| return nil, errors.Errorf("failed to get %s: %s", id, resp.Status) | ||
| } | ||
| data := struct { | ||
| Entry []Entry `xml:"entry"` | ||
| }{} | ||
|
|
||
| if err := xml.NewDecoder(resp.Body).Decode(&data); err != nil { | ||
| return nil, errors.Wrapf(err, "failed to decode %s", id) | ||
| } | ||
|
|
||
| sort.Slice(data.Entry, func(i, j int) bool { | ||
| return data.Entry[i].Published.After(data.Entry[j].Published) | ||
| }) | ||
|
|
||
| // set channel or playlist id. Need to override this for RSS feed because yt always returns channel id here | ||
| for i := range data.Entry { | ||
| data.Entry[i].ChannelID = id | ||
| } | ||
|
|
||
| return data.Entry, nil | ||
| } | ||
|
|
||
| func (c *Feed) url(id string, feedType Type) (string, error) { | ||
| switch feedType { | ||
| case FTChannel, FTDefault: | ||
| return c.ChannelBaseURL + id, nil | ||
| case FTPlaylist: | ||
| return c.PlaylistBaseURL + id, nil | ||
| } | ||
| return "", errors.Errorf("unknown feed type %s", feedType) | ||
| } | ||
|
|
||
| // Entry represents a YouTube channel entry. | ||
| type Entry struct { | ||
| ChannelID string `xml:"http://www.youtube.com/xml/schemas/2015 channelId"` | ||
| VideoID string `xml:"http://www.youtube.com/xml/schemas/2015 videoId"` | ||
| Title string `xml:"title"` | ||
| Link struct { | ||
| Href string `xml:"href,attr"` | ||
| } `xml:"link"` | ||
| Published time.Time `xml:"published"` | ||
|
|
||
| Media struct { | ||
| Description template.HTML `xml:"description"` | ||
| Thumbnail struct { | ||
| URL string `xml:"url,attr"` | ||
| } `xml:"thumbnail"` | ||
| } `xml:"http://search.yahoo.com/mrss/ group"` | ||
|
|
||
| Author struct { | ||
| Name string `xml:"name"` | ||
| URI string `xml:"uri"` | ||
| } `xml:"author"` | ||
| File string | ||
| } | ||
|
|
||
| // UID returns the unique identifier of the entry. | ||
| func (e *Entry) UID() string { | ||
| return e.ChannelID + ":" + e.VideoID | ||
| } |
Oops, something went wrong.