add support of yt playlists

_example/etc/fm-yt.yml

@@ -80,14 +80,15 @@ feeds:
     link: http://example.com
     language: "ru-ru"
     image: images/yt-example.png
-    ext_date: yyyyddmm
     sources:
+      - name: Точка
+        url: https://localhost:8080/yt/rss/PLZVQqcKxEn_6YaOniJmxATjODSVUbbMkd
       - name: Живой Гвоздь
-        url: https://feedmaster.umputun.com/yt/rss/UCWAIvx2yYLK_xTYD4F2mUNw
+        url: https://localhost:8080/yt/rss/UCWAIvx2yYLK_xTYD4F2mUNw
       - name: Alexandr Plushev
-        url: https://feedmaster.umputun.com/yt/rss/UCTVk323gzizpujtn2T_BL7w
+        url: https://localhost:8080/yt/rss/UCTVk323gzizpujtn2T_BL7w
       - name: Дилетант
-        url: https://feedmaster.umputun.com/yt/rss/UCuIE7-5QzeAR6EdZXwDRwuQ
+        url: https://localhost:8080/yt/rss/UCuIE7-5QzeAR6EdZXwDRwuQ
 
   echo-msk-filtered-example:
     title: Example with filtering from the feed
@@ -106,18 +107,20 @@ youtube:
   base_url: http://localhost:8080/yt/media
   dl_template: yt-dlp --extract-audio --audio-format=mp3 --audio-quality=0 -f m4a/bestaudio "https://www.youtube.com/watch?v={{.ID}}" --no-progress -o {{.FileName}}.tmp
   base_chan_url: "https://www.youtube.com/feeds/videos.xml?channel_id="
+  base_playlist_url: "https://www.youtube.com/feeds/videos.xml?playlist_id="
   update: 60s
-  max_per_channel: 5
+  max_per_channel: 3
   files_location: ./var/yt
   rss_location: ./var/rss
   channels:
-  - {id: UCWAIvx2yYLK_xTYD4F2mUNw, name: "Живой Гвоздь"}
-  - {id: UCTVk323gzizpujtn2T_BL7w, name: "Alexandr Plushev"}
-  - {id: UCuIE7-5QzeAR6EdZXwDRwuQ, name: "Дилетант"}
+  - {id: UCWAIvx2yYLK_xTYD4F2mUNw, name: "Живой Гвоздь", lang: "ru-ru"}
+  - {id: UCTVk323gzizpujtn2T_BL7w, name: "Alexandr Plushev", lang: "ru-ru"}
+  - {id: UCuIE7-5QzeAR6EdZXwDRwuQ, name: "Дилетант", type: "channel", lang: "ru-ru"}
+  - {id: PLZVQqcKxEn_6YaOniJmxATjODSVUbbMkd, name: "Точка", type: "playlist", lang: "ru-ru"}
 
 system:
   update: 1m
   max_per_feed: 10
   max_total: 50
   max_keep: 1000
-  base_url: https://feedmaster.umputun.com
+  base_url: http://localhost:8080

app/api/server.go

@@ -40,7 +40,7 @@ type Server struct {
 
 // YoutubeSvc provides access to youtube's audio rss
 type YoutubeSvc interface {
-	RSSFeed(cinfo youtube.ChannelInfo) (string, error)
+	RSSFeed(cinfo youtube.FeedInfo) (string, error)
 }
 
 // Run starts http server for API with all routes
@@ -214,7 +214,7 @@ func (s *Server) getListCtrl(w http.ResponseWriter, r *http.Request) {
 func (s *Server) getYoutubeFeedCtrl(w http.ResponseWriter, r *http.Request) {
 	channel := chi.URLParam(r, "channel")
 
-	res, err := s.YoutubeSvc.RSSFeed(youtube.ChannelInfo{ID: channel})
+	res, err := s.YoutubeSvc.RSSFeed(youtube.FeedInfo{ID: channel})
 	if err != nil {
 		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to read yt list")
 		return

app/main.go

@@ -17,13 +17,12 @@ import (
 	bolt "go.etcd.io/bbolt"
 	"gopkg.in/yaml.v2"
 
-	"github.com/umputun/feed-master/app/youtube"
-	"github.com/umputun/feed-master/app/youtube/channel"
-	"github.com/umputun/feed-master/app/youtube/store"
-
 	"github.com/umputun/feed-master/app/api"
-	"github.com/umputun/feed-master/app/feed"
+	rssfeed "github.com/umputun/feed-master/app/feed"
 	"github.com/umputun/feed-master/app/proc"
+	"github.com/umputun/feed-master/app/youtube"
+	ytfeed "github.com/umputun/feed-master/app/youtube/feed"
+	"github.com/umputun/feed-master/app/youtube/store"
 )
 
 type options struct {
@@ -91,10 +90,11 @@ func main() {
 		log.Printf("[INFO] starting youtube processor for %d channels", len(conf.YouTube.Channels))
 		outWr := log.ToWriter(log.Default(), "DEBUG")
 		errWr := log.ToWriter(log.Default(), "INFO")
-		dwnl := channel.NewDownloader(conf.YouTube.DlTemplate, outWr, errWr, opts.YtLocation)
-		fd := channel.Feed{Client: &http.Client{Timeout: 10 * time.Second}, BaseURL: conf.YouTube.BaseChanURL}
+		dwnl := ytfeed.NewDownloader(conf.YouTube.DlTemplate, outWr, errWr, opts.YtLocation)
+		fd := ytfeed.Feed{Client: &http.Client{Timeout: 10 * time.Second},
+			ChannelBaseURL: conf.YouTube.BaseChanURL, PlaylistBaseURL: conf.YouTube.BasePlaylistURL}
 		ytSvc = youtube.Service{
-			Channels:       conf.YouTube.Channels,
+			Feeds:          conf.YouTube.Channels,
 			Downloader:     dwnl,
 			ChannelService: &fd,
 			Store:          &store.BoltDB{DB: db},
@@ -155,7 +155,7 @@ func makeBoltDB(dbFile string) (*bolt.DB, error) {
 }
 
 func makeTwitter(opts options) *proc.TwitterClient {
-	twitterFmtFn := func(item feed.Item) string {
+	twitterFmtFn := func(item rssfeed.Item) string {
 		b1 := bytes.Buffer{}
 		if err := template.Must(template.New("twi").Parse(opts.TwitterTemplate)).Execute(&b1, item); err != nil { // nolint
 			// template failed to parse record, backup predefined format

app/main_test.go

@@ -44,10 +44,11 @@ system:
 youtube: 
   dl_template: yt-dlp --extract-audio --audio-format=mp3 --audio-quality=0 -f m4a/bestaudio "https://www.youtube.com/watch?v={{.ID}}" --no-progress -o {{.Filename}}.tmp
   base_chan_url: "https://www.youtube.com/feeds/videos.xml?channel_id="
+  base_playlist_url: "https://www.youtube.com/feeds/videos.xml?playlist_id="
   rss_location: ./var/rss
   channels:
-  - {id: id1, name: name1}
-  - {id: id2, name: name2}
+  - {id: id1, name: name1, type: playlist}
+  - {id: id2, name: name2, lang: ru-ru, type: channel}
 `)
 
 	assert.Nil(t, ioutil.WriteFile("/tmp/fm.yml", data, 0777), "failed write yml") // nolint
@@ -60,10 +61,12 @@ youtube:
 	assert.Equal(t, "https://bbb.com/u1", r.Feeds["second"].Sources[0].URL)
 	assert.Equal(t, "^filterme*", r.Feeds["filtered"].Filter.Title)
 	assert.Equal(t, time.Second*600, r.System.UpdateInterval)
-	assert.Equal(t, []youtube.ChannelInfo{{Name: "name1", ID: "id1"}, {Name: "name2", ID: "id2"}},
+	assert.Equal(t, []youtube.FeedInfo{{Name: "name1", ID: "id1", Type: "playlist"},
+		{Name: "name2", ID: "id2", Type: "channel", Language: "ru-ru"}},
 		r.YouTube.Channels, "2 yt")
 	assert.Equal(t, "yt-dlp --extract-audio --audio-format=mp3 --audio-quality=0 -f m4a/bestaudio \"https://www.youtube.com/watch?v={{.ID}}\" --no-progress -o {{.Filename}}.tmp", r.YouTube.DlTemplate)
 	assert.Equal(t, "https://www.youtube.com/feeds/videos.xml?channel_id=", r.YouTube.BaseChanURL)
+	assert.Equal(t, "https://www.youtube.com/feeds/videos.xml?playlist_id=", r.YouTube.BasePlaylistURL)
 	assert.Equal(t, "./var/rss", r.YouTube.RSSLocation)
 }
 

app/proc/processor.go

@@ -45,14 +45,15 @@ type Conf struct {
 	} `yaml:"system"`
 
 	YouTube struct {
-		DlTemplate     string                `yaml:"dl_template"`
-		BaseChanURL    string                `yaml:"base_chan_url"`
-		Channels       []youtube.ChannelInfo `yaml:"channels"`
-		BaseURL        string                `yaml:"base_url"`
-		UpdateInterval time.Duration         `yaml:"update"`
-		MaxItems       int                   `yaml:"max_per_channel"`
-		FilesLocation  string                `yaml:"files_location"`
-		RSSLocation    string                `yaml:"rss_location"`
+		DlTemplate      string             `yaml:"dl_template"`
+		BaseChanURL     string             `yaml:"base_chan_url"`
+		BasePlaylistURL string             `yaml:"base_playlist_url"`
+		Channels        []youtube.FeedInfo `yaml:"channels"`
+		BaseURL         string             `yaml:"base_url"`
+		UpdateInterval  time.Duration      `yaml:"update"`
+		MaxItems        int                `yaml:"max_per_channel"`
+		FilesLocation   string             `yaml:"files_location"`
+		RSSLocation     string             `yaml:"rss_location"`
 	} `yaml:"youtube"`
 }
 
@@ -83,7 +84,7 @@ type YTChannel struct {
 	Name string
 }
 
-// Do activates loop of goroutine for each feed, concurrency limited by p.Conf.Concurrent
+// Do activate loop of goroutine for each feed, concurrency limited by p.Conf.Concurrent
 func (p *Processor) Do() {
 	log.Printf("[INFO] activate processor, feeds=%d, %+v", len(p.Conf.Feeds), p.Conf)
 	p.setDefaults()

app/youtube/channel/downloader.go → app/youtube/feed/downloader.go

@@ -1,4 +1,4 @@
-package channel
+package feed
 
 import (
 	"bytes"

app/youtube/channel/downloader_test.go → app/youtube/feed/downloader_test.go

@@ -1,4 +1,4 @@
-package channel
+package feed
 
 import (
 	"bytes"

app/youtube/feed/feed.go

@@ -0,0 +1,110 @@
+// Package feed provided parser and downloader for youtube feeds and entries.
+package feed
+
+import (
+	"context"
+	"encoding/xml"
+	"html/template"
+	"net/http"
+	"sort"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+// Feed represents a YouTube channel feed.
+type Feed struct {
+	Client          *http.Client
+	ChannelBaseURL  string
+	PlaylistBaseURL string
+}
+
+// Type represents the type of YouTube feed.
+type Type string
+
+// enum for the different YouTube feed types.
+const (
+	FTDefault  = Type("")
+	FTChannel  = Type("channel")
+	FTPlaylist = Type("playlist")
+)
+
+// Get xml/rss feed for channel
+// https://www.youtube.com/feeds/videos.xml?channel_id=UCPU28A9z_ka_R5dQfecHJlA
+func (c *Feed) Get(ctx context.Context, id string, feedType Type) ([]Entry, error) {
+
+	feedURL, err := c.url(id, feedType)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get feed url")
+	}
+
+	req, err := http.NewRequest("GET", feedURL, http.NoBody)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to create request for %s", id)
+	}
+	resp, err := c.Client.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get channel %s", id)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.Errorf("failed to get %s: %s", id, resp.Status)
+	}
+	data := struct {
+		Entry []Entry `xml:"entry"`
+	}{}
+
+	if err := xml.NewDecoder(resp.Body).Decode(&data); err != nil {
+		return nil, errors.Wrapf(err, "failed to decode %s", id)
+	}
+
+	sort.Slice(data.Entry, func(i, j int) bool {
+		return data.Entry[i].Published.After(data.Entry[j].Published)
+	})
+
+	// set channel or playlist id. Need to override this for RSS feed because yt always returns channel id here
+	for i := range data.Entry {
+		data.Entry[i].ChannelID = id
+	}
+
+	return data.Entry, nil
+}
+
+func (c *Feed) url(id string, feedType Type) (string, error) {
+	switch feedType {
+	case FTChannel, FTDefault:
+		return c.ChannelBaseURL + id, nil
+	case FTPlaylist:
+		return c.PlaylistBaseURL + id, nil
+	}
+	return "", errors.Errorf("unknown feed type %s", feedType)
+}
+
+// Entry represents a YouTube channel entry.
+type Entry struct {
+	ChannelID string `xml:"http://www.youtube.com/xml/schemas/2015 channelId"`
+	VideoID   string `xml:"http://www.youtube.com/xml/schemas/2015 videoId"`
+	Title     string `xml:"title"`
+	Link      struct {
+		Href string `xml:"href,attr"`
+	} `xml:"link"`
+	Published time.Time `xml:"published"`
+
+	Media struct {
+		Description template.HTML `xml:"description"`
+		Thumbnail   struct {
+			URL string `xml:"url,attr"`
+		} `xml:"thumbnail"`
+	} `xml:"http://search.yahoo.com/mrss/ group"`
+
+	Author struct {
+		Name string `xml:"name"`
+		URI  string `xml:"uri"`
+	} `xml:"author"`
+	File string
+}
+
+// UID returns the unique identifier of the entry.
+func (e *Entry) UID() string {
+	return e.ChannelID + ":" + e.VideoID
+}