Add duration to yt-based items (#69)

* add `<itunes:duration>` in seconds to yt entries and the final rss

* lint: false-positive inclusion warn

app/api/server.go

@@ -143,6 +143,7 @@ func (s *Server) getFeedCtrl(w http.ResponseWriter, r *http.Request) {
 		Link:          s.Conf.Feeds[feedName].Link,
 		PubDate:       items[0].PubDate,
 		LastBuildDate: time.Now().Format(time.RFC822Z),
+		NsItunes:      "http://www.itunes.com/dtds/podcast-1.0.dtd",
 	}
 
 	// replace link to UI page
@@ -212,7 +213,15 @@ func (s *Server) getListCtrl(w http.ResponseWriter, r *http.Request) {
 func (s *Server) getYoutubeFeedCtrl(w http.ResponseWriter, r *http.Request) {
 	channel := chi.URLParam(r, "channel")
 
-	res, err := s.YoutubeSvc.RSSFeed(youtube.FeedInfo{ID: channel})
+	fi := youtube.FeedInfo{ID: channel}
+	for _, f := range s.Conf.YouTube.Channels {
+		if f.ID == channel {
+			fi = f
+			break
+		}
+	}
+
+	res, err := s.YoutubeSvc.RSSFeed(fi)
 	if err != nil {
 		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to read yt list")
 		return

app/feed/item.go

@@ -21,7 +21,7 @@ type Item struct {
 	Enclosure Enclosure     `xml:"enclosure"`
 	GUID      string        `xml:"guid"`
 	Author    string        `xml:"author,omitempty"`
-
+	Duration  string        `xml:"itunes:duration,omitempty"`
 	// Internal
 	DT   time.Time `xml:"-"`
 	Junk bool      `xml:"-"`

app/feed/parser.go

@@ -19,6 +19,7 @@ import (
 // Rss2 feed
 type Rss2 struct {
 	XMLName       xml.Name `xml:"rss"`
+	NsItunes      string   `xml:"xmlns:itunes,attr"`
 	Version       string   `xml:"version,attr"`
 	Title         string   `xml:"channel>title"`
 	Language      string   `xml:"channel>language"`
@@ -94,7 +95,6 @@ func Parse(uri string) (result Rss2, err error) {
 	if err != nil {
 		return result, errors.Wrapf(err, "failed to read body, url: %s", uri)
 	}
-
 	result, err = parseFeedContent(body)
 	if err != nil {
 		return Rss2{}, errors.Wrap(err, "parsing error")
@@ -152,6 +152,7 @@ func parseFeedContent(content []byte) (Rss2, error) {
 				v.ItemList[i].Description = v.ItemList[i].Content
 			}
 		}
+		v.NsItunes = "http://www.itunes.com/dtds/podcast-1.0.dtd"
 		return v, nil
 	}
 

app/youtube/feed/feed.go

@@ -103,7 +103,9 @@ type Entry struct {
 		Name string `xml:"name"`
 		URI  string `xml:"uri"`
 	} `xml:"author"`
-	File string
+
+	File     string
+	Duration int // seconds
 }
 
 // UID returns the unique identifier of the entry.
@@ -112,8 +114,10 @@ func (e *Entry) UID() string {
 }
 
 func (e *Entry) String() string {
-	return fmt.Sprintf("{ChannelID:%s, VideoID:%s, Title:%q, Published:%s, Updated:%s, Author:%s, File:%s}",
-		e.ChannelID, e.VideoID, e.Title, e.Published.Format(time.RFC3339), e.Updated.Format(time.RFC3339),
-		e.Author.Name, e.File,
+	tz, _ := time.LoadLocation("Local")
+
+	return fmt.Sprintf("{ChannelID:%s, VideoID:%s, Title:%q, Published:%s, Updated:%s, Author:%s, File:%s, Duration:%ds}",
+		e.ChannelID, e.VideoID, e.Title, e.Published.In(tz).Format(time.RFC3339), e.Updated.In(tz).Format(time.RFC3339),
+		e.Author.Name, e.File, e.Duration,
 	)
 }

app/youtube/service.go

@@ -6,6 +6,7 @@ import (
 	"crypto/sha1"
 	"encoding/xml"
 	"fmt"
+	"io"
 	"os"
 	"path"
 	"strings"
@@ -14,6 +15,7 @@ import (
 	log "github.com/go-pkgz/lgr"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
+	"github.com/tcolgate/mp3"
 
 	rssfeed "github.com/umputun/feed-master/app/feed"
 	ytfeed "github.com/umputun/feed-master/app/youtube/feed"
@@ -116,6 +118,11 @@ func (s *Service) RSSFeed(fi FeedInfo) (string, error) {
 			fileSize = int(fileInfo.Size())
 		}
 
+		duration := ""
+		if entry.Duration > 0 {
+			duration = fmt.Sprintf("%d", entry.Duration)
+		}
+
 		items = append(items, rssfeed.Item{
 			Title:       entry.Title,
 			Description: entry.Media.Description,
@@ -128,12 +135,14 @@ func (s *Service) RSSFeed(fi FeedInfo) (string, error) {
 				Type:   "audio/mpeg",
 				Length: fileSize,
 			},
-			DT: time.Now(),
+			Duration: duration,
+			DT:       time.Now(),
 		})
 	}
 
 	rss := rssfeed.Rss2{
 		Version:       "2.0",
+		NsItunes:      "http://www.itunes.com/dtds/podcast-1.0.dtd",
 		ItemList:      items,
 		Title:         fi.Name,
 		Description:   "generated by feed-master",
@@ -290,6 +299,9 @@ func (s *Service) update(entry ytfeed.Entry, file string, fi FeedInfo) ytfeed.En
 	if !strings.Contains(entry.Title, fi.Name) { // if title doesn't contains channel name add it
 		entry.Title = fi.Name + ": " + entry.Title
 	}
+
+	entry.Duration = s.duration(file)
+
 	return entry
 }
 
@@ -329,6 +341,31 @@ func (s *Service) makeFileName(entry ytfeed.Entry) string {
 	return fmt.Sprintf("%x", h.Sum(nil))
 }
 
+// duration scans MP3 file from provided file and returns its duration in seconds, ignoring possible errors
+func (s *Service) duration(fname string) int {
+	fh, err := os.Open(fname) //nolint:gosec // this is not an inclusion as file was created by us
+	if err != nil {
+		log.Printf("[WARN] can't get duration, failed to open file %s: %v", fname, err)
+		return 0
+	}
+	defer fh.Close() //nolint
+
+	d := mp3.NewDecoder(fh)
+	var f mp3.Frame
+	var skipped int
+	var duration float64
+
+	for err == nil {
+		if err = d.Decode(&f, &skipped); err != nil && err != io.EOF {
+			log.Printf("[WARN] can't decode mp3 file %s: %v", fname, err)
+			return 0
+		}
+		duration += f.Duration().Seconds()
+	}
+
+	return int(duration)
+}
+
 type stats struct {
 	entries   int
 	processed int

app/youtube/store/store.go

@@ -47,8 +47,7 @@ func (s *BoltDB) Save(entry feed.Entry) (bool, error) {
 			return errors.Wrapf(jerr, "marshal entry %s", entry.VideoID)
 		}
 
-		log.Printf("[INFO] save %s - {ChannelID:%s, VideoID:%s, Title:%s, File:%s, Author:%s, Published:%s}",
-			string(key), entry.ChannelID, entry.VideoID, entry.Title, entry.File, entry.Author.Name, entry.Published)
+		log.Printf("[INFO] save %s - %s", string(key), entry.String())
 
 		e = bucket.Put(key, jdata)
 		if e != nil {
@@ -188,8 +187,7 @@ func (s *BoltDB) SetProcessed(entry feed.Entry) error {
 			return nil
 		}
 
-		log.Printf("[INFO] set processed %s - {ChannelID:%s, VideoID:%s, Title:%s, File:%s, Author:%s, Published:%s}",
-			string(key), entry.ChannelID, entry.VideoID, entry.Title, entry.File, entry.Author.Name, entry.Published)
+		log.Printf("[INFO] set processed %s - %s", string(key), entry.String())
 
 		e = bucket.Put(key, []byte(entry.Published.Format(time.RFC3339)))
 		if e != nil {

yt-test.yml

@@ -0,0 +1,36 @@
+feeds:
+  yt-example:
+    title: Some cool channels
+    description: an example of youtube-based podcas
+    link: http://example.com
+    language: "ru-ru"
+    image: images/yt-example.png
+    sources:
+      - name: Точка
+        url: http://localhost:8080/yt/rss/PLZVQqcKxEn_6YaOniJmxATjODSVUbbMkd
+      - name: Живой Гвоздь
+        url: http://localhost:8080/yt/rss/UCWAIvx2yYLK_xTYD4F2mUNw
+      - name: Дилетант
+        url: http://localhost:8080/yt/rss/UCuIE7-5QzeAR6EdZXwDRwuQ
+
+
+youtube:
+  base_url: http://localhost:8080/yt/media
+  dl_template: yt-dlp --extract-audio --audio-format=mp3 --audio-quality=0 -f m4a/bestaudio "https://www.youtube.com/watch?v={{.ID}}" --no-progress -o {{.FileName}}.tmp
+  base_chan_url: "https://www.youtube.com/feeds/videos.xml?channel_id="
+  base_playlist_url: "https://www.youtube.com/feeds/videos.xml?playlist_id="
+  update: 60s
+  max_per_channel: 3
+  files_location: ./var/yt
+  rss_location: ./var/rss
+  channels:
+  - {id: UCWAIvx2yYLK_xTYD4F2mUNw, name: "Живой Гвоздь", lang: "ru-ru"}
+  - {id: UCuIE7-5QzeAR6EdZXwDRwuQ, name: "Дилетант", type: "channel", lang: "ru-ru"}
+  - {id: PLZVQqcKxEn_6YaOniJmxATjODSVUbbMkd, name: "Точка", type: "playlist", lang: "ru-ru"}
+
+system:
+  update: 1m
+  max_per_feed: 10
+  max_total: 50
+  max_keep: 1000
+  base_url: http://localhost:8080