diff --git a/README.md b/README.md index 4368f34..1409760 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ youtube: # youtube configuration, optional channels: # list of youtube channels to download and process # id: channel or playlist id, name: channel or playlist name, type: "channel" or "playlist", # lang: language of the channel, keep: override default keep value - # filter: criteria to include and exclude videos + # filter: criteria to include and exclude videos, can be regex - {id: UCWAIvx2yYLK_xTYD4F2mUNw, name: "Живой Гвоздь", lang: "ru-ru"} - {id: UCuIE7-5QzeAR6EdZXwDRwuQ, name: "Дилетант", type: "channel", lang: "ru-ru", "keep": 10} - {id: PLZVQqcKxEn_6YaOniJmxATjODSVUbbMkd, name: "Точка", type: "playlist", lang: "ru-ru", filter: {include: "ТОЧКА", exclude: "STAR'цы Live"}} diff --git a/app/youtube/service.go b/app/youtube/service.go index 83d5778..6cbcd35 100644 --- a/app/youtube/service.go +++ b/app/youtube/service.go @@ -219,11 +219,11 @@ func (s *Service) procChannels(ctx context.Context) error { break } - isRelevant, err := s.isRelevant(entry, feedInfo) + isAllowed, err := s.isAllowed(entry, feedInfo) if err != nil { return errors.Wrapf(err, "failed to check if entry %s is relevant", entry.VideoID) } - if !isRelevant { + if !isAllowed { allStats.ignored++ continue } @@ -363,8 +363,8 @@ func (s *Service) isNew(entry ytfeed.Entry, fi FeedInfo) (ok bool, err error) { return true, nil } -// isRelevant checks if entry matches all filters for the channel feed -func (s *Service) isRelevant(entry ytfeed.Entry, fi FeedInfo) (ok bool, err error) { +// isAllowed checks if entry matches all filters for the channel feed +func (s *Service) isAllowed(entry ytfeed.Entry, fi FeedInfo) (ok bool, err error) { matchedIncludeFilter := true if fi.Filter.Include != "" { matchedIncludeFilter, err = regexp.MatchString(fi.Filter.Include, entry.Title) diff --git a/app/youtube/service_test.go b/app/youtube/service_test.go index 5bbc52c..f2c949e 100644 --- a/app/youtube/service_test.go +++ b/app/youtube/service_test.go @@ -112,7 +112,7 @@ func TestService_Do(t *testing.T) { } // nolint:dupl // test if very similar to TestService_RSSFeed -func TestService_DoIsRelevantFilter(t *testing.T) { +func TestService_DoIsAllowedFilter(t *testing.T) { chans := &mocks.ChannelServiceMock{ GetFunc: func(ctx context.Context, chanID string, feedType ytfeed.Type) ([]ytfeed.Entry, error) { @@ -144,6 +144,7 @@ func TestService_DoIsRelevantFilter(t *testing.T) { svc := Service{ Feeds: []FeedInfo{ {ID: "channel1", Name: "name1", Type: ytfeed.FTChannel, Filter: FeedFilter{Include: "Prefix2", Exclude: "title3"}}, + {ID: "channel2", Name: "name2", Type: ytfeed.FTChannel, Filter: FeedFilter{Include: "^\\w{7}:", Exclude: "\\w+3$"}}, }, Downloader: downloader, ChannelService: chans, @@ -160,18 +161,29 @@ func TestService_DoIsRelevantFilter(t *testing.T) { err = svc.Do(ctx) assert.EqualError(t, err, "context deadline exceeded") - require.Equal(t, 2, len(chans.GetCalls())) + require.Equal(t, 4, len(chans.GetCalls())) assert.Equal(t, "channel1", chans.GetCalls()[0].ChanID) assert.Equal(t, ytfeed.FTChannel, chans.GetCalls()[0].FeedType) - assert.Equal(t, "channel1", chans.GetCalls()[1].ChanID) + assert.Equal(t, "channel2", chans.GetCalls()[1].ChanID) + assert.Equal(t, ytfeed.FTChannel, chans.GetCalls()[1].FeedType) + assert.Equal(t, "channel1", chans.GetCalls()[2].ChanID) + assert.Equal(t, "channel2", chans.GetCalls()[3].ChanID) res, err := boltStore.Load("channel1", 10) require.NoError(t, err) assert.Equal(t, 1, len(res), "one entry for channel1, skipped irrelevant ones") assert.Equal(t, "vid2", res[0].VideoID) - require.Equal(t, 1, len(downloader.GetCalls())) + res, err = boltStore.Load("channel2", 10) + require.NoError(t, err) + assert.Equal(t, 2, len(res), "two entries for channel2, skipped irrelevant one") + assert.Equal(t, "vid2", res[0].VideoID) + assert.Equal(t, "vid1", res[1].VideoID) + + require.Equal(t, 3, len(downloader.GetCalls())) require.Equal(t, "vid2", downloader.GetCalls()[0].ID) + require.Equal(t, "vid1", downloader.GetCalls()[1].ID) + require.Equal(t, "vid2", downloader.GetCalls()[2].ID) require.True(t, downloader.GetCalls()[0].Fname != "") rssData, err := os.ReadFile("/tmp/channel1.xml") @@ -180,8 +192,17 @@ func TestService_DoIsRelevantFilter(t *testing.T) { assert.Contains(t, string(rssData), "channel1::vid2") assert.Contains(t, string(rssData), "1234") - require.Equal(t, 1, len(duration.FileCalls())) + rssData, err = os.ReadFile("/tmp/channel2.xml") + require.NoError(t, err) + t.Logf("%s", string(rssData)) + assert.Contains(t, string(rssData), "channel2::vid2") + assert.Contains(t, string(rssData), "channel2::vid1") + assert.Contains(t, string(rssData), "1234") + + require.Equal(t, 3, len(duration.FileCalls())) assert.Equal(t, "/tmp/4308c33c7ddb107c2d0c13a905e4c6962001bab4.mp3", duration.FileCalls()[0].Fname) + assert.Equal(t, "/tmp/3be877c750abb87daee80c005fe87e7a3f824fed.mp3", duration.FileCalls()[1].Fname) + assert.Equal(t, "/tmp/648f79b3a05ececb8a37600aa0aee332f0374e01.mp3", duration.FileCalls()[2].Fname) } // nolint:dupl // test if very similar to TestService_RSSFeed