safer RemoveOld #57

app/youtube/service.go

@@ -227,12 +227,9 @@ func (s *Service) procChannels(ctx context.Context) error {
 		allStats.processed += processed
 
 		if changed { // save rss feed to fs if there are new entries
-			removed, err := s.removeOld(feedInfo)
-			if err != nil {
-				log.Printf("[WARN] failed to remove old entries for %s: %v", feedInfo.Name, err)
-			} else {
-				allStats.removed += removed
-			}
+			removed := s.removeOld(feedInfo)
+			allStats.removed += removed
+
 			rss, rssErr := s.RSSFeed(feedInfo)
 			if rssErr != nil {
 				log.Printf("[WARN] failed to generate rss for %s: %s", feedInfo.Name, rssErr)
@@ -250,13 +247,14 @@ func (s *Service) procChannels(ctx context.Context) error {
 }
 
 // removeOld deletes old entries from store and corresponding files
-func (s *Service) removeOld(fi FeedInfo) (int, error) {
+func (s *Service) removeOld(fi FeedInfo) int {
 	removed := 0
 	keep := s.keep(fi)
 	files, err := s.Store.RemoveOld(fi.ID, keep+1)
-	if err != nil {
-		return 0, errors.Wrapf(err, "failed to remove old meta data for %s", fi.ID)
+	if err != nil { // even with error we get a list of files to remove
+		log.Printf("[WARN] failed to remove some old meta data for %s, %v", fi.ID, err)
 	}
+
 	for _, f := range files {
 		if e := os.Remove(f); e != nil {
 			log.Printf("[WARN] failed to remove file %s: %v", f, e)
@@ -265,7 +263,7 @@ func (s *Service) removeOld(fi FeedInfo) (int, error) {
 		removed++
 		log.Printf("[INFO] removed %s for %s (%s)", f, fi.ID, fi.Name)
 	}
-	return removed, nil
+	return removed
 }
 
 func (s *Service) keep(fi FeedInfo) int {

app/youtube/store/store.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	log "github.com/go-pkgz/lgr"
+	"github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
 	"github.com/umputun/feed-master/app/youtube/feed"
 	bolt "go.etcd.io/bbolt"
@@ -122,11 +123,13 @@ func (s *BoltDB) Channels() (result []string, err error) {
 
 // RemoveOld removes old entries from bolt and returns the list of removed entry.File
 // the caller should delete the files
+// important: this method returns the list of removed keys even if there was an error
 func (s *BoltDB) RemoveOld(channelID string, keep int) ([]string, error) {
 	deleted := 0
 	var res []string
 
 	err := s.DB.Update(func(tx *bolt.Tx) (e error) {
+		errs := new(multierror.Error)
 		bucket := tx.Bucket([]byte(channelID))
 		if bucket == nil {
 			return fmt.Errorf("no bucket for %s", channelID)
@@ -141,14 +144,17 @@ func (s *BoltDB) RemoveOld(channelID string, keep int) ([]string, error) {
 					log.Printf("[WARN] failed to unmarshal, %v", err)
 					continue
 				}
+				if err := bucket.Delete(k); err != nil {
+					errs = multierror.Append(errs, errors.Wrapf(err, "failed to delete %s (%s)", string(k), item.File))
+					continue
+				}
 				res = append(res, item.File)
-
-				e = bucket.Delete(k)
 				deleted++
 			}
 		}
-		return e
+		return errs.ErrorOrNil()
 	})
+
 	return res, err
 }