Add:Podcast quickmatch attempts quick matching unmatched episodes #983

server/controllers/PodcastController.js

@@ -1,10 +1,9 @@
 const axios = require('axios')
 const fs = require('../libs/fsExtra')
-const Path = require('path')
 const Logger = require('../Logger')
-const { parsePodcastRssFeedXml } = require('../utils/podcastUtils')
+const { getPodcastFeed, findMatchingEpisodes } = require('../utils/podcastUtils')
 const LibraryItem = require('../objects/LibraryItem')
-const { getFileTimestampsWithIno, sanitizeFilename } = require('../utils/fileUtils')
+const { getFileTimestampsWithIno } = require('../utils/fileUtils')
 const filePerms = require('../utils/filePerms')
 
 class PodcastController {
@@ -91,32 +90,17 @@ class PodcastController {
  }
  }
 
- getPodcastFeed(req, res) {
+ async getPodcastFeed(req, res) {
  var url = req.body.rssFeed
  if (!url) {
  return res.status(400).send('Bad request')
  }
- var includeRaw = req.query.raw == 1 // Include raw json
 
- axios.get(url).then(async (data) => {
- if (!data || !data.data) {
- Logger.error('Invalid podcast feed request response')
- return res.status(500).send('Bad response from feed request')
- }
- Logger.debug(`[PodcastController] Podcast feed size ${(data.data.length / 1024 / 1024).toFixed(2)}MB`)
- var payload = await parsePodcastRssFeedXml(data.data, false, includeRaw)
- if (!payload) {
- return res.status(500).send('Invalid podcast RSS feed')
- }
-
- // RSS feed may be a private RSS feed
- payload.podcast.metadata.feedUrl = url
-
- res.json(payload)
- }).catch((error) => {
- console.error('Failed', error)
- res.status(500).send(error)
- })
+ const podcast = await getPodcastFeed(url)
+ if (!podcast) {
+ return res.status(404).send('Podcast RSS feed request failed or invalid response data')
+ }
+ res.json({ podcast })
  }
 
  async getOPMLFeeds(req, res) {
@@ -177,9 +161,7 @@ class PodcastController {
  if (!searchTitle) {
  return res.sendStatus(500)
  }
- searchTitle = searchTitle.toLowerCase().trim()
-
- const episodes = await this.podcastManager.findEpisode(rssFeedUrl, searchTitle)
+ const episodes = await findMatchingEpisodes(rssFeedUrl, searchTitle)
  res.json({
  episodes: episodes || []
  })

server/managers/PodcastManager.js

@@ -1,7 +1,6 @@
 const fs = require('../libs/fsExtra')
-const axios = require('axios')
 
-const { parsePodcastRssFeedXml } = require('../utils/podcastUtils')
+const { getPodcastFeed } = require('../utils/podcastUtils')
 const Logger = require('../Logger')
 
 const { downloadFile, removeFile } = require('../utils/fileUtils')
@@ -226,7 +225,7 @@ class PodcastManager {
  Logger.error(`[PodcastManager] checkPodcastForNewEpisodes no feed url for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id})`)
  return false
  }
- var feed = await this.getPodcastFeed(podcastLibraryItem.media.metadata.feedUrl)
+ var feed = await getPodcastFeed(podcastLibraryItem.media.metadata.feedUrl)
  if (!feed || !feed.episodes) {
  Logger.error(`[PodcastManager] checkPodcastForNewEpisodes invalid feed payload for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id})`, feed)
  return false
@@ -262,7 +261,7 @@ class PodcastManager {
  }
 
  async findEpisode(rssFeedUrl, searchTitle) {
- const feed = await this.getPodcastFeed(rssFeedUrl).catch(() => {
+ const feed = await getPodcastFeed(rssFeedUrl).catch(() => {
  return null
  })
  if (!feed || !feed.episodes) {
@@ -292,25 +291,6 @@ class PodcastManager {
  return matches.sort((a, b) => a.levenshtein - b.levenshtein)
  }
 
- getPodcastFeed(feedUrl, excludeEpisodeMetadata = false) {
- Logger.debug(`[PodcastManager] getPodcastFeed for "${feedUrl}"`)
- return axios.get(feedUrl, { timeout: 5000 }).then(async (data) => {
- if (!data || !data.data) {
- Logger.error('Invalid podcast feed request response')
- return false
- }
- Logger.debug(`[PodcastManager] getPodcastFeed for "${feedUrl}" success - parsing xml`)
- var payload = await parsePodcastRssFeedXml(data.data, excludeEpisodeMetadata)
- if (!payload) {
- return false
- }
- return payload.podcast
- }).catch((error) => {
- Logger.error('[PodcastManager] getPodcastFeed Error', error)
- return false
- })
- }
-
  async getOPMLFeeds(opmlText) {
  var extractedFeeds = opmlParser.parse(opmlText)
  if (!extractedFeeds || !extractedFeeds.length) {
@@ -323,7 +303,7 @@ class PodcastManager {
  var rssFeedData = []
 
  for (let feed of extractedFeeds) {
- var feedData = await this.getPodcastFeed(feed.feedUrl, true)
+ var feedData = await getPodcastFeed(feed.feedUrl, true)
  if (feedData) {
  feedData.metadata.feedUrl = feed.feedUrl
  rssFeedData.push(feedData)

server/objects/entities/PodcastEpisode.js

@@ -107,6 +107,9 @@ class PodcastEpisode {
  if (this.episode) return `${this.episode} - ${this.title}`
  return this.title
  }
+ get enclosureUrl() {
+ return this.enclosure ? this.enclosure.url : null
+ }
 
  setData(data, index = 1) {
  this.id = getId('ep')

server/scanner/Scanner.js

@@ -7,6 +7,7 @@ const { groupFilesIntoLibraryItemPaths, getLibraryItemFileData, scanFolder } = r
 const { comparePaths } = require('../utils/index')
 const { getIno } = require('../utils/fileUtils')
 const { ScanResult, LogLevel } = require('../utils/constants')
+const { findMatchingEpisodesInFeed, getPodcastFeed } = require('../utils/podcastUtils')
 
 const MediaFileScanner = require('./MediaFileScanner')
 const BookFinder = require('../finders/BookFinder')
@@ -684,7 +685,7 @@ class Scanner {
  var updatePayload = {}
  var hasUpdated = false
 
- if (libraryItem.mediaType === 'book') {
+ if (libraryItem.isBook) {
  var searchISBN = options.isbn || libraryItem.media.metadata.isbn
  var searchASIN = options.asin || libraryItem.media.metadata.asin
 
@@ -708,7 +709,7 @@ class Scanner {
  }
 
  updatePayload = await this.quickMatchBookBuildUpdatePayload(libraryItem, matchData, options)
- } else { // Podcast quick match
+ } else if (libraryItem.isPodcast) { // Podcast quick match
  var results = await this.podcastFinder.search(searchTitle)
  if (!results.length) {
  return {
@@ -739,6 +740,10 @@ class Scanner {
  }
 
  if (hasUpdated) {
+ if (libraryItem.isPodcast && libraryItem.media.metadata.feedUrl) { // Quick match all unmatched podcast episodes
+ await this.quickMatchPodcastEpisodes(libraryItem, options)
+ }
+
  await this.db.updateLibraryItem(libraryItem)
  this.emitter('item_updated', libraryItem.toJSONExpanded())
  }
@@ -762,16 +767,18 @@ class Scanner {
  itunesArtistId: matchData.artistId || null,
  releaseDate: matchData.releaseDate || null,
  imageUrl: matchData.cover || null,
+ feedUrl: matchData.feedUrl || null,
  description: matchData.descriptionPlain || null
  }
 
  for (const key in matchDataTransformed) {
  if (matchDataTransformed[key]) {
  if (key === 'genres') {
  if ((!libraryItem.media.metadata.genres || options.overrideDetails)) {
+ // TODO: Genres array or string?
  updatePayload.metadata[key] = matchDataTransformed[key].split(',').map(v => v.trim()).filter(v => !!v)
  }
- } else if (!libraryItem.media.metadata[key] || options.overrideDetails) {
+ } else if (libraryItem.media.metadata[key] !== matchDataTransformed[key] && (!libraryItem.media.metadata[key] || options.overrideDetails)) {
  updatePayload.metadata[key] = matchDataTransformed[key]
  }
  }
@@ -854,6 +861,61 @@ class Scanner {
  return updatePayload
  }
 
+ async quickMatchPodcastEpisodes(libraryItem, options = {}) {
+ const episodesToQuickMatch = libraryItem.media.episodes.filter(ep => !ep.enclosureUrl) // Only quick match episodes without enclosure
+ if (!episodesToQuickMatch.length) return false
+
+ const feed = await getPodcastFeed(libraryItem.media.metadata.feedUrl)
+ if (!feed) {
+ Logger.error(`[Scanner] quickMatchPodcastEpisodes: Unable to quick match episodes feed not found for "${libraryItem.media.metadata.feedUrl}"`)
+ return false
+ }
+
+ var episodesWereUpdated = false
+ for (const episode of episodesToQuickMatch) {
+ const episodeMatches = findMatchingEpisodesInFeed(feed, episode.title)
+ if (episodeMatches && episodeMatches.length) {
+ const wasUpdated = this.updateEpisodeWithMatch(libraryItem, episode, episodeMatches[0].episode, options)
+ if (wasUpdated) episodesWereUpdated = true
+ }
+ }
+ return episodesWereUpdated
+ }
+
+ updateEpisodeWithMatch(libraryItem, episode, episodeToMatch, options = {}) {
+ Logger.debug(`[Scanner] quickMatchPodcastEpisodes: Found episode match for "${episode.title}" => ${episodeToMatch.title}`)
+ const matchDataTransformed = {
+ title: episodeToMatch.title || '',
+ subtitle: episodeToMatch.subtitle || '',
+ description: episodeToMatch.description || '',
+ enclosure: episodeToMatch.enclosure || null,
+ episode: episodeToMatch.episode || '',
+ episodeType: episodeToMatch.episodeType || '',
+ season: episodeToMatch.season || '',
+ pubDate: episodeToMatch.pubDate || '',
+ publishedAt: episodeToMatch.publishedAt
+ }
+ const updatePayload = {}
+ for (const key in matchDataTransformed) {
+ if (matchDataTransformed[key]) {
+ if (key === 'enclosure') {
+ if (!episode.enclosure || JSON.stringify(episode.enclosure) !== JSON.stringify(matchDataTransformed.enclosure)) {
+ updatePayload[key] = {
+ ...matchDataTransformed.enclosure
+ }
+ }
+ } else if (episode[key] !== matchDataTransformed[key] && (!episode[key] || options.overrideDetails)) {
+ updatePayload[key] = matchDataTransformed[key]
+ }
+ }
+ }
+
+ if (Object.keys(updatePayload).length) {
+ return libraryItem.media.updateEpisode(episode.id, updatePayload)
+ }
+ return false
+ }
+
  async matchLibraryItems(library) {
  if (library.mediaType === 'podcast') {
  Logger.error(`[Scanner] matchLibraryItems: Match all not supported for podcasts yet`)

server/utils/podcastUtils.js

@@ -1,5 +1,6 @@
 const Logger = require('../Logger')
-const { xmlToJSON } = require('./index')
+const axios = require('axios')
+const { xmlToJSON, levenshteinDistance } = require('./index')
 const htmlSanitizer = require('../utils/htmlSanitizer')
 
 function extractFirstArrayItem(json, key) {
@@ -173,4 +174,65 @@ module.exports.parsePodcastRssFeedXml = async (xml, excludeEpisodeMetadata = fal
  podcast
  }
  }
+}
+
+module.exports.getPodcastFeed = (feedUrl, excludeEpisodeMetadata = false) => {
+ Logger.debug(`[podcastUtils] getPodcastFeed for "${feedUrl}"`)
+ return axios.get(feedUrl, { timeout: 6000 }).then(async (data) => {
+ if (!data || !data.data) {
+ Logger.error(`[podcastUtils] getPodcastFeed: Invalid podcast feed request response (${feedUrl})`)
+ return false
+ }
+ Logger.debug(`[podcastUtils] getPodcastFeed for "${feedUrl}" success - parsing xml`)
+ var payload = await this.parsePodcastRssFeedXml(data.data, excludeEpisodeMetadata)
+ if (!payload) {
+ return false
+ }
+
+ // RSS feed may be a private RSS feed
+ payload.podcast.metadata.feedUrl = feedUrl
+
+ return payload.podcast
+ }).catch((error) => {
+ Logger.error('[podcastUtils] getPodcastFeed Error', error)
+ return false
+ })
+}
+
+// Return array of episodes ordered by closest match (Levenshtein distance of 6 or less)
+module.exports.findMatchingEpisodes = async (feedUrl, searchTitle) => {
+ const feed = await this.getPodcastFeed(feedUrl).catch(() => {
+ return null
+ })
+
+ return this.findMatchingEpisodesInFeed(feed, searchTitle)
+}
+
+module.exports.findMatchingEpisodesInFeed = (feed, searchTitle) => {
+ searchTitle = searchTitle.toLowerCase().trim()
+ if (!feed || !feed.episodes) {
+ return null
+ }
+
+ const matches = []
+ feed.episodes.forEach(ep => {
+ if (!ep.title) return
+
+ const epTitle = ep.title.toLowerCase().trim()
+ if (epTitle === searchTitle) {
+ matches.push({
+ episode: ep,
+ levenshtein: 0
+ })
+ } else {
+ const levenshtein = levenshteinDistance(searchTitle, epTitle, true)
+ if (levenshtein <= 6 && epTitle.length > levenshtein) {
+ matches.push({
+ episode: ep,
+ levenshtein
+ })
+ }
+ }
+ })
+ return matches.sort((a, b) => a.levenshtein - b.levenshtein)
 }