Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve multi-episode and season snatches. Fixes #229. Fixes #4750 #4675

Merged
merged 17 commits into from
Aug 1, 2018
Merged

Improve multi-episode and season snatches. Fixes #229. Fixes #4750 #4675

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
2244b5b
Rename Season search mode to Backlog search mode
medariox Jul 13, 2018
fe095c5
Improve multi-episode and season snatches. Fixes #229
medariox Jul 15, 2018
18798ef
Merge branch 'develop' into feature/imp-multi-snatches
medariox Jul 15, 2018
6b21f58
Remove unused import
medariox Jul 15, 2018
2567c84
Add support for multi-episodes cache results, remove naive quality co…
medariox Jul 17, 2018
82e81f9
Add TODO
medariox Jul 17, 2018
85bbf62
Merge branch 'develop' into feature/imp-multi-snatches
medariox Jul 17, 2018
474beb6
Better formatting
medariox Jul 18, 2018
aaeb220
Merge branch 'develop' into feature/imp-multi-snatches
medariox Jul 19, 2018
42c748d
Remove support to download seasons for multi-episodes (torrents only)
medariox Jul 19, 2018
042f687
Use plural for better representation
medariox Jul 24, 2018
13806b2
Merge branch 'develop' into feature/imp-multi-snatches
medariox Jul 27, 2018
b79097e
Review
medariox Jul 29, 2018
5d7bffa
Merge branch 'develop' into feature/imp-multi-snatches
medariox Jul 29, 2018
e1c220c
Update CHANGELOG.md
medariox Jul 29, 2018
c9b763a
Merge remote-tracking branch 'remotes/origin/develop' into feature/im…
p0psicles Aug 1, 2018
43a1dda
Moved v0.2.9 changelog items to 'Unreleased'
p0psicles Aug 1, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 14 additions & 18 deletions medusa/providers/generic_provider.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,21 +247,24 @@ def find_search_results(self, series, episodes, search_mode, forced_search=False

results = {}
items_list = []
season_search = (
len(episodes) > 1 or manual_search_type == 'season'
) and search_mode == 'sponly'
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The line breaks don't help readability in this case, IMO.


for episode in episodes:
if not manual_search:
cache_result = self.cache.search_cache(episode, forced_search=forced_search,
down_cur_quality=download_current_quality)
cache_result = self.cache.find_needed_episodes(
episode, forced_search=forced_search, down_cur_quality=download_current_quality
)
if cache_result:
if episode.episode not in results:
results[episode.episode] = cache_result
else:
results[episode.episode].extend(cache_result)

for episode_no in cache_result:
if episode_no not in results:
results[episode_no] = cache_result[episode_no]
else:
results[episode_no] += cache_result[episode_no]
continue

search_strings = []
season_search = (len(episodes) > 1 or manual_search_type == 'season') and search_mode == 'sponly'
if season_search:
search_strings = self._get_season_search_strings(episode)
elif search_mode == 'eponly':
Expand All @@ -273,13 +276,11 @@ def find_search_results(self, series, episodes, search_mode, forced_search=False
search_string, ep_obj=episode, manual_search=manual_search
)

# In season search, we can't loop in episodes lists as we only need one episode to get the season string
# In season search, we can't loop in episodes lists as we
# only need one episode to get the season string
if search_mode == 'sponly':
break

if len(results) == len(episodes):
return results

# Remove duplicate items
unique_items = self.remove_duplicate_mappings(items_list)
log.debug('Found {0} unique items', len(unique_items))
Expand All @@ -303,8 +304,6 @@ def find_search_results(self, series, episodes, search_mode, forced_search=False
# unpack all of the quality lists into a single sorted list
items_list = list(sorted_items)

cl = []

# Move through each item and parse it into a quality
search_results = []
for item in items_list:
Expand Down Expand Up @@ -443,6 +442,7 @@ def find_search_results(self, series, episodes, search_mode, forced_search=False
search_result.actual_season = int(sql_results[0][b'season'])
search_result.actual_episodes = [int(sql_results[0][b'episode'])]

cl = []
# Iterate again over the search results, and see if there is anything we want.
for search_result in search_results:

Expand Down Expand Up @@ -521,10 +521,6 @@ def make_id(name):

return re.sub(r'[^\w\d_]', '_', str(name).strip().lower())

def search_rss(self, episodes):
"""Find cached needed episodes."""
return self.cache.find_needed_episodes(episodes)

def seed_ratio(self):
"""Return ratio."""
return ''
Expand Down
84 changes: 52 additions & 32 deletions medusa/search/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -475,87 +475,107 @@ def wanted_episodes(series_obj, from_date):
u'reason': should_search_reason,
}
)

ep_obj = series_obj.get_episode(episode[b'season'], episode[b'episode'])
ep_obj.wanted_quality = [i for i in all_qualities if i > cur_quality]
ep_obj.wanted_quality = [
quality
for quality in all_qualities
if Quality.is_higher_quality(
cur_quality, quality, allowed_qualities, preferred_qualities
)
]
wanted.append(ep_obj)

return wanted


def search_for_needed_episodes(force=False):
"""
Check providers for details on wanted episodes.

:return: episodes we have a search hit for
"""
found_results = {}

"""Check providers for details on wanted episodes."""
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you complete this docstring please?

"""
Check providers for details on wanted episodes.

:param force: ???
:return: a list of ???
"""

show_list = app.showList
from_date = datetime.date.fromordinal(1)
episodes = []

for cur_show in show_list:
if cur_show.paused:
log.debug(u'Not checking for needed episodes of {0} because the show is paused', cur_show.name)
log.debug(
u'Not checking for needed episodes of {0} because the show is paused',
cur_show.name,
)
continue
episodes.extend(wanted_episodes(cur_show, from_date))

if not episodes and not force:
# nothing wanted so early out, ie: avoid whatever arbitrarily
# complex thing a provider cache update entails, for example,
# reading rss feeds
return list(itervalues(found_results))

original_thread_name = threading.currentThread().name
return []

providers = enabled_providers(u'daily')

if not providers:
log.warning(u'No NZB/Torrent providers found or enabled in the application config for daily searches.'
u' Please check your settings')
return list(itervalues(found_results))
log.warning(
u'No NZB/Torrent providers found or enabled in the application config for daily searches.'
u' Please check your settings'
)
return []

original_thread_name = threading.currentThread().name
log.info(u'Using daily search providers')

for cur_provider in providers:
threading.currentThread().name = u'{thread} :: [{provider}]'.format(thread=original_thread_name,
provider=cur_provider.name)
threading.currentThread().name = u'{thread} :: [{provider}]'.format(
thread=original_thread_name, provider=cur_provider.name
)
cur_provider.cache.update_cache()

single_results = {}
multi_results = []
for cur_provider in providers:
threading.currentThread().name = u'{thread} :: [{provider}]'.format(thread=original_thread_name,
provider=cur_provider.name)
threading.currentThread().name = u'{thread} :: [{provider}]'.format(
thread=original_thread_name, provider=cur_provider.name
)
try:
cur_found_results = cur_provider.search_rss(episodes)
found_results = cur_provider.cache.find_needed_episodes(episodes)
except AuthException as error:
log.error(u'Authentication error: {0}', ex(error))
continue

# pick a single result for each episode, respecting existing results
for cur_ep in cur_found_results:
if not cur_ep.series or cur_ep.series.paused:
log.debug(u'Skipping {0} because the show is paused ', cur_ep.pretty_name())
for episode_no, results in iteritems(found_results):
if results[0].series.paused:
log.debug(u'Skipping {0} because the show is paused.', results[0].series.name)
continue

# if all results were rejected move on to the next episode
wanted_results = filter_results(cur_found_results[cur_ep])
wanted_results = filter_results(results)
if not wanted_results:
log.debug(u'All found results for {0} were rejected.', cur_ep.pretty_name())
log.debug(u'All found results for {0} were rejected.', results[0].series.name)
continue

best_result = pick_result(wanted_results)
# if it's already in the list (from another provider) and the newly found quality is no better then skip it
if cur_ep in found_results and best_result.quality <= found_results[cur_ep].quality:
continue

# Skip the result if search delay is enabled for the provider.
if delay_search(best_result):
continue

found_results[cur_ep] = best_result
if episode_no in (SEASON_RESULT, MULTI_EP_RESULT):
multi_results.append(best_result)
else:
# if it's already in the list (from another provider) and
# the newly found quality is no better then skip it
if episode_no in single_results:
allowed_qualities, preferred_qualities = results[0].series.current_qualities
if not Quality.is_higher_quality(
single_results[episode_no].quality,
best_result.quality,
allowed_qualities,
preferred_qualities
):
continue
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Log a debug message when a result is skipped.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wouldn't add this. It's going to be very spammy.


single_results[episode_no] = best_result

threading.currentThread().name = original_thread_name

return list(itervalues(found_results))
return combine_results(multi_results, list(itervalues(single_results)))


def delay_search(best_result):
Expand Down
82 changes: 43 additions & 39 deletions medusa/tv/cache.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,10 @@
app,
db,
)
from medusa.common import (
MULTI_EP_RESULT,
SEASON_RESULT,
)
from medusa.helper.common import episode_num
from medusa.helper.exceptions import AuthException
from medusa.logger.adapters.style import BraceAdapter
Expand Down Expand Up @@ -413,7 +417,7 @@ def add_cache_entry(self, name, url, seeders, leechers, size, pubdate, parsed_re
proper_tags = '|'.join(parse_result.proper_tags)

if not self.item_in_cache(url):
log.debug('Added RSS item: {0} to cache: {1} with url {2}', name, self.provider_id, url)
log.debug('Added item: {0} to cache: {1} with url {2}', name, self.provider_id, url)
return [
b'INSERT INTO [{name}] '
b' (name, season, episodes, indexerid, url, time, quality, '
Expand All @@ -427,7 +431,7 @@ def add_cache_entry(self, name, url, seeders, leechers, size, pubdate, parsed_re
seeders, leechers, size, pubdate, proper_tags, cur_timestamp, parse_result.series.indexer]
]
else:
log.debug('Updating RSS item: {0} to cache: {1}', name, self.provider_id)
log.debug('Updating item: {0} to cache: {1}', name, self.provider_id)
return [
b'UPDATE [{name}] '
b'SET name=?, season=?, episodes=?, indexer=?, indexerid=?, '
Expand All @@ -441,13 +445,6 @@ def add_cache_entry(self, name, url, seeders, leechers, size, pubdate, parsed_re
seeders, leechers, size, pubdate, proper_tags, url]
]

def search_cache(self, episode, forced_search=False,
down_cur_quality=False):
"""Search cache for needed episodes."""
needed_eps = self.find_needed_episodes(episode, forced_search,
down_cur_quality)
return needed_eps[episode] if episode in needed_eps else []

def item_in_cache(self, url):
"""Check if the url is already available for the specific provider."""
cache_db_con = self._get_db()
Expand Down Expand Up @@ -475,7 +472,7 @@ def find_needed_episodes(self, episode, forced_search=False,
b' season = ? AND'
b' episodes LIKE ?'.format(name=self.provider_id),
[episode.series.indexer, episode.series.series_id, episode.season,
b'%|{0}|%'.format(episode.episode)]
'%|{0}|%'.format(episode.episode)]
)
else:
for ep_obj in episode:
Expand All @@ -491,7 +488,7 @@ def find_needed_episodes(self, episode, forced_search=False,
for x in ep_obj.wanted_quality))
),
[ep_obj.series.indexer, ep_obj.series.series_id, ep_obj.season,
b'%|{0}|%'.format(ep_obj.episode)]]
'%|{0}|%'.format(ep_obj.episode)]]
)

if results:
Expand Down Expand Up @@ -532,35 +529,44 @@ def find_needed_episodes(self, episode, forced_search=False,
log.debug('{0} is not an anime, skipping', series_obj.name)
continue

# get season and ep data (ignoring multi-eps for now)
search_result.season = int(cur_result[b'season'])
if search_result.season == -1:
continue

cur_ep = cur_result[b'episodes'].split('|')[1]
if not cur_ep:
continue

cur_ep = int(cur_ep)

# build a result object
search_result.quality = int(cur_result[b'quality'])
search_result.release_group = cur_result[b'release_group']
search_result.version = cur_result[b'version']
search_result.name = cur_result[b'name']
search_result.url = cur_result[b'url']
search_result.season = int(cur_result[b'season'])
search_result.actual_season = search_result.season

# if the show says we want that episode then add it to the list
if not series_obj.want_episode(search_result.season, cur_ep, search_result.quality,
forced_search, down_cur_quality):
log.debug('Ignoring {0}', cur_result[b'name'])
sql_episodes = cur_result[b'episodes'].strip('|')
# Season result
if not sql_episodes:
ep_objs = self.series.get_all_episodes(search_result.season)
actual_episodes = [ep.episode for ep in ep_objs]
episode_number = SEASON_RESULT
# Multi or single episode result
else:
actual_episodes = [int(ep) for ep in sql_episodes.split('|')]
ep_objs = [series_obj.get_episode(search_result.season, ep) for ep in actual_episodes]
if len(actual_episodes) == 1:
episode_number = actual_episodes[0]
else:
episode_number = MULTI_EP_RESULT

all_wanted = True
for cur_ep in actual_episodes:
# if the show says we want that episode then add it to the list
if not series_obj.want_episode(search_result.season, cur_ep, search_result.quality,
forced_search, down_cur_quality):
log.debug('Ignoring {0} because one or more episodes are unwanted', cur_result[b'name'])
all_wanted = False
break

if not all_wanted:
continue

search_result.episodes = [series_obj.get_episode(search_result.season, cur_ep)]

search_result.actual_episodes = [search_result.episodes[0].episode]
search_result.actual_season = search_result.season

# build a result object
search_result.name = cur_result[b'name']
search_result.url = cur_result[b'url']
search_result.episodes = ep_objs
search_result.actual_episodes = actual_episodes

log.debug(
'{id}: Using cached results from {provider} for series {show_name!r} episode {ep}', {
Expand All @@ -582,15 +588,13 @@ def find_needed_episodes(self, episode, forced_search=False,

# FIXME: Should be changed to search_result.search_type
search_result.forced_search = forced_search

search_result.download_current_quality = down_cur_quality

episode_object = search_result.episodes[0]
# add it to the list
if episode_object not in needed_eps:
needed_eps[episode_object] = [search_result]
if episode_number not in needed_eps:
needed_eps[episode_number] = [search_result]
else:
needed_eps[episode_object].append(search_result)
needed_eps[episode_number].append(search_result)

# datetime stamp this search so cache gets cleared
self.searched = time()
Expand Down