From 6d7008725558ce60e9076270a8283cb80e773ea4 Mon Sep 17 00:00:00 2001 From: Administrator Date: Thu, 18 Jul 2024 18:29:51 +0000 Subject: [PATCH 1/5] Fix: media-enhancer creates blank items for missing seasons and episodes --- src/program/libraries/symlink.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/program/libraries/symlink.py b/src/program/libraries/symlink.py index 55aae294..cb549192 100644 --- a/src/program/libraries/symlink.py +++ b/src/program/libraries/symlink.py @@ -90,11 +90,13 @@ def process_shows(directory: Path, item_type: str, is_anime: bool = False) -> Sh show_item = Show({'imdb_id': imdb_id.group(), 'title': title.group(1)}) if is_anime: show_item.is_anime = True + seasons = {} for season in os.listdir(directory / show): if not (season_number := re.search(r'(\d+)', season)): logger.log("NOT_FOUND", f"Can't extract season number at path {directory / show / season}") continue season_item = Season({'number': int(season_number.group())}) + episodes = {} for episode in os.listdir(directory / show / season): if not (episode_number := re.search(r's\d+e(\d+)', episode)): logger.log("NOT_FOUND", f"Can't extract episode number at path {directory / show / season / episode}") @@ -110,6 +112,13 @@ def process_shows(directory: Path, item_type: str, is_anime: bool = False) -> Sh episode_item.set("update_folder", "updated") if is_anime: episode_item.is_anime = True - season_item.add_episode(episode_item) - show_item.add_season(season_item) - yield show_item + #season_item.add_episode(episode_item) + episodes[int(episode_number.group(1))] = episode_item + if len(episodes) > 0: + for i in range(1, max(episodes.keys())+1): + season_item.add_episode(episodes.get(i, Episode({'number': i}))) + seasons[int(season_number.group())] = season_item + if len(seasons) > 0: + for i in range(1, max(seasons.keys())+1): + show_item.add_season(seasons.get(i, Season({'number': i}))) + yield show_item \ No newline at end of file From 720f6013351fe4102ffa0d48c8765fcb8a773612 Mon Sep 17 00:00:00 2001 From: Administrator Date: Thu, 18 Jul 2024 20:44:12 +0000 Subject: [PATCH 2/5] Fix: Check for items in queue by _id instead of comparing objects to prevent issues. --- src/program/db/db_functions.py | 5 ++-- src/program/program.py | 55 +++++++++++++++++++++------------- 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/src/program/db/db_functions.py b/src/program/db/db_functions.py index 1bd0e70b..98b9f4d6 100644 --- a/src/program/db/db_functions.py +++ b/src/program/db/db_functions.py @@ -69,8 +69,9 @@ def _run_thread_with_db_item(fn, service, program, input_item: MediaItem | None) with db.Session() as session: if isinstance(input_item, (Movie, Show, Season, Episode)): item = input_item - if not _check_for_and_run_insertion_required(session, item): - item = _get_item_from_db(session, item) + if not _check_for_and_run_insertion_required(session, item): + pass + item = _get_item_from_db(session, item) #session.merge(item) for res in fn(item): diff --git a/src/program/program.py b/src/program/program.py index 805402ff..149a970d 100644 --- a/src/program/program.py +++ b/src/program/program.py @@ -193,9 +193,9 @@ def _retry_library(self) -> None: with db.Session() as session: items_to_submit = session.execute(select(MediaItem).where( (MediaItem.last_state!="Completed" ) & ( (MediaItem.type == 'show') | (MediaItem.type == 'movie') )).order_by(MediaItem.scraped_at.desc())).unique().scalars().all() session.expunge_all() - logger.log("PROGRAM", f"Found {len(items_to_submit)} items to retry") - for item in items_to_submit: - self._push_event_queue(Event(emitted_by=self.__class__, item=item)) + logger.log("PROGRAM", f"Found {len(items_to_submit)} items to retry") + for item in items_to_submit: + self._push_event_queue(Event(emitted_by=self.__class__, item=item)) def _schedule_functions(self) -> None: """Schedule each service based on its update interval.""" @@ -237,25 +237,37 @@ def _schedule_services(self) -> None: coalesce=False, ) logger.log("PROGRAM", f"Scheduled {service_cls.__name__} to run every {update_interval} seconds.") - + def _id_in_queue(self, id): + for i in self.queued_items: + if i._id == id: + return True + return False + def _id_in_running_items(self, id): + for i in self.running_items: + if i._id == id: + return True + return False def _push_event_queue(self, event): with self.mutex: if( not event.item in self.queued_items and not event.item in self.running_items): - if ( isinstance(event.item, Show) - and (any( [s for s in event.item.seasons if s in self.queued_items or s in self.running_items]) - or any([e for e in [s.episodes for s in event.item.seasons] if e in self.queued_items or e in self.running_items]) ) - ): - return - if isinstance(event.item, Season) and any( [e for e in event.item.episodes if e in self.queued_items or e in self.running_items] ): - return - if hasattr(event.item, "parent") and event.item.parent in self.queued_items : - return - if hasattr(event.item, "parent") and hasattr(event.item.parent, "parent") and event.item.parent.parent and event.item.parent.parent in self.queued_items : - return - if hasattr(event.item, "parent") and event.item.parent in self.running_items : - return - if hasattr(event.item, "parent") and hasattr(event.item.parent, "parent") and event.item.parent.parent and event.item.parent.parent in self.running_items : - return + if hasattr(event.item, "_id"): + if isinstance(event.item, Show): + for s in event.item.seasons: + if self._id_in_queue(s._id) or self._id_in_running_items(s._id): + return + for e in s.episodes: + if self._id_in_queue(e._id) or self._id_in_running_items(e._id): + return + return + if isinstance(event.item, Season): + for e in event.item.episodes: + if self._id_in_queue(e._id) or self._id_in_running_items(e._id): + return + return + if hasattr(event.item, "parent") and ( self._id_in_queue(event.item.parent._id) or self._id_in_running_items(event.item.parent._id) ): + return + if hasattr(event.item, "parent") and hasattr(event.item.parent, "parent") and event.item.parent.parent and ( self._id_in_queue(event.item.parent.parent._id) or self._id_in_running_items(event.item.parent.parent._id)): + return self.queued_items.append(event.item) self.event_queue.put(event) if not isinstance(event.item, (Show, Movie, Episode, Season)): @@ -282,7 +294,10 @@ def add_to_running(self, item, service_name): if item is None: return if item not in self.running_items: - self.running_items.append(item) + if isinstance(item, MediaItem) and not self._id_in_running_items(item._id): + self.running_items.append(item) + elif not isinstance(item, MediaItem): + self.running_items.append(item) logger.log("PROGRAM", f"Item {item.log_string} started running section {service_name}" ) def _process_future_item(self, future: Future, service: Service, orig_item: MediaItem) -> None: From 06219fccbf7af48dd1f7f6e843afef8dca40e8b6 Mon Sep 17 00:00:00 2001 From: Administrator Date: Thu, 18 Jul 2024 21:37:30 +0000 Subject: [PATCH 3/5] Fix: Remove erronous returns --- src/program/program.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/program/program.py b/src/program/program.py index 149a970d..34bde256 100644 --- a/src/program/program.py +++ b/src/program/program.py @@ -258,12 +258,11 @@ def _push_event_queue(self, event): for e in s.episodes: if self._id_in_queue(e._id) or self._id_in_running_items(e._id): return - return + if isinstance(event.item, Season): for e in event.item.episodes: if self._id_in_queue(e._id) or self._id_in_running_items(e._id): return - return if hasattr(event.item, "parent") and ( self._id_in_queue(event.item.parent._id) or self._id_in_running_items(event.item.parent._id) ): return if hasattr(event.item, "parent") and hasattr(event.item.parent, "parent") and event.item.parent.parent and ( self._id_in_queue(event.item.parent.parent._id) or self._id_in_running_items(event.item.parent.parent._id)): From c6fde8cd4750b8304369546fd15b3233c8b931a1 Mon Sep 17 00:00:00 2001 From: Administrator Date: Thu, 18 Jul 2024 23:05:27 +0000 Subject: [PATCH 4/5] Feat: Try and scrape entire show/season again if backoff timer allows when in partially completed --- src/program/scrapers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/scrapers/__init__.py b/src/program/scrapers/__init__.py index 7a9aa8cd..04cf2db4 100644 --- a/src/program/scrapers/__init__.py +++ b/src/program/scrapers/__init__.py @@ -55,7 +55,7 @@ def yield_incomplete_children(self, item: MediaItem) -> Union[List[Season], List return None def partial_state(self, item: MediaItem) -> bool: - if item.state != States.PartiallyCompleted: + if item.state != States.PartiallyCompleted or self.can_we_scrape(item): return False if isinstance(item, Show): sres = [s for s in item.seasons if s.state != States.Completed and s.is_released and self.should_submit(s)] From 91aa929afa462ea152ade520cd9768c2f400f6c3 Mon Sep 17 00:00:00 2001 From: Administrator Date: Fri, 19 Jul 2024 00:26:38 +0000 Subject: [PATCH 5/5] Fix: Allow shows and seasons to return scraped state if they're scraped so they can proceed to the next phase --- src/program/media/item.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/program/media/item.py b/src/program/media/item.py index c38bd0da..470d2fdc 100644 --- a/src/program/media/item.py +++ b/src/program/media/item.py @@ -340,14 +340,14 @@ def _determine_state(self): if len(self.episodes) > 0: if all(episode.state == States.Completed for episode in self.episodes): return States.Completed - if any(episode.state == States.Completed for episode in self.episodes): - return States.PartiallyCompleted if all(episode.state == States.Symlinked for episode in self.episodes): return States.Symlinked if all(episode.file and episode.folder for episode in self.episodes): return States.Downloaded if self.is_scraped(): return States.Scraped + if any(episode.state == States.Completed for episode in self.episodes): + return States.PartiallyCompleted if any(episode.state == States.Indexed for episode in self.episodes): return States.Indexed if any(episode.state == States.Requested for episode in self.episodes): @@ -493,18 +493,17 @@ def get_season_index_by_id(self, item_id): def _determine_state(self): if all(season.state == States.Completed for season in self.seasons): return States.Completed - - if any( - season.state in (States.Completed, States.PartiallyCompleted) - for season in self.seasons - ): - return States.PartiallyCompleted if all(season.state == States.Symlinked for season in self.seasons): return States.Symlinked if all(season.state == States.Downloaded for season in self.seasons): return States.Downloaded if self.is_scraped(): return States.Scraped + if any( + season.state in (States.Completed, States.PartiallyCompleted) + for season in self.seasons + ): + return States.PartiallyCompleted if any(season.state == States.Indexed for season in self.seasons): return States.Indexed if any(season.state == States.Requested for season in self.seasons):