diff --git a/CHANGES b/CHANGES index 25123363..58befd5e 100644 --- a/CHANGES +++ b/CHANGES @@ -3,6 +3,7 @@ ## Version 4.9.0 * Adding #196 Support for HEVC Apple Silicon M1 encoder (thanks to Kay Singh) +* Adding #323 ignore errors options options for queue (thanks to Don Gafford) ## Version 4.8.1 diff --git a/fastflix/application.py b/fastflix/application.py index e840d1bc..d98d97c8 100644 --- a/fastflix/application.py +++ b/fastflix/application.py @@ -119,9 +119,9 @@ def register_app(): logger.exception("Could not set application ID for Windows, please raise issue in github with above error") -def start_app(worker_queue, status_queue, log_queue, queue_list, queue_lock): +def start_app(worker_queue, status_queue, log_queue, queue_lock): app = create_app() - app.fastflix = FastFlix(queue=queue_list, queue_lock=queue_lock) + app.fastflix = FastFlix(queue_lock=queue_lock) app.fastflix.log_queue = log_queue app.fastflix.status_queue = status_queue app.fastflix.worker_queue = worker_queue diff --git a/fastflix/conversion_worker.py b/fastflix/conversion_worker.py index eeb77ba6..49d5dead 100644 --- a/fastflix/conversion_worker.py +++ b/fastflix/conversion_worker.py @@ -13,6 +13,7 @@ from fastflix.language import t from fastflix.shared import file_date from fastflix.models.video import Video +from fastflix.ff_queue import save_queue, get_queue logger = logging.getLogger("fastflix-core") @@ -53,9 +54,11 @@ def allow_sleep_mode(): logger.debug("System has been allowed to enter sleep mode again") -def get_next_video(queue_list, queue_lock) -> Optional[Video]: +def get_next_video(queue_lock) -> Optional[Video]: with queue_lock: - logger.debug(f"Retrieving next video from {queue_list}") + logger.debug(f"Retrieving next video from {queue_path}") + queue_list = get_queue(queue_file=queue_path) + for video in queue_list: if ( not video.status.complete @@ -70,7 +73,6 @@ def get_next_video(queue_list, queue_lock) -> Optional[Video]: def set_status( current_video: Video, - queue_list, queue_lock, complete=None, success=None, @@ -84,6 +86,7 @@ def set_status( return with queue_lock: + queue_list = get_queue(queue_file=queue_path) for i, video in enumerate(queue_list): if video.uuid == current_video.uuid: video_pos = i @@ -113,11 +116,14 @@ def set_status( if reset_commands: video_copy.status.current_command = 0 + logger.debug(f"Set status of {current_video.uuid} to {video_copy.status}") queue_list.insert(video_pos, video_copy) + save_queue(queue_list, queue_path) + @reusables.log_exception(log="fastflix-core") -def queue_worker(gui_proc, worker_queue, status_queue, log_queue, queue_list, queue_lock: Lock): +def queue_worker(gui_proc, worker_queue, status_queue, log_queue, queue_lock: Lock): runner = BackgroundRunner(log_queue=log_queue) # Command looks like (video_uuid, command_uuid, command, work_dir) @@ -126,6 +132,7 @@ def queue_worker(gui_proc, worker_queue, status_queue, log_queue, queue_list, qu currently_encoding = False paused = False video: Optional[Video] = None + ignore_errors = False def start_command(): nonlocal currently_encoding @@ -149,7 +156,7 @@ def start_command(): video.video_settings.conversion_commands[video.status.current_command].command, work_dir=str(video.work_path), ) - set_status(video, queue_list=queue_list, queue_lock=queue_lock, running=True) + set_status(video, queue_lock=queue_lock, running=True) status_queue.put(("queue",)) # status_queue.put(("running", commands_to_run[0][0], commands_to_run[0][1], runner.started_at.isoformat())) @@ -157,32 +164,37 @@ def start_command(): while True: if currently_encoding and not runner.is_alive(): reusables.remove_file_handlers(logger) + log_queue.put("STOP_TIMER") + skip_commands = False + if runner.error_detected: logger.info(t("Error detected while converting")) - # Stop working! currently_encoding = False - set_status(video, queue_list=queue_list, queue_lock=queue_lock, errored=True) - status_queue.put(("error",)) - allow_sleep_mode() - if gui_died: - return - continue + set_status(video, queue_lock=queue_lock, errored=True) + if ignore_errors: + skip_commands = True + else: + # Stop working! + status_queue.put(("error",)) + allow_sleep_mode() + if gui_died: + return + continue # Successfully encoded, do next one if it exists # First check if the current video has more commands video.status.current_command += 1 - log_queue.put("STOP_TIMER") - if len(video.video_settings.conversion_commands) > video.status.current_command: + if not skip_commands and len(video.video_settings.conversion_commands) > video.status.current_command: logger.debug("About to run next command for this video") - set_status(video, queue_list=queue_list, queue_lock=queue_lock, next_command=True) + set_status(video, queue_lock=queue_lock, next_command=True) status_queue.put(("queue",)) start_command() continue else: logger.debug(f"{video.uuid} has been completed") - set_status(video, queue_list=queue_list, queue_lock=queue_lock, next_command=True, complete=True) + set_status(video, queue_lock=queue_lock, next_command=True, complete=True) status_queue.put(("queue",)) video = None @@ -192,7 +204,7 @@ def start_command(): logger.debug(t("Queue has been paused")) continue - if video := get_next_video(queue_list=queue_list, queue_lock=queue_lock): + if video := get_next_video(queue_lock=queue_lock): start_command() continue else: @@ -233,15 +245,22 @@ def start_command(): # Request looks like (queue command, log_dir, (commands)) log_path = Path(request[1]) if not currently_encoding and not paused: - video = get_next_video(queue_list=queue_list, queue_lock=queue_lock) + video = get_next_video(queue_lock=queue_lock) if video: start_command() + if request[0] == "ignore error": + logger.info("Have been told to ignore errors") + ignore_errors = True + if request[0] == "stop on error": + logger.info("Have been told to stop on errors") + ignore_errors = False + if request[0] == "cancel": logger.debug(t("Cancel has been requested, killing encoding")) runner.kill() if video: - set_status(video, queue_list=queue_list, queue_lock=queue_lock, reset_commands=True, cancelled=True) + set_status(video, queue_lock=queue_lock, reset_commands=True, cancelled=True) currently_encoding = False allow_sleep_mode() status_queue.put(("cancelled", video.uuid if video else "")) @@ -257,7 +276,7 @@ def start_command(): logger.debug(t("Command worker received request to resume encoding")) if not currently_encoding: if not video: - video = get_next_video(queue_list=queue_list, queue_lock=queue_lock) + video = get_next_video(queue_lock=queue_lock) if video: start_command() diff --git a/fastflix/data/languages.yaml b/fastflix/data/languages.yaml index 1b48d000..056ef30e 100644 --- a/fastflix/data/languages.yaml +++ b/fastflix/data/languages.yaml @@ -6262,3 +6262,15 @@ HEVC coding profile - must match bit depth: por: Perfil de codificação HEVC - deve corresponder à profundidade do bit swe: HEVC-kodningsprofil - måste matcha bitdjupet pol: Profil kodowania HEVC - musi być zgodny z głębią bitową +Ignore Errors: + eng: Ignore Errors + deu: Fehler ignorieren + fra: Ignorer les erreurs + ita: Ignorare gli errori + spa: Ignorar errores + zho: 忽略错误 + jpn: エラーを無視する + rus: Игнорировать ошибки + por: Ignorar erros + swe: Ignorera fel + pol: Ignoruj błędy \ No newline at end of file diff --git a/fastflix/entry.py b/fastflix/entry.py index 3e04d9cf..034f6701 100644 --- a/fastflix/entry.py +++ b/fastflix/entry.py @@ -26,13 +26,13 @@ sys.exit(1) -def separate_app_process(worker_queue, status_queue, log_queue, queue_list, queue_lock): +def separate_app_process(worker_queue, status_queue, log_queue, queue_lock): """This prevents any QT components being imported in the main process""" from fastflix.application import start_app freeze_support() try: - start_app(worker_queue, status_queue, log_queue, queue_list, queue_lock) + start_app(worker_queue, status_queue, log_queue, queue_lock) except Exception as err: print(f"Could not start GUI process - Error: {err}", file=sys.stderr) raise err @@ -128,7 +128,7 @@ def main(): try: gui_proc = Process( target=separate_app_process, - args=(worker_queue, status_queue, log_queue, queue_list, queue_lock), + args=(worker_queue, status_queue, log_queue, queue_lock), ) gui_proc.start() except Exception: @@ -136,7 +136,7 @@ def main(): return exit_status try: - queue_worker(gui_proc, worker_queue, status_queue, log_queue, queue_list, queue_lock) + queue_worker(gui_proc, worker_queue, status_queue, log_queue, queue_lock) exit_status = 0 except Exception: logger.exception("Exception occurred while running FastFlix core") diff --git a/fastflix/ff_queue.py b/fastflix/ff_queue.py index c8e294b9..7334d85f 100644 --- a/fastflix/ff_queue.py +++ b/fastflix/ff_queue.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from typing import List +from typing import List, Optional import os from pathlib import Path import logging @@ -17,7 +17,7 @@ logger = logging.getLogger("fastflix") -def get_queue(queue_file: Path, config: Config) -> List[Video]: +def get_queue(queue_file: Path) -> List[Video]: if not queue_file.exists(): return [] @@ -68,12 +68,17 @@ def get_queue(queue_file: Path, config: Config) -> List[Video]: return queue -def save_queue(queue: List[Video], queue_file: Path, config: Config): +def save_queue(queue: List[Video], queue_file: Path, config: Optional[Config] = None): items = [] - queue_covers = config.work_path / "covers" - queue_covers.mkdir(parents=True, exist_ok=True) - queue_data = config.work_path / "queue_extras" - queue_data.mkdir(parents=True, exist_ok=True) + + if config is not None: + queue_covers = config.work_path / "covers" + queue_covers.mkdir(parents=True, exist_ok=True) + queue_data = config.work_path / "queue_extras" + queue_data.mkdir(parents=True, exist_ok=True) + else: + queue_data = Path() + queue_covers = Path() def update_conversion_command(vid, old_path: str, new_path: str): for command in vid["video_settings"]["conversion_commands"]: @@ -87,28 +92,29 @@ def update_conversion_command(vid, old_path: str, new_path: str): video["source"] = os.fspath(video["source"]) video["work_path"] = os.fspath(video["work_path"]) video["video_settings"]["output_path"] = os.fspath(video["video_settings"]["output_path"]) - if metadata := video["video_settings"]["video_encoder_settings"].get("hdr10plus_metadata"): - new_metadata_file = queue_data / f"{uuid.uuid4().hex}_metadata.json" - try: - shutil.copy(metadata, new_metadata_file) - except OSError: - logger.exception("Could not save HDR10+ metadata file to queue recovery location, removing HDR10+") - - update_conversion_command( - video, - str(metadata), - str(new_metadata_file), - ) - video["video_settings"]["video_encoder_settings"]["hdr10plus_metadata"] = str(new_metadata_file) - for track in video["video_settings"]["attachment_tracks"]: - if track.get("file_path"): - new_file = queue_covers / f'{uuid.uuid4().hex}_{track["file_path"].name}' + if config: + if metadata := video["video_settings"]["video_encoder_settings"].get("hdr10plus_metadata"): + new_metadata_file = queue_data / f"{uuid.uuid4().hex}_metadata.json" try: - shutil.copy(track["file_path"], new_file) + shutil.copy(metadata, new_metadata_file) except OSError: - logger.exception("Could not save cover to queue recovery location, removing cover") - update_conversion_command(video, str(track["file_path"]), str(new_file)) - track["file_path"] = str(new_file) + logger.exception("Could not save HDR10+ metadata file to queue recovery location, removing HDR10+") + + update_conversion_command( + video, + str(metadata), + str(new_metadata_file), + ) + video["video_settings"]["video_encoder_settings"]["hdr10plus_metadata"] = str(new_metadata_file) + for track in video["video_settings"]["attachment_tracks"]: + if track.get("file_path"): + new_file = queue_covers / f'{uuid.uuid4().hex}_{track["file_path"].name}' + try: + shutil.copy(track["file_path"], new_file) + except OSError: + logger.exception("Could not save cover to queue recovery location, removing cover") + update_conversion_command(video, str(track["file_path"]), str(new_file)) + track["file_path"] = str(new_file) items.append(video) try: diff --git a/fastflix/models/fastflix.py b/fastflix/models/fastflix.py index 9302f6ed..1bed0882 100644 --- a/fastflix/models/fastflix.py +++ b/fastflix/models/fastflix.py @@ -30,5 +30,4 @@ class FastFlix(BaseModel): log_queue: Any = None current_video: Optional[Video] = None - queue: Any = None queue_lock: Any = None diff --git a/fastflix/widgets/main.py b/fastflix/widgets/main.py index 03061f2c..cf675cc8 100644 --- a/fastflix/widgets/main.py +++ b/fastflix/widgets/main.py @@ -34,7 +34,7 @@ from fastflix.language import t from fastflix.models.fastflix_app import FastFlixApp from fastflix.models.video import Status, Video, VideoSettings, Crop -from fastflix.ff_queue import save_queue +from fastflix.ff_queue import save_queue, get_queue from fastflix.resources import ( get_icon, main_icon, @@ -1793,16 +1793,18 @@ def encode_video(self): self.video_options.queue.reset_pause_encode() return - if not self.app.fastflix.queue or self.app.fastflix.current_video: + queue_list = self.get_queue_list() + + if not queue_list or self.app.fastflix.current_video: add_current = True - if self.app.fastflix.queue and self.app.fastflix.current_video: + if queue_list and self.app.fastflix.current_video: add_current = yes_no_message("Add current video to queue?", yes_text="Yes", no_text="No") if add_current: if not self.add_to_queue(): return requests = ["add_items", str(self.app.fastflix.log_path)] - for video in self.app.fastflix.queue: + for video in queue_list: if video.status.ready: break else: @@ -1819,7 +1821,7 @@ def encode_video(self): def get_commands(self): commands = [] - for video in self.app.fastflix.queue: + for video in self.get_queue_list(): if video.status.complete or video.status.error: continue for command in video.video_settings.conversion_commands: @@ -1842,7 +1844,8 @@ def add_to_queue(self): return False source_in_queue = False - for video in self.app.fastflix.queue: + queue_list = self.get_queue_list() + for video in queue_list: if video.status.complete: continue if self.app.fastflix.current_video.source == video.source: @@ -1856,7 +1859,8 @@ def add_to_queue(self): # return with self.app.fastflix.queue_lock: - self.app.fastflix.queue.append(copy.deepcopy(self.app.fastflix.current_video)) + queue_list.append(copy.deepcopy(self.app.fastflix.current_video)) + save_queue(queue_list, self.app.fastflix.queue_path, self.app.fastflix.config) self.video_options.update_queue() self.video_options.show_queue() @@ -1866,10 +1870,12 @@ def add_to_queue(self): self.app.fastflix.worker_queue.put(tuple(requests)) self.clear_current_video() - with self.app.fastflix.queue_lock: - save_queue(self.app.fastflix.queue, self.app.fastflix.queue_path, self.app.fastflix.config) return True + def get_queue_list(self): + with self.app.fastflix.queue_lock: + return get_queue(self.app.fastflix.queue_path) + @reusables.log_exception("fastflix", show_traceback=False) def conversion_complete(self, return_code): self.converting = False @@ -1961,12 +1967,12 @@ def dragMoveEvent(self, event): def status_update(self): logger.debug(f"Updating queue from command worker") - with self.app.fastflix.queue_lock: - save_queue(self.app.fastflix.queue, self.app.fastflix.queue_path, self.app.fastflix.config) + # with self.app.fastflix.queue_lock: + # save_queue(self.app.fastflix.queue, self.app.fastflix.queue_path, self.app.fastflix.config) self.video_options.update_queue() def find_video(self, uuid) -> Video: - for video in self.app.fastflix.queue: + for video in self.get_queue_list(): if uuid == video.uuid: return video raise FlixError(f'{t("No video found for")} {uuid}') @@ -1996,14 +2002,20 @@ def run(self): self.main.status_update_signal.emit() self.app.processEvents() if status[0] == "complete": + logger.debug("GUI received status queue complete") self.main.completed.emit(0) elif status[0] == "error": + logger.debug("GUI received status queue errored") self.main.completed.emit(1) elif status[0] == "cancelled": + logger.debug("GUI received status queue errored") self.main.cancelled.emit("|".join(status[1:])) elif status[0] == "exit": + logger.debug("GUI received ask to exit") try: self.terminate() finally: self.main.close_event.emit() return + else: + logger.debug(f"GUI received status {status[0]}") diff --git a/fastflix/widgets/panels/debug_panel.py b/fastflix/widgets/panels/debug_panel.py index 339c3035..5aa6fe82 100644 --- a/fastflix/widgets/panels/debug_panel.py +++ b/fastflix/widgets/panels/debug_panel.py @@ -46,7 +46,7 @@ def reset(self): self.addTab(self.get_textbox(Box(self.app.fastflix.config.dict())), "Config") self.addTab(self.get_textbox(Box(self.get_ffmpeg_details())), "FFmpeg Details") - self.addTab(self.get_textbox(BoxList(self.app.fastflix.queue)), "Queue") + # self.addTab(self.get_textbox(BoxList(self.app.fastflix.queue)), "Queue") self.addTab(self.get_textbox(Box(self.app.fastflix.encoders)), "Encoders") self.addTab(self.get_textbox(BoxList(self.app.fastflix.audio_encoders)), "Audio Encoders") if self.app.fastflix.current_video: diff --git a/fastflix/widgets/panels/queue_panel.py b/fastflix/widgets/panels/queue_panel.py index a51ff497..b8e64090 100644 --- a/fastflix/widgets/panels/queue_panel.py +++ b/fastflix/widgets/panels/queue_panel.py @@ -222,6 +222,10 @@ def __init__(self, parent, app: FastFlixApp): self.pause_encode.setFixedWidth(120) self.pause_encode.setToolTip(t("Pause / Resume the current command")) + self.ignore_errors = QtWidgets.QCheckBox(t("Ignore Errors")) + self.ignore_errors.toggled.connect(self.ignore_failures) + self.ignore_errors.setFixedWidth(120) + self.after_done_combo = QtWidgets.QComboBox() self.after_done_combo.addItem("None") actions = set() @@ -241,6 +245,7 @@ def __init__(self, parent, app: FastFlixApp): self.after_done_combo.setMaximumWidth(150) top_layout.addWidget(QtWidgets.QLabel(t("After Conversion"))) top_layout.addWidget(self.after_done_combo, QtCore.Qt.AlignRight) + top_layout.addWidget(self.ignore_errors, QtCore.Qt.AlignRight) top_layout.addWidget(self.pause_encode, QtCore.Qt.AlignRight) top_layout.addWidget(self.pause_queue, QtCore.Qt.AlignRight) top_layout.addWidget(self.clear_queue, QtCore.Qt.AlignRight) @@ -254,7 +259,8 @@ def __init__(self, parent, app: FastFlixApp): save_queue([], queue_file=self.app.fastflix.queue_path, config=self.app.fastflix.config) def queue_startup_check(self): - new_queue = get_queue(self.app.fastflix.queue_path, self.app.fastflix.config) + with self.app.fastflix.queue_lock: + new_queue = get_queue(self.app.fastflix.queue_path) # self.app.fastflix.queue.append(item) reset_vids = [] remove_vids = [] @@ -279,21 +285,22 @@ def queue_startup_check(self): title="Recover Queue Items", ): with self.app.fastflix.queue_lock: - for item in new_queue: - self.app.fastflix.queue.append(item) - # self.app.fastflix.queue = [] - with self.app.fastflix.queue_lock: - save_queue(self.app.fastflix.queue, self.app.fastflix.queue_path, self.app.fastflix.config) + save_queue(new_queue, self.app.fastflix.queue_path, self.app.fastflix.config) + else: + with self.app.fastflix.queue_lock: + save_queue([], self.app.fastflix.queue_path, self.app.fastflix.config) self.new_source() def reorder(self, update=True): super().reorder(update=update) with self.app.fastflix.queue_lock: - for i in range(len(self.app.fastflix.queue)): - self.app.fastflix.queue.pop() + queue_list = get_queue(self.app.fastflix.queue_path) + for i in range(len(queue_list)): + queue_list.pop() for track in self.tracks: - self.app.fastflix.queue.append(track.video) + queue_list.append(track.video) + save_queue(queue_list, self.app.fastflix.queue_path, self.app.fastflix.config) for track in self.tracks: track.widgets.up_button.setDisabled(False) @@ -306,7 +313,11 @@ def new_source(self): for track in self.tracks: track.close() self.tracks = [] - for i, video in enumerate(self.app.fastflix.queue, start=1): + + with self.app.fastflix.queue_lock: + new_queue = get_queue(self.app.fastflix.queue_path) + + for i, video in enumerate(new_queue, start=1): self.tracks.append(EncodeItem(self, video, index=i)) if self.tracks: self.tracks[0].widgets.up_button.setDisabled(True) @@ -317,22 +328,21 @@ def clear_complete(self): for queued_item in self.tracks: if queued_item.video.status.complete: self.remove_item(queued_item.video, part_of_clear=True) - with self.app.fastflix.queue_lock: - save_queue(self.app.fastflix.queue, self.app.fastflix.queue_path, self.app.fastflix.config) self.new_source() def remove_item(self, video, part_of_clear=False): with self.app.fastflix.queue_lock: - for i, vid in enumerate(self.app.fastflix.queue): + queue_list = get_queue(self.app.fastflix.queue_path) + for i, vid in enumerate(queue_list): if vid.uuid == video.uuid: pos = i break else: logger.error("No matching video found to remove from queue") return - self.app.fastflix.queue.pop(pos) + queue_list.pop(pos) if not part_of_clear: - save_queue(self.app.fastflix.queue, self.app.fastflix.queue_path, self.app.fastflix.config) + save_queue(queue_list, self.app.fastflix.queue_path, self.app.fastflix.config) if not part_of_clear: self.new_source() @@ -349,11 +359,12 @@ def pause_resume_queue(self): if self.paused: self.pause_queue.setText(t("Pause Queue")) self.pause_queue.setIcon(self.app.style().standardIcon(QtWidgets.QStyle.SP_MediaPause)) - for i, video in enumerate(self.app.fastflix.queue): - if video.status.ready: - self.main.converting = True - self.main.set_convert_button(False) - break + with self.app.fastflix.queue_lock: + for i, video in enumerate(get_queue(self.app.fastflix.queue_path)): + if video.status.ready: + self.main.converting = True + self.main.set_convert_button(False) + break self.app.fastflix.worker_queue.put(["resume queue"]) else: self.pause_queue.setText(t("Resume Queue")) @@ -381,6 +392,12 @@ def pause_resume_encode(self): self.app.fastflix.worker_queue.put(["pause encode"]) self.encode_paused = not self.encode_paused + def ignore_failures(self): + if self.ignore_errors.isChecked(): + self.app.fastflix.worker_queue.put(["ignore error"]) + else: + self.app.fastflix.worker_queue.put(["stop on error"]) + @reusables.log_exception("fastflix", show_traceback=False) def set_after_done(self): option = self.after_done_combo.currentText() @@ -398,7 +415,8 @@ def set_after_done(self): def retry_video(self, current_video): with self.app.fastflix.queue_lock: - for i, video in enumerate(self.app.fastflix.queue): + queue_list = get_queue(self.app.fastflix.queue_path) + for i, video in enumerate(queue_list): if video.uuid == current_video.uuid: video_pos = i break @@ -406,11 +424,11 @@ def retry_video(self, current_video): logger.error(f"Can't find video {current_video.uuid} in queue to update its status") return - video = self.app.fastflix.queue.pop(video_pos) + video = queue_list.pop(video_pos) video.status.cancelled = False video.status.current_command = 0 - self.app.fastflix.queue.insert(video_pos, video) - save_queue(self.app.fastflix.queue, self.app.fastflix.queue_path, self.app.fastflix.config) + queue_list.insert(video_pos, video) + save_queue(queue_list, self.app.fastflix.queue_path, self.app.fastflix.config) self.new_source()