Update on the reencoding function

PixivHelper.py

@@ -1005,19 +1005,7 @@ def convert_ugoira(ugoira_file, exportname, ffmpeg, codec, param, extension, ima
     # if not os.path.exists(os.path.abspath(ffmpeg)):
     #     raise PixivException(f"Cannot find ffmpeg executables => {ffmpeg}", errorCode=PixivException.MISSING_CONFIG)
 
-    d = tempfile.mkdtemp(prefix="convert_ugoira")
-    d = d.replace(os.sep, '/')
-
-    # Issue #1035
-    if not os.path.exists(d):
-        new_temp = os.path.abspath(f"ugoira_{int(datetime.now().timestamp())}")
-        new_temp = new_temp.replace(os.sep, '/')
-        os.makedirs(new_temp)
-        print_and_log("warn", f"Cannot create temp folder at {d}, using current folder as the temp location => {new_temp}")
-        d = new_temp
-        # check again if still fail
-        if not os.path.exists(d):
-            raise PixivException(f"Cannot create temp folder => {d}", errorCode=PixivException.OTHER_ERROR)
+    d = create_temp_dir(prefix="convert_ugoira")
 
     if exportname is None or len(exportname) == 0:
         name = '.'.join(ugoira_file.split('.')[:-1])
@@ -1080,6 +1068,23 @@ def convert_ugoira(ugoira_file, exportname, ffmpeg, codec, param, extension, ima
         print()
 
 
+def create_temp_dir(prefix: str=None) -> str:
+    d = tempfile.mkdtemp(prefix=prefix)
+    d = d.replace(os.sep, '/')
+
+    # Issue #1035
+    if not os.path.exists(d):
+        new_temp = os.path.abspath(f"file_{int(datetime.now().timestamp())}")
+        new_temp = new_temp.replace(os.sep, '/')
+        os.makedirs(new_temp)
+        print_and_log("warn", f"Cannot create temp folder at {d}, using current folder as the temp location => {new_temp}")
+        d = new_temp
+        # check again if still fail
+        if not os.path.exists(d):
+            raise PixivException(f"Cannot create temp folder => {d}", errorCode=PixivException.OTHER_ERROR)
+    return d
+
+
 def ffmpeg_progress_report(p: subprocess.Popen) -> subprocess.Popen:
     chatter = ""
     while p.stderr:

PixivImage.py

@@ -482,8 +482,7 @@ def WriteXMP(self, filename):
         import tempfile
 
         # need to use temp file due to bad unicode support for pyexiv2 in windows
-        d = tempfile.mkdtemp(prefix="xmp")
-        d = d.replace(os.sep, '/')
+        d = PixivHelper.create_temp_dir(prefix="xmp")
         tempname = f"{d}/{self.imageId}.xmp"
 
         info = codecs.open(tempname, 'wb', encoding='utf-8')

PixivImageHandler.py

@@ -4,6 +4,8 @@
 import os
 import re
 import sys
+import shutil
+import time
 import traceback
 import pathlib
 import urllib
@@ -33,6 +35,7 @@ def process_image(caller,
                   image_response_count=-1,
                   notifier=None,
                   useblacklist=True,
+                  reencoding=False,
                   manga_series_order=-1,
                   manga_series_parent=None) -> int:
     # caller function/method
@@ -67,7 +70,7 @@ def process_image(caller,
             in_db = True
 
         # skip if already recorded in db and alwaysCheckFileSize is disabled and overwrite is disabled.
-        if in_db and not config.alwaysCheckFileSize and not config.overwrite:
+        if in_db and not config.alwaysCheckFileSize and not config.overwrite and not reencoding:
             PixivHelper.print_and_log(None, f'Already downloaded in DB: {image_id}')
             gc.collect()
             return PixivConstant.PIXIVUTIL_SKIP_DUPLICATE_NO_WAIT
@@ -442,39 +445,111 @@ def process_manga_series(caller,
 def process_ugoira_local(caller, config):
     directory = config.rootDirectory
     counter = 0
+    d = ""
+    res = None
+    counter = 0
+    list_done = list()
 
     try:
         print('')
-        counter = 0
-        for zip in pathlib.Path(directory).rglob('*.zip'):
-            counter += 1
-            PixivHelper.print_and_log(None, f"# Ugoira {counter}")
-            zip_name = os.path.splitext(os.path.basename(zip))[0]
-            PixivHelper.print_and_log("info", f"Deleting old ugoira files ...", newline = False)
-            for file in pathlib.Path(os.path.dirname(zip)).rglob(f'{zip_name}.*'):
-                file_ext = os.path.splitext(os.path.basename(file))[1]
-                if  ((("ugoira" in file_ext) and (config.createUgoira))     or
-                    (("gif" in file_ext) and (config.createGif))            or
-                    (("png" in file_ext) and (config.createApng))           or
-                    (("webm" in file_ext) and (config.createWebm))          or
-                    (("webp" in file_ext) and (config.createWebp))):
-                    abs_file_path = os.path.abspath(file)
-                    PixivHelper.print_and_log("debug", f"Deleting {abs_file_path}")
-                    os.remove(abs_file_path)
-            PixivHelper.print_and_log(None, f" done.")
-            # Get id artwork
-            image_id = zip_name.partition("_")[0]
-            process_image(  caller,
-                            config,
-                            artist=None,
-                            image_id=image_id,
-                            useblacklist=False)
+        for extension in ["ugoira", "zip"]: # always ugoira then zip
+            for zip in pathlib.Path(directory).rglob(f'*.{extension}'):
+                zip_name = os.path.splitext(os.path.basename(zip))[0]
+                zip_dir = os.path.dirname(zip)
+                image_id = zip_name.partition("_")[0]
+                if image_id not in list_done:
+                    counter += 1
+                    PixivHelper.print_and_log(None, f"# Ugoira {counter}")
+                    PixivHelper.print_and_log("info", f"Deleting old animated files ...", newline = False)
+                    d = PixivHelper.create_temp_dir(prefix="reencoding")
+
+                    # List and move all files related to the image_id
+                    for file in os.listdir(zip_dir):
+                        if os.path.isfile(os.path.join(zip_dir,file)) and zip_name in file:
+                            file_basename = os.path.basename(file)
+                            file_ext = os.path.splitext(file_basename)[1]
+                            if  ((("gif" in file_ext) and (config.createGif))           or
+                                (("png" in file_ext) and (config.createApng))           or
+                                (("webm" in file_ext) and (config.createWebm))          or
+                                (("webp" in file_ext) and (config.createWebp))          or
+                                (("ugoira"in file_ext) and (config.createUgoira))       or
+                                ("zip" in file_ext)):
+                                abs_file_path = os.path.abspath(os.path.join(zip_dir,file))
+                                PixivHelper.print_and_log("debug", f"Moving {abs_file_path} to {d}")
+                                if ("zip" in file_ext) or ("ugoira" in file_ext):
+                                    shutil.copy2(abs_file_path, os.path.join(d, file_basename))
+                                else:
+                                    shutil.move(abs_file_path, os.path.join(d, file_basename))
+                    PixivHelper.print_and_log(None, f" done.")
+
+                    # Process artwork locally
+                    if "ugoira" in extension and not config.overwrite:
+                        try:
+                            msg = Fore.YELLOW + Style.NORMAL + f'Processing Image Id: {image_id}' + Style.RESET_ALL
+                            PixivHelper.print_and_log(None, msg)
+                            PixivDownloadHandler.handle_ugoira(None, str(zip), config, None)
+                            res = PixivConstant.PIXIVUTIL_OK
+                        except PixivException as ex:
+                            PixivHelper.print_and_log('error', f'PixivException for Image ID ({image_id}): {ex}')
+                            PixivHelper.print_and_log('error', f'Stack Trace: {sys.exc_info()}')
+                            res = PixivConstant.PIXIVUTIL_NOT_OK
+                        except Exception as ex:
+                            PixivHelper.print_and_log('error', f'Exception for Image ID ({image_id}): {ex}')
+                            PixivHelper.print_and_log('error', f'Stack Trace: {sys.exc_info()}')
+                            exc_type, exc_value, exc_traceback = sys.exc_info()
+                            traceback.print_exception(exc_type, exc_value, exc_traceback)
+                            res = PixivConstant.PIXIVUTIL_NOT_OK
+                        finally:
+                            if res == PixivConstant.PIXIVUTIL_NOT_OK:
+                                PixivHelper.print_and_log('warn', f'Failed to process Image ID {image_id} locally: will retry with online infos')
+                                PixivHelper.print_and_log('debug', f'Removing corrupted ugoira {zip}')
+                                os.remove(zip)
+
+                    # Process artwork with online infos
+                    if "zip" in extension or res == PixivConstant.PIXIVUTIL_NOT_OK or ("ugoira" in extension and config.overwrite):
+                        res = process_image(caller,
+                                            config,
+                                            artist=None,
+                                            image_id=image_id,
+                                            useblacklist=False,
+                                            reencoding=True)
+                        if res == PixivConstant.PIXIVUTIL_NOT_OK:
+                            PixivHelper.print_and_log("warn", f"Cannot process Image Id: {image_id}, restoring old animated files...", newline = False)
+                            for file_name in os.listdir(d):
+                                PixivHelper.print_and_log("debug", f"Moving back {os.path.join(d, file_name)} to {os.path.join(zip_dir, file_name)}")
+                                shutil.move(os.path.join(d, file_name), os.path.join(zip_dir, file_name)) # overwrite corrupted file generated
+                            PixivHelper.print_and_log(None, f" done.")
+                            print('')
+
+                    # Checking result
+                    list_file_zipdir = os.listdir(zip_dir)
+                    for file_name in os.listdir(d):
+                        if file_name not in list_file_zipdir and config.backupOldFile:
+                            split_name = file_name.rsplit(".", 1)
+                            new_name = file_name + "." + str(int(time.time()))
+                            if len(split_name) == 2:
+                                new_name = split_name[0] + "." + str(int(time.time())) + "." + split_name[1]
+                            PixivHelper.print_and_log('warn', f"Could not found the animated file re-encoded ==> {file_name}, backing up to: {new_name}")
+                            PixivHelper.print_and_log('warn', f"The new encoded file may have another name or the artist may have change its name")
+                            PixivHelper.print_and_log("debug", f"Rename and move {os.path.join(d, file_name)} to {os.path.join(zip_dir, new_name)}")
+                            shutil.move(os.path.join(d, file_name), os.path.join(zip_dir, new_name))
+                    print('')
+
+                    # Delete temp path
+                    if os.path.exists(d) and d != "":
+                        PixivHelper.print_and_log("debug", f"Deleting path {d}")
+                        shutil.rmtree(d)
+                    list_done.append(image_id)
         if counter == 0:
-            PixivHelper.print_and_log('info', "No zip file found for re-encoding ugoira.")
-
+            PixivHelper.print_and_log('info', "No zip file or ugoira found to re-encode animated files.")
+    
     except Exception as ex:
         if isinstance(ex, KeyboardInterrupt):
             raise
         PixivHelper.print_and_log('error', 'Error at process_ugoira_local(): %s' %str(sys.exc_info()))
         PixivHelper.print_and_log('error', 'failed')
         raise
+    finally:
+        if os.path.exists(d) and d != "":
+            PixivHelper.print_and_log("debug", f"Deleting path {d} in finally")
+            shutil.rmtree(d)

PixivUtil2.py

@@ -736,13 +736,19 @@ def menu_ugoira_reencode(opisvalid, args, options):
     __log__.info('Re-encode Ugoira (u)')
     msg = Fore.YELLOW + Style.NORMAL + f'WARNING: THIS ACTION CANNOT BE UNDO !' + Style.RESET_ALL
     PixivHelper.print_and_log(None, msg)
-    msg = Fore.YELLOW + Style.NORMAL + f'You are about to re-encode and overwrite all of your ugoira based on its zip file.' + Style.RESET_ALL
+    msg = Fore.YELLOW + Style.NORMAL + f'You are about to re-encode and overwrite all of your stored ugoira and its related files (gif, webm ...).' + Style.RESET_ALL
     PixivHelper.print_and_log(None, msg)
     arg = input(Fore.YELLOW + Style.BRIGHT + 'Do you really want to proceed ? [y/n, default is no]: ' + Style.RESET_ALL).rstrip("\r") or 'n'
     sure = arg.lower()
     if sure not in ('y', 'n'):
         PixivHelper.print_and_log("error", f"Invalid args for ugoira reencode: {arg}, valid values are [y/n].")
         return
+    if __config__.overwrite:
+        arg = input(Fore.YELLOW + Style.BRIGHT + 'Overwrite option is set to True, all animated files will be re-download from Pixiv and not re-encode locally. Do you still want to proceed ? [y/n, default is no]: ' + Style.RESET_ALL).rstrip("\r") or 'n'
+        sure = arg.lower()
+        if sure not in ('y', 'n'):
+            PixivHelper.print_and_log("error", f"Invalid args for ugoira reencode: {arg}, valid values are [y/n].")
+            return
     if sure == 'y':
         PixivImageHandler.process_ugoira_local(sys.modules[__name__], __config__)