From ff6358ea0741304f4506515c778e0526c4933ace Mon Sep 17 00:00:00 2001 From: mugulmd Date: Thu, 5 Jan 2023 07:01:54 -0800 Subject: [PATCH 01/29] [ui] new Thumbnail disk cache --- meshroom/ui/app.py | 2 + meshroom/ui/components/thumbnail.py | 82 +++++++++++++++++++ .../ui/qml/ImageGallery/ImageDelegate.qml | 3 +- 3 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 meshroom/ui/components/thumbnail.py diff --git a/meshroom/ui/app.py b/meshroom/ui/app.py index 36f7472c22..982e8e43db 100644 --- a/meshroom/ui/app.py +++ b/meshroom/ui/app.py @@ -15,6 +15,7 @@ from meshroom.ui.components.clipboard import ClipboardHelper from meshroom.ui.components.filepath import FilepathHelper from meshroom.ui.components.scene3D import Scene3DHelper, Transformations3DHelper +from meshroom.ui.components.thumbnail import ThumbnailCache from meshroom.ui.palette import PaletteManager from meshroom.ui.reconstruction import Reconstruction from meshroom.ui.utils import QmlInstantEngine @@ -146,6 +147,7 @@ def __init__(self, args): self.engine.rootContext().setContextProperty("Scene3DHelper", Scene3DHelper(parent=self)) self.engine.rootContext().setContextProperty("Transformations3DHelper", Transformations3DHelper(parent=self)) self.engine.rootContext().setContextProperty("Clipboard", ClipboardHelper(parent=self)) + self.engine.rootContext().setContextProperty("ThumbnailCache", ThumbnailCache(parent=self)) # additional context properties self.engine.rootContext().setContextProperty("_PaletteManager", PaletteManager(self.engine, parent=self)) diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py new file mode 100644 index 0000000000..6cacbe3d2e --- /dev/null +++ b/meshroom/ui/components/thumbnail.py @@ -0,0 +1,82 @@ +from PySide2.QtCore import QObject, Slot, QSize, QUrl, Qt +from PySide2.QtGui import QImageReader, QImageWriter + +import hashlib +import pathlib +import os + + +class ThumbnailCache(QObject): + """ + ThumbnailCache provides an abstraction for the thumbnail cache on disk, available in QML. + + For a given image file, it ensures the corresponding thumbnail exists (by creating it if necessary) + and gives access to it. + + The default cache location is a subdirectory of the user's home directory: + ~/Meshroom/thumbnails. + + The main use case for thumbnails in Meshroom is in the ImageGallery. + """ + + thumbnailDir = os.path.join(pathlib.Path.home(), 'Meshroom', 'thumbnails') + thumbnailSize = QSize(100, 100) + + @Slot(QUrl, result=QUrl) + def thumbnail(self, imgSource): + """ + Retrieve the filepath of the thumbnail corresponding to a given image. + If the thumbnail does not exist on disk, it is created. + + Args: + imgSource (QUrl): the filepath to the input image + + Returns: + QUrl: the filepath to the corresponding thumbnail + """ + # Safety check + if not imgSource.isValid(): + return None + + imgPath = imgSource.toLocalFile() + + # Use SHA1 hashing to associate a unique thumbnail to the image + digest = hashlib.sha1(imgPath.encode('utf-8')).hexdigest() + path = os.path.join(ThumbnailCache.thumbnailDir, f'{digest}.jpg') + source = QUrl.fromLocalFile(path) + + # Check if thumbnail already exists + if os.path.exists(path): + return source + + # Thumbnail does not exist, therefore we create it: + # 1. read the image + # 2. scale it to thumbnail dimensions + # 3. write it in the cache + print(f'[ThumbnailCache] Creating thumbnail {path} for image {imgPath}') + + # Initialize image reader object + reader = QImageReader() + reader.setFileName(imgPath) + reader.setAutoTransform(True) + + # Read image and check for potential errors + img = reader.read() + if img.isNull(): + print(f'[ThumbnailCache] Error when reading image: {reader.errorString()}') + return None + + # Make sure the thumbnail directory exists before writing into it + os.makedirs(ThumbnailCache.thumbnailDir, exist_ok=True) + + # Scale image while preserving aspect ratio + thumbnail = img.scaled(ThumbnailCache.thumbnailSize, aspectMode=Qt.KeepAspectRatio) + + # Write thumbnail to disk and check for potential errors + writer = QImageWriter(path) + success = writer.write(thumbnail) + if not success: + print(f'[ThumbnailCache] Error when writing thumbnail: {writer.errorString()}') + return None + + return source diff --git a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml index 484c172f6b..3dad8556c6 100644 --- a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml +++ b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml @@ -79,8 +79,7 @@ Item { Image { anchors.fill: parent anchors.margins: 4 - source: root.source - sourceSize: Qt.size(100, 100) + source: ThumbnailCache.thumbnail(root.source) asynchronous: true autoTransform: true fillMode: Image.PreserveAspectFit From 1819eacbdb6f16adcdd00adaeb372bf448e824e9 Mon Sep 17 00:00:00 2001 From: mugulmd Date: Thu, 5 Jan 2023 08:59:26 -0800 Subject: [PATCH 02/29] [ui] override default thumbnail directory with environment variable --- meshroom/ui/app.py | 5 +++++ meshroom/ui/components/thumbnail.py | 1 + 2 files changed, 6 insertions(+) diff --git a/meshroom/ui/app.py b/meshroom/ui/app.py index 982e8e43db..66fbd278c4 100644 --- a/meshroom/ui/app.py +++ b/meshroom/ui/app.py @@ -116,6 +116,11 @@ def __init__(self, args): pwd = os.path.dirname(__file__) self.setWindowIcon(QIcon(os.path.join(pwd, "img/meshroom.svg"))) + # User specified thumbnail directory + thumbnailDir = os.getenv('MESHROOM_THUMBNAIL_DIR') + if thumbnailDir is not None: + ThumbnailCache.thumbnailDir = thumbnailDir + # QML engine setup qmlDir = os.path.join(pwd, "qml") url = os.path.join(qmlDir, "main.qml") diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index 6cacbe3d2e..17ee29c452 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -15,6 +15,7 @@ class ThumbnailCache(QObject): The default cache location is a subdirectory of the user's home directory: ~/Meshroom/thumbnails. + This location can be overriden with the MESHROOM_THUMBNAIL_DIR environment variable. The main use case for thumbnails in Meshroom is in the ImageGallery. """ From 7566544164ecbe9680adca82eb8b664dbcd1216b Mon Sep 17 00:00:00 2001 From: mugulmd Date: Fri, 6 Jan 2023 02:10:34 -0800 Subject: [PATCH 03/29] [ui] remove old thumbnails from disk cache when launching app --- meshroom/ui/app.py | 3 ++ meshroom/ui/components/thumbnail.py | 47 +++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/meshroom/ui/app.py b/meshroom/ui/app.py index 66fbd278c4..417176d890 100644 --- a/meshroom/ui/app.py +++ b/meshroom/ui/app.py @@ -121,6 +121,9 @@ def __init__(self, args): if thumbnailDir is not None: ThumbnailCache.thumbnailDir = thumbnailDir + # Clean thumbnail directory + ThumbnailCache.clean() + # QML engine setup qmlDir = os.path.join(pwd, "qml") url = os.path.join(qmlDir, "main.qml") diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index 17ee29c452..a1ba739f8d 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -1,9 +1,10 @@ from PySide2.QtCore import QObject, Slot, QSize, QUrl, Qt from PySide2.QtGui import QImageReader, QImageWriter -import hashlib -import pathlib import os +import pathlib +import hashlib +import time class ThumbnailCache(QObject): @@ -17,12 +18,22 @@ class ThumbnailCache(QObject): ~/Meshroom/thumbnails. This location can be overriden with the MESHROOM_THUMBNAIL_DIR environment variable. + This class also takes care of cleaning the thumbnail directory, + i.e. scanning this directory and removing thumbnails that have not been used for too long. + The default time limit is one week. + The main use case for thumbnails in Meshroom is in the ImageGallery. """ + # Thumbnail cache directory thumbnailDir = os.path.join(pathlib.Path.home(), 'Meshroom', 'thumbnails') + + # Thumbnail dimensions limit (the actual dimensions of a thumbnail will depend on the aspect ratio) thumbnailSize = QSize(100, 100) + # Time limit for thumbnail storage on disk, expressed in days + storageTimeLimit = 7 + @Slot(QUrl, result=QUrl) def thumbnail(self, imgSource): """ @@ -48,6 +59,8 @@ def thumbnail(self, imgSource): # Check if thumbnail already exists if os.path.exists(path): + # Update last modification time + pathlib.Path(path).touch(exist_ok=True) return source # Thumbnail does not exist, therefore we create it: @@ -81,3 +94,33 @@ def thumbnail(self, imgSource): return None return source + + @staticmethod + def clean(): + """ + Scan the thumbnail directory and + remove all thumbnails that have not been used for more than storageTimeLimit days. + """ + # Get current time + now = time.time() + + # Scan thumbnail directory and gather all thumbnails to remove + toRemove = [] + for entry in os.scandir(ThumbnailCache.thumbnailDir): + if not entry.is_file(): + continue + + # Compute storage duration since last usage of thumbnail + lastUsage = os.path.getmtime(entry.path) + storageTime = now - lastUsage + print(f'[ThumbnailCache] Thumbnail {entry.name} has been stored for {storageTime}s') + + # Mark as removable if storage time exceeds limit + if storageTime > ThumbnailCache.storageTimeLimit * 3600 * 24: + print(f'[ThumbnailCache] {entry.name} exceeded storage time limit') + toRemove.append(entry.path) + + # Remove all thumbnails marked as removable + for path in toRemove: + print(f'[ThumbnailCache] Remove {path}') + os.remove(path) From bfdcfb52cc747b82565654b348178d9ce537ccef Mon Sep 17 00:00:00 2001 From: mugulmd Date: Fri, 6 Jan 2023 02:17:18 -0800 Subject: [PATCH 04/29] [ui] ThumbnailCache: use logging instead of print --- meshroom/ui/components/thumbnail.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index a1ba739f8d..899f88984e 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -5,6 +5,7 @@ import pathlib import hashlib import time +import logging class ThumbnailCache(QObject): @@ -67,7 +68,7 @@ def thumbnail(self, imgSource): # 1. read the image # 2. scale it to thumbnail dimensions # 3. write it in the cache - print(f'[ThumbnailCache] Creating thumbnail {path} for image {imgPath}') + logging.debug(f'[ThumbnailCache] Creating thumbnail {path} for image {imgPath}') # Initialize image reader object reader = QImageReader() @@ -77,7 +78,7 @@ def thumbnail(self, imgSource): # Read image and check for potential errors img = reader.read() if img.isNull(): - print(f'[ThumbnailCache] Error when reading image: {reader.errorString()}') + logging.error(f'[ThumbnailCache] Error when reading image: {reader.errorString()}') return None # Make sure the thumbnail directory exists before writing into it @@ -90,7 +91,7 @@ def thumbnail(self, imgSource): writer = QImageWriter(path) success = writer.write(thumbnail) if not success: - print(f'[ThumbnailCache] Error when writing thumbnail: {writer.errorString()}') + logging.error(f'[ThumbnailCache] Error when writing thumbnail: {writer.errorString()}') return None return source @@ -113,14 +114,14 @@ def clean(): # Compute storage duration since last usage of thumbnail lastUsage = os.path.getmtime(entry.path) storageTime = now - lastUsage - print(f'[ThumbnailCache] Thumbnail {entry.name} has been stored for {storageTime}s') + logging.debug(f'[ThumbnailCache] Thumbnail {entry.name} has been stored for {storageTime}s') # Mark as removable if storage time exceeds limit if storageTime > ThumbnailCache.storageTimeLimit * 3600 * 24: - print(f'[ThumbnailCache] {entry.name} exceeded storage time limit') + logging.debug(f'[ThumbnailCache] {entry.name} exceeded storage time limit') toRemove.append(entry.path) # Remove all thumbnails marked as removable for path in toRemove: - print(f'[ThumbnailCache] Remove {path}') + logging.debug(f'[ThumbnailCache] Remove {path}') os.remove(path) From b372946516de4fb27b45eea584788942ec34dbd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Vital?= Date: Fri, 6 Jan 2023 12:47:02 +0100 Subject: [PATCH 05/29] [ui] ThumbnailCache: check cache dir existence before cleaning --- meshroom/ui/components/thumbnail.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index 899f88984e..85f5e2bb0e 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -102,6 +102,11 @@ def clean(): Scan the thumbnail directory and remove all thumbnails that have not been used for more than storageTimeLimit days. """ + # Check if thumbnail directory exists + if not os.path.exists(ThumbnailCache.thumbnailDir): + logging.debug('[ThumbnailCache] Thumbnail directory does not exist yet.') + return + # Get current time now = time.time() From a96d88692310be9d7ba76710c239c72ab25f8e95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Vital?= Date: Fri, 6 Jan 2023 14:27:37 +0100 Subject: [PATCH 06/29] [ui] override default time limit for thumbnails on disk with env var --- meshroom/ui/app.py | 5 +++++ meshroom/ui/components/thumbnail.py | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/meshroom/ui/app.py b/meshroom/ui/app.py index 417176d890..5a3ed33f3f 100644 --- a/meshroom/ui/app.py +++ b/meshroom/ui/app.py @@ -121,6 +121,11 @@ def __init__(self, args): if thumbnailDir is not None: ThumbnailCache.thumbnailDir = thumbnailDir + # User specifed time limit for thumbnails on disk (expressed in days) + thumbnailTimeLimit = float(os.getenv('MESHROOM_THUMBNAIL_TIME_LIMIT')) + if thumbnailTimeLimit is not None: + ThumbnailCache.storageTimeLimit = thumbnailTimeLimit + # Clean thumbnail directory ThumbnailCache.clean() diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index 85f5e2bb0e..e1ca37375d 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -21,7 +21,8 @@ class ThumbnailCache(QObject): This class also takes care of cleaning the thumbnail directory, i.e. scanning this directory and removing thumbnails that have not been used for too long. - The default time limit is one week. + The default time limit is 90 days. + This time limit can be overriden with the MESHROOM_THUMBNAIL_TIME_LIMIT environment variable. The main use case for thumbnails in Meshroom is in the ImageGallery. """ @@ -33,7 +34,7 @@ class ThumbnailCache(QObject): thumbnailSize = QSize(100, 100) # Time limit for thumbnail storage on disk, expressed in days - storageTimeLimit = 7 + storageTimeLimit = 90 @Slot(QUrl, result=QUrl) def thumbnail(self, imgSource): From 220bcfb9e2466081c0dbad7e216f512d97a34a9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Vital?= Date: Fri, 6 Jan 2023 15:24:50 +0100 Subject: [PATCH 07/29] [ui] maximum number of thumbnails on disk --- meshroom/ui/app.py | 9 ++++++-- meshroom/ui/components/thumbnail.py | 36 +++++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/meshroom/ui/app.py b/meshroom/ui/app.py index 5a3ed33f3f..65953efcef 100644 --- a/meshroom/ui/app.py +++ b/meshroom/ui/app.py @@ -122,9 +122,14 @@ def __init__(self, args): ThumbnailCache.thumbnailDir = thumbnailDir # User specifed time limit for thumbnails on disk (expressed in days) - thumbnailTimeLimit = float(os.getenv('MESHROOM_THUMBNAIL_TIME_LIMIT')) + thumbnailTimeLimit = os.getenv('MESHROOM_THUMBNAIL_TIME_LIMIT') if thumbnailTimeLimit is not None: - ThumbnailCache.storageTimeLimit = thumbnailTimeLimit + ThumbnailCache.storageTimeLimit = float(thumbnailTimeLimit) + + # User specifed maximum number of thumbnails on disk + thumbnailMaxNumberOnDisk = os.getenv('MESHROOM_MAX_THUMBNAILS_ON_DISK') + if thumbnailMaxNumberOnDisk is not None: + ThumbnailCache.maxThumbnailsOnDisk = int(thumbnailMaxNumberOnDisk) # Clean thumbnail directory ThumbnailCache.clean() diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index e1ca37375d..45ac2dffdd 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -21,8 +21,14 @@ class ThumbnailCache(QObject): This class also takes care of cleaning the thumbnail directory, i.e. scanning this directory and removing thumbnails that have not been used for too long. - The default time limit is 90 days. - This time limit can be overriden with the MESHROOM_THUMBNAIL_TIME_LIMIT environment variable. + This operation also ensures that the number of thumbnails on disk does not exceed a certain limit, + by removing thumbnails if necessary (from least recently used to most recently used). + + The default time limit is 90 days, + and can be overriden with the MESHROOM_THUMBNAIL_TIME_LIMIT environment variable. + + The default maximum number of thumbnails on disk is 100000, + and can be overriden with the MESHROOM_MAX_THUMBNAILS_ON_DISK. The main use case for thumbnails in Meshroom is in the ImageGallery. """ @@ -36,6 +42,9 @@ class ThumbnailCache(QObject): # Time limit for thumbnail storage on disk, expressed in days storageTimeLimit = 90 + # Maximum number of thumbnails in the cache directory + maxThumbnailsOnDisk = 100000 + @Slot(QUrl, result=QUrl) def thumbnail(self, imgSource): """ @@ -100,8 +109,9 @@ def thumbnail(self, imgSource): @staticmethod def clean(): """ - Scan the thumbnail directory and - remove all thumbnails that have not been used for more than storageTimeLimit days. + Scan the thumbnail directory and: + 1. remove all thumbnails that have not been used for more than storageTimeLimit days + 2. ensure that the number of thumbnails on disk does not exceed maxThumbnailsOnDisk. """ # Check if thumbnail directory exists if not os.path.exists(ThumbnailCache.thumbnailDir): @@ -113,6 +123,7 @@ def clean(): # Scan thumbnail directory and gather all thumbnails to remove toRemove = [] + remaining = [] for entry in os.scandir(ThumbnailCache.thumbnailDir): if not entry.is_file(): continue @@ -122,12 +133,27 @@ def clean(): storageTime = now - lastUsage logging.debug(f'[ThumbnailCache] Thumbnail {entry.name} has been stored for {storageTime}s') - # Mark as removable if storage time exceeds limit if storageTime > ThumbnailCache.storageTimeLimit * 3600 * 24: + # Mark as removable if storage time exceeds limit logging.debug(f'[ThumbnailCache] {entry.name} exceeded storage time limit') toRemove.append(entry.path) + else: + # Store path and last usage time for potentially sorting and removing later + remaining.append((entry.path, lastUsage)) # Remove all thumbnails marked as removable for path in toRemove: logging.debug(f'[ThumbnailCache] Remove {path}') os.remove(path) + + # Check if number of thumbnails on disk exceeds limit + if len(remaining) > ThumbnailCache.maxThumbnailsOnDisk: + nbToRemove = len(remaining) - ThumbnailCache.maxThumbnailsOnDisk + logging.debug(f'[ThumbnailCache] Too many thumbnails: {len(remaining)} remaining, {nbToRemove} will be removed') + + # Sort by last usage order and remove excess + remaining.sort(key=lambda elt: elt[1]) + for i in range(nbToRemove): + path = remaining[i][0] + logging.debug(f'[ThumbnailCache] Remove {path}') + os.remove(path) From 60c8e779d58d6b2d7f89b8070b8614e7ea29656d Mon Sep 17 00:00:00 2001 From: mugulmd Date: Mon, 9 Jan 2023 05:52:16 -0800 Subject: [PATCH 08/29] [ui] create thumbnails asynchronously --- meshroom/ui/app.py | 21 +--- meshroom/ui/components/thumbnail.py | 105 +++++++++++++----- .../ui/qml/ImageGallery/ImageDelegate.qml | 15 ++- 3 files changed, 96 insertions(+), 45 deletions(-) diff --git a/meshroom/ui/app.py b/meshroom/ui/app.py index 65953efcef..9126027f52 100644 --- a/meshroom/ui/app.py +++ b/meshroom/ui/app.py @@ -116,23 +116,10 @@ def __init__(self, args): pwd = os.path.dirname(__file__) self.setWindowIcon(QIcon(os.path.join(pwd, "img/meshroom.svg"))) - # User specified thumbnail directory - thumbnailDir = os.getenv('MESHROOM_THUMBNAIL_DIR') - if thumbnailDir is not None: - ThumbnailCache.thumbnailDir = thumbnailDir - - # User specifed time limit for thumbnails on disk (expressed in days) - thumbnailTimeLimit = os.getenv('MESHROOM_THUMBNAIL_TIME_LIMIT') - if thumbnailTimeLimit is not None: - ThumbnailCache.storageTimeLimit = float(thumbnailTimeLimit) - - # User specifed maximum number of thumbnails on disk - thumbnailMaxNumberOnDisk = os.getenv('MESHROOM_MAX_THUMBNAILS_ON_DISK') - if thumbnailMaxNumberOnDisk is not None: - ThumbnailCache.maxThumbnailsOnDisk = int(thumbnailMaxNumberOnDisk) - - # Clean thumbnail directory - ThumbnailCache.clean() + # Initialize thumbnail cache: + # - read related environment variables + # - clean cache directory and make sure it exists on disk + ThumbnailCache.initialize() # QML engine setup qmlDir = os.path.join(pwd, "qml") diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index 45ac2dffdd..5f16c2f66f 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -1,19 +1,23 @@ +from meshroom.common import Signal + from PySide2.QtCore import QObject, Slot, QSize, QUrl, Qt from PySide2.QtGui import QImageReader, QImageWriter import os -import pathlib +from pathlib import Path import hashlib import time import logging +from multiprocessing.pool import ThreadPool class ThumbnailCache(QObject): - """ - ThumbnailCache provides an abstraction for the thumbnail cache on disk, available in QML. + """ThumbnailCache provides an abstraction for the thumbnail cache on disk, available in QML. For a given image file, it ensures the corresponding thumbnail exists (by creating it if necessary) and gives access to it. + Since creating thumbnails can be long (as it requires to read the full image from disk) + it is performed asynchronously to avoid blocking the main thread. The default cache location is a subdirectory of the user's home directory: ~/Meshroom/thumbnails. @@ -34,7 +38,7 @@ class ThumbnailCache(QObject): """ # Thumbnail cache directory - thumbnailDir = os.path.join(pathlib.Path.home(), 'Meshroom', 'thumbnails') + thumbnailDir = os.path.join(Path.home(), 'Meshroom', 'thumbnails') # Thumbnail dimensions limit (the actual dimensions of a thumbnail will depend on the aspect ratio) thumbnailSize = QSize(100, 100) @@ -45,39 +49,90 @@ class ThumbnailCache(QObject): # Maximum number of thumbnails in the cache directory maxThumbnailsOnDisk = 100000 + # Signal to notify listeners that a thumbnail was created and written on disk + # This signal has one argument: the url of the image that the thumbnail is associated to + thumbnailCreated = Signal(QUrl) + + # Thread pool for running createThumbnail asynchronously on a fixed number of worker threads + pool = ThreadPool() + + @staticmethod + def initialize(): + """Initialize static fields in cache class and cache directory on disk.""" + # User specified thumbnail directory + dir = os.getenv('MESHROOM_THUMBNAIL_DIR') + if dir is not None: + ThumbnailCache.thumbnailDir = dir + + # User specifed time limit for thumbnails on disk (expressed in days) + timeLimit = os.getenv('MESHROOM_THUMBNAIL_TIME_LIMIT') + if timeLimit is not None: + ThumbnailCache.storageTimeLimit = float(timeLimit) + + # User specifed maximum number of thumbnails on disk + maxOnDisk = os.getenv('MESHROOM_MAX_THUMBNAILS_ON_DISK') + if maxOnDisk is not None: + ThumbnailCache.maxThumbnailsOnDisk = int(maxOnDisk) + + # Clean thumbnail directory + ThumbnailCache.clean() + + # Make sure the thumbnail directory exists before writing into it + os.makedirs(ThumbnailCache.thumbnailDir, exist_ok=True) + + @staticmethod + def thumbnailPath(imgPath): + """Use SHA1 hashing to associate a unique thumbnail to an image. + + Args: + imgPath (str): filepath to the input image + + Returns: + str: filepath to the corresponding thumbnail + """ + digest = hashlib.sha1(imgPath.encode('utf-8')).hexdigest() + path = os.path.join(ThumbnailCache.thumbnailDir, f'{digest}.jpg') + return path + @Slot(QUrl, result=QUrl) def thumbnail(self, imgSource): - """ - Retrieve the filepath of the thumbnail corresponding to a given image. - If the thumbnail does not exist on disk, it is created. + """Retrieve the filepath of the thumbnail corresponding to a given image. + + If the thumbnail does not exist on disk, it will be created asynchronously. + When this is done, the createdThumbnail signal is emitted. Args: - imgSource (QUrl): the filepath to the input image + imgSource (QUrl): location of the input image Returns: - QUrl: the filepath to the corresponding thumbnail + QUrl: location of the corresponding thumbnail if it exists, otherwise None """ - # Safety check if not imgSource.isValid(): return None imgPath = imgSource.toLocalFile() - - # Use SHA1 hashing to associate a unique thumbnail to the image - digest = hashlib.sha1(imgPath.encode('utf-8')).hexdigest() - path = os.path.join(ThumbnailCache.thumbnailDir, f'{digest}.jpg') + path = ThumbnailCache.thumbnailPath(imgPath) source = QUrl.fromLocalFile(path) # Check if thumbnail already exists if os.path.exists(path): # Update last modification time - pathlib.Path(path).touch(exist_ok=True) + Path(path).touch(exist_ok=True) return source - # Thumbnail does not exist, therefore we create it: - # 1. read the image - # 2. scale it to thumbnail dimensions - # 3. write it in the cache + # Thumbnail does not exist + # create it in a worker thread to avoid UI freeze + ThumbnailCache.pool.apply_async(self.createThumbnail, args=(imgSource,)) + return None + + def createThumbnail(self, imgSource): + """Load an image, resize it to thumbnail dimensions and save the result in the cache directory. + + Args: + imgSource (QUrl): location of the input image + """ + imgPath = imgSource.toLocalFile() + path = ThumbnailCache.thumbnailPath(imgPath) logging.debug(f'[ThumbnailCache] Creating thumbnail {path} for image {imgPath}') # Initialize image reader object @@ -89,10 +144,7 @@ def thumbnail(self, imgSource): img = reader.read() if img.isNull(): logging.error(f'[ThumbnailCache] Error when reading image: {reader.errorString()}') - return None - - # Make sure the thumbnail directory exists before writing into it - os.makedirs(ThumbnailCache.thumbnailDir, exist_ok=True) + return # Scale image while preserving aspect ratio thumbnail = img.scaled(ThumbnailCache.thumbnailSize, aspectMode=Qt.KeepAspectRatio) @@ -102,14 +154,13 @@ def thumbnail(self, imgSource): success = writer.write(thumbnail) if not success: logging.error(f'[ThumbnailCache] Error when writing thumbnail: {writer.errorString()}') - return None - return source + # Notify listeners + self.thumbnailCreated.emit(imgSource) @staticmethod def clean(): - """ - Scan the thumbnail directory and: + """Scan the thumbnail directory and: 1. remove all thumbnails that have not been used for more than storageTimeLimit days 2. ensure that the number of thumbnails on disk does not exceed maxThumbnailsOnDisk. """ diff --git a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml index 3dad8556c6..baf75847c9 100644 --- a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml +++ b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml @@ -31,6 +31,19 @@ Item { property var metadata: metadataStr ? JSON.parse(viewpoint.get("metadata").value) : {} } + onSourceChanged: { + thumbnail.source = ThumbnailCache.thumbnail(root.source) + } + + Connections { + target: ThumbnailCache + function onThumbnailCreated(imgSource) { + if (imgSource == root.source) { + thumbnail.source = ThumbnailCache.thumbnail(root.source) + } + } + } + MouseArea { id: imageMA anchors.fill: parent @@ -77,9 +90,9 @@ Item { border.color: isCurrentItem ? imageLabel.palette.highlight : Qt.darker(imageLabel.palette.highlight) border.width: imageMA.containsMouse || root.isCurrentItem ? 2 : 0 Image { + id: thumbnail anchors.fill: parent anchors.margins: 4 - source: ThumbnailCache.thumbnail(root.source) asynchronous: true autoTransform: true fillMode: Image.PreserveAspectFit From af2c9c0fd36ef7b7a56f3a1346c51a1a0afdf320 Mon Sep 17 00:00:00 2001 From: mugulmd Date: Mon, 9 Jan 2023 06:21:10 -0800 Subject: [PATCH 09/29] [ui] dispatch thumbnail creation signal from grid --- meshroom/ui/qml/ImageGallery/ImageDelegate.qml | 16 ++++++---------- meshroom/ui/qml/ImageGallery/ImageGallery.qml | 13 +++++++++++++ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml index baf75847c9..62f936800f 100644 --- a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml +++ b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml @@ -31,17 +31,13 @@ Item { property var metadata: metadataStr ? JSON.parse(viewpoint.get("metadata").value) : {} } - onSourceChanged: { - thumbnail.source = ThumbnailCache.thumbnail(root.source) + // update thumbnail location + // can be called from the GridView when a new thumbnail has been written on disk + function updateThumbnail() { + thumbnail.source = ThumbnailCache.thumbnail(root.source); } - - Connections { - target: ThumbnailCache - function onThumbnailCreated(imgSource) { - if (imgSource == root.source) { - thumbnail.source = ThumbnailCache.thumbnail(root.source) - } - } + onSourceChanged: { + updateThumbnail(); } MouseArea { diff --git a/meshroom/ui/qml/ImageGallery/ImageGallery.qml b/meshroom/ui/qml/ImageGallery/ImageGallery.qml index c89afcce1a..63402980c5 100644 --- a/meshroom/ui/qml/ImageGallery/ImageGallery.qml +++ b/meshroom/ui/qml/ImageGallery/ImageGallery.qml @@ -201,6 +201,19 @@ Panel { } } + // Update grid item when corresponding thumbnail is computed + Connections { + target: ThumbnailCache + function onThumbnailCreated(imgSource) { + for (let i = 0; i < grid.count; i++) { + let item = grid.itemAtIndex(i); // item is an ImageDelegate + if (item && item.source == imgSource) { + item.updateThumbnail(); + } + } + } + } + model: SortFilterDelegateModel { id: sortedModel model: m.viewpoints From 88c85e9496a4788673f93e22abae5e36f1e5a4b9 Mon Sep 17 00:00:00 2001 From: mugulmd Date: Tue, 10 Jan 2023 00:31:50 -0800 Subject: [PATCH 10/29] [ui] thumbnail busy indicator when loading --- meshroom/ui/qml/ImageGallery/ImageDelegate.qml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml index 62f936800f..d174004a97 100644 --- a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml +++ b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml @@ -93,6 +93,10 @@ Item { autoTransform: true fillMode: Image.PreserveAspectFit } + BusyIndicator { + anchors.centerIn: parent + running: !Filepath.exists(thumbnail.source) + } } // Image basename Label { From cd955893e3faa46ba0d31375ad81cfef6b30277b Mon Sep 17 00:00:00 2001 From: mugulmd Date: Tue, 10 Jan 2023 02:06:00 -0800 Subject: [PATCH 11/29] [ui] ImageGallery: default value for grid view cache buffer --- meshroom/ui/qml/ImageGallery/ImageGallery.qml | 1 - 1 file changed, 1 deletion(-) diff --git a/meshroom/ui/qml/ImageGallery/ImageGallery.qml b/meshroom/ui/qml/ImageGallery/ImageGallery.qml index 63402980c5..ac519c91b5 100644 --- a/meshroom/ui/qml/ImageGallery/ImageGallery.qml +++ b/meshroom/ui/qml/ImageGallery/ImageGallery.qml @@ -157,7 +157,6 @@ Panel { Layout.fillWidth: true Layout.fillHeight: true - cacheBuffer: 10000 // Magic number that seems to work well, even with lots of images visible: !intrinsicsFilterButton.checked From 268f60834411a2f602d6188971448d56dea010ce Mon Sep 17 00:00:00 2001 From: mugulmd Date: Mon, 16 Jan 2023 02:52:01 -0800 Subject: [PATCH 12/29] [ui] ThumbnailCache: use QStandardPath for default cache dir --- meshroom/ui/components/thumbnail.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index 5f16c2f66f..2ea542b315 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -1,6 +1,6 @@ from meshroom.common import Signal -from PySide2.QtCore import QObject, Slot, QSize, QUrl, Qt +from PySide2.QtCore import QObject, Slot, QSize, QUrl, Qt, QStandardPaths from PySide2.QtGui import QImageReader, QImageWriter import os @@ -19,9 +19,7 @@ class ThumbnailCache(QObject): Since creating thumbnails can be long (as it requires to read the full image from disk) it is performed asynchronously to avoid blocking the main thread. - The default cache location is a subdirectory of the user's home directory: - ~/Meshroom/thumbnails. - This location can be overriden with the MESHROOM_THUMBNAIL_DIR environment variable. + The default cache location can be overriden with the MESHROOM_THUMBNAIL_DIR environment variable. This class also takes care of cleaning the thumbnail directory, i.e. scanning this directory and removing thumbnails that have not been used for too long. @@ -38,7 +36,7 @@ class ThumbnailCache(QObject): """ # Thumbnail cache directory - thumbnailDir = os.path.join(Path.home(), 'Meshroom', 'thumbnails') + thumbnailDir = os.path.join(QStandardPaths.writableLocation(QStandardPaths.CacheLocation), 'Meshroom', 'thumbnails') # Thumbnail dimensions limit (the actual dimensions of a thumbnail will depend on the aspect ratio) thumbnailSize = QSize(100, 100) From dfb120c6d73d7bdceffe6e02aeeebd8a40013d8f Mon Sep 17 00:00:00 2001 From: mugulmd Date: Mon, 16 Jan 2023 02:57:13 -0800 Subject: [PATCH 13/29] [ui] ThumbnailCache: limit number of worker threads to 3 --- meshroom/ui/components/thumbnail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index 2ea542b315..a2ca2baeaf 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -52,7 +52,7 @@ class ThumbnailCache(QObject): thumbnailCreated = Signal(QUrl) # Thread pool for running createThumbnail asynchronously on a fixed number of worker threads - pool = ThreadPool() + pool = ThreadPool(processes=3) @staticmethod def initialize(): From a2e6a6881182c7b4904a30fdcc3320054b29b1d3 Mon Sep 17 00:00:00 2001 From: mugulmd Date: Mon, 16 Jan 2023 03:17:56 -0800 Subject: [PATCH 14/29] [ui] ImageDelegate: check URL is not empty instead of OS call --- meshroom/ui/qml/ImageGallery/ImageDelegate.qml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml index d174004a97..ac77066907 100644 --- a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml +++ b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml @@ -95,7 +95,7 @@ Item { } BusyIndicator { anchors.centerIn: parent - running: !Filepath.exists(thumbnail.source) + running: Filepath.urlToString(thumbnail.source).length === 0 } } // Image basename From 7abbf1bbf38c1e280a0a2252c44fb1910398ba1b Mon Sep 17 00:00:00 2001 From: mugulmd Date: Mon, 16 Jan 2023 03:43:26 -0800 Subject: [PATCH 15/29] [ui] ThumbnailCache: reduce number of system calls --- meshroom/ui/components/thumbnail.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index a2ca2baeaf..d5b1fcd1fe 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -5,6 +5,7 @@ import os from pathlib import Path +import stat import hashlib import time import logging @@ -173,22 +174,28 @@ def clean(): # Scan thumbnail directory and gather all thumbnails to remove toRemove = [] remaining = [] - for entry in os.scandir(ThumbnailCache.thumbnailDir): - if not entry.is_file(): + for f_name in os.listdir(ThumbnailCache.thumbnailDir): + pathname = os.path.join(ThumbnailCache.thumbnailDir, f_name) + + # System call to get current item info + f_stat = os.stat(pathname, follow_symlinks=False) + + # Check if this is a regular file + if not stat.S_ISREG(f_stat.st_mode): continue # Compute storage duration since last usage of thumbnail - lastUsage = os.path.getmtime(entry.path) + lastUsage = f_stat.st_mtime storageTime = now - lastUsage - logging.debug(f'[ThumbnailCache] Thumbnail {entry.name} has been stored for {storageTime}s') + logging.debug(f'[ThumbnailCache] Thumbnail {f_name} has been stored for {storageTime}s') if storageTime > ThumbnailCache.storageTimeLimit * 3600 * 24: # Mark as removable if storage time exceeds limit - logging.debug(f'[ThumbnailCache] {entry.name} exceeded storage time limit') - toRemove.append(entry.path) + logging.debug(f'[ThumbnailCache] {f_name} exceeded storage time limit') + toRemove.append(pathname) else: # Store path and last usage time for potentially sorting and removing later - remaining.append((entry.path, lastUsage)) + remaining.append((pathname, lastUsage)) # Remove all thumbnails marked as removable for path in toRemove: From 855113551490c83dbf93ce8d2dc9b6ef2fc9f6a3 Mon Sep 17 00:00:00 2001 From: mugulmd Date: Mon, 16 Jan 2023 04:57:56 -0800 Subject: [PATCH 16/29] [ui] ThumbnailCache: catch FileNotFoundError when removing thumbnails --- meshroom/ui/components/thumbnail.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index d5b1fcd1fe..db918bec7a 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -200,7 +200,10 @@ def clean(): # Remove all thumbnails marked as removable for path in toRemove: logging.debug(f'[ThumbnailCache] Remove {path}') - os.remove(path) + try: + os.remove(path) + except FileNotFoundError: + logging.error(f'[ThumbnailCache] Tried to remove {path} but this file does not exist') # Check if number of thumbnails on disk exceeds limit if len(remaining) > ThumbnailCache.maxThumbnailsOnDisk: @@ -212,4 +215,7 @@ def clean(): for i in range(nbToRemove): path = remaining[i][0] logging.debug(f'[ThumbnailCache] Remove {path}') - os.remove(path) + try: + os.remove(path) + except FileNotFoundError: + logging.error(f'[ThumbnailCache] Tried to remove {path} but this file does not exist') From 80174e22af1e6e170327620f394fc3d93982600d Mon Sep 17 00:00:00 2001 From: mugulmd Date: Mon, 16 Jan 2023 06:21:48 -0800 Subject: [PATCH 17/29] [ui] ImageGallery: faster thumbnail dispatch using caller ID --- meshroom/ui/components/thumbnail.py | 20 +++++++++++-------- .../ui/qml/ImageGallery/ImageDelegate.qml | 3 ++- meshroom/ui/qml/ImageGallery/ImageGallery.qml | 11 +++++----- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index db918bec7a..c646df9cd3 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -49,8 +49,10 @@ class ThumbnailCache(QObject): maxThumbnailsOnDisk = 100000 # Signal to notify listeners that a thumbnail was created and written on disk - # This signal has one argument: the url of the image that the thumbnail is associated to - thumbnailCreated = Signal(QUrl) + # This signal has two argument: + # - the url of the image that the thumbnail is associated to + # - an identifier for the caller, e.g. the component that sent the request (useful for faster dispatch in QML) + thumbnailCreated = Signal(QUrl, int) # Thread pool for running createThumbnail asynchronously on a fixed number of worker threads pool = ThreadPool(processes=3) @@ -93,15 +95,16 @@ def thumbnailPath(imgPath): path = os.path.join(ThumbnailCache.thumbnailDir, f'{digest}.jpg') return path - @Slot(QUrl, result=QUrl) - def thumbnail(self, imgSource): + @Slot(QUrl, int, result=QUrl) + def thumbnail(self, imgSource, callerID): """Retrieve the filepath of the thumbnail corresponding to a given image. If the thumbnail does not exist on disk, it will be created asynchronously. - When this is done, the createdThumbnail signal is emitted. + When this is done, the thumbnailCreated signal is emitted. Args: imgSource (QUrl): location of the input image + callerID (int): identifier for the object that requested the thumbnail Returns: QUrl: location of the corresponding thumbnail if it exists, otherwise None @@ -121,14 +124,15 @@ def thumbnail(self, imgSource): # Thumbnail does not exist # create it in a worker thread to avoid UI freeze - ThumbnailCache.pool.apply_async(self.createThumbnail, args=(imgSource,)) + ThumbnailCache.pool.apply_async(self.createThumbnail, args=(imgSource,callerID,)) return None - def createThumbnail(self, imgSource): + def createThumbnail(self, imgSource, callerID): """Load an image, resize it to thumbnail dimensions and save the result in the cache directory. Args: imgSource (QUrl): location of the input image + callerID (int): identifier for the object that requested the thumbnail """ imgPath = imgSource.toLocalFile() path = ThumbnailCache.thumbnailPath(imgPath) @@ -155,7 +159,7 @@ def createThumbnail(self, imgSource): logging.error(f'[ThumbnailCache] Error when writing thumbnail: {writer.errorString()}') # Notify listeners - self.thumbnailCreated.emit(imgSource) + self.thumbnailCreated.emit(imgSource, callerID) @staticmethod def clean(): diff --git a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml index ac77066907..734b8f71d4 100644 --- a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml +++ b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml @@ -11,6 +11,7 @@ Item { id: root property variant viewpoint + property int cellID property bool isCurrentItem: false property alias source: _viewpoint.source property alias metadata: _viewpoint.metadata @@ -34,7 +35,7 @@ Item { // update thumbnail location // can be called from the GridView when a new thumbnail has been written on disk function updateThumbnail() { - thumbnail.source = ThumbnailCache.thumbnail(root.source); + thumbnail.source = ThumbnailCache.thumbnail(root.source, cellID); } onSourceChanged: { updateThumbnail(); diff --git a/meshroom/ui/qml/ImageGallery/ImageGallery.qml b/meshroom/ui/qml/ImageGallery/ImageGallery.qml index ac519c91b5..8ad0da94e4 100644 --- a/meshroom/ui/qml/ImageGallery/ImageGallery.qml +++ b/meshroom/ui/qml/ImageGallery/ImageGallery.qml @@ -203,12 +203,10 @@ Panel { // Update grid item when corresponding thumbnail is computed Connections { target: ThumbnailCache - function onThumbnailCreated(imgSource) { - for (let i = 0; i < grid.count; i++) { - let item = grid.itemAtIndex(i); // item is an ImageDelegate - if (item && item.source == imgSource) { - item.updateThumbnail(); - } + function onThumbnailCreated(imgSource, callerID) { + let item = grid.itemAtIndex(callerID); // item is an ImageDelegate + if (item && item.source == imgSource) { + item.updateThumbnail(); } } } @@ -257,6 +255,7 @@ Panel { id: imageDelegate viewpoint: object.value + cellID: DelegateModel.filteredIndex width: grid.cellWidth height: grid.cellHeight readOnly: m.readOnly From 7e371edc6168ff6dc4ac5a1f62b457f56bcde1d8 Mon Sep 17 00:00:00 2001 From: mugulmd Date: Mon, 16 Jan 2023 06:45:59 -0800 Subject: [PATCH 18/29] [ui] ThumbnailCache: resolve default cache dir in initialize method --- meshroom/ui/components/thumbnail.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index c646df9cd3..f0c4983da8 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -37,7 +37,8 @@ class ThumbnailCache(QObject): """ # Thumbnail cache directory - thumbnailDir = os.path.join(QStandardPaths.writableLocation(QStandardPaths.CacheLocation), 'Meshroom', 'thumbnails') + # Cannot be initialized here as it depends on the organization and application names + thumbnailDir = '' # Thumbnail dimensions limit (the actual dimensions of a thumbnail will depend on the aspect ratio) thumbnailSize = QSize(100, 100) @@ -60,10 +61,13 @@ class ThumbnailCache(QObject): @staticmethod def initialize(): """Initialize static fields in cache class and cache directory on disk.""" - # User specified thumbnail directory + # Thumbnail directory: default or user specified dir = os.getenv('MESHROOM_THUMBNAIL_DIR') if dir is not None: ThumbnailCache.thumbnailDir = dir + else: + ThumbnailCache.thumbnailDir = os.path.join(QStandardPaths.writableLocation(QStandardPaths.CacheLocation), + 'thumbnails') # User specifed time limit for thumbnails on disk (expressed in days) timeLimit = os.getenv('MESHROOM_THUMBNAIL_TIME_LIMIT') From 0926b20d435b9a5f4da75b1922c2e2f7761d1d51 Mon Sep 17 00:00:00 2001 From: mugulmd Date: Tue, 17 Jan 2023 02:42:35 -0800 Subject: [PATCH 19/29] [ui] fallback for dispatching thumbnails in case cellID changed --- meshroom/ui/qml/ImageGallery/ImageGallery.qml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/meshroom/ui/qml/ImageGallery/ImageGallery.qml b/meshroom/ui/qml/ImageGallery/ImageGallery.qml index 8ad0da94e4..1ded88b08c 100644 --- a/meshroom/ui/qml/ImageGallery/ImageGallery.qml +++ b/meshroom/ui/qml/ImageGallery/ImageGallery.qml @@ -207,6 +207,14 @@ Panel { let item = grid.itemAtIndex(callerID); // item is an ImageDelegate if (item && item.source == imgSource) { item.updateThumbnail(); + return; + } + // fallback in case the ImageDelegate cellID changed + for (let idx = 0; idx < grid.count; idx++) { + item = grid.itemAtIndex(idx); + if (item && item.source == imgSource) { + item.updateThumbnail(); + } } } } From 374b9abb770a2b4f01e8e7541b6412ae6e94501a Mon Sep 17 00:00:00 2001 From: mugulmd Date: Tue, 17 Jan 2023 02:53:03 -0800 Subject: [PATCH 20/29] [ui] ThumbnailCache: replace thread pool with custom thread spawning This is so we can control in which order the requests are handled, as we want LIFO ordering instead of FIFO for smoother interaction. --- meshroom/ui/components/thumbnail.py | 156 ++++++++++++++++------------ 1 file changed, 90 insertions(+), 66 deletions(-) diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index f0c4983da8..4575d3c258 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -9,7 +9,7 @@ import hashlib import time import logging -from multiprocessing.pool import ThreadPool +from threading import Thread class ThumbnailCache(QObject): @@ -55,8 +55,10 @@ class ThumbnailCache(QObject): # - an identifier for the caller, e.g. the component that sent the request (useful for faster dispatch in QML) thumbnailCreated = Signal(QUrl, int) - # Thread pool for running createThumbnail asynchronously on a fixed number of worker threads - pool = ThreadPool(processes=3) + # Threads info and LIFO structure for running createThumbnail asynchronously + maxWorkerThreads = 3 + activeWorkerThreads = 0 + requests = [] @staticmethod def initialize(): @@ -85,6 +87,70 @@ def initialize(): # Make sure the thumbnail directory exists before writing into it os.makedirs(ThumbnailCache.thumbnailDir, exist_ok=True) + @staticmethod + def clean(): + """Scan the thumbnail directory and: + 1. remove all thumbnails that have not been used for more than storageTimeLimit days + 2. ensure that the number of thumbnails on disk does not exceed maxThumbnailsOnDisk. + """ + # Check if thumbnail directory exists + if not os.path.exists(ThumbnailCache.thumbnailDir): + logging.debug('[ThumbnailCache] Thumbnail directory does not exist yet.') + return + + # Get current time + now = time.time() + + # Scan thumbnail directory and gather all thumbnails to remove + toRemove = [] + remaining = [] + for f_name in os.listdir(ThumbnailCache.thumbnailDir): + pathname = os.path.join(ThumbnailCache.thumbnailDir, f_name) + + # System call to get current item info + f_stat = os.stat(pathname, follow_symlinks=False) + + # Check if this is a regular file + if not stat.S_ISREG(f_stat.st_mode): + continue + + # Compute storage duration since last usage of thumbnail + lastUsage = f_stat.st_mtime + storageTime = now - lastUsage + logging.debug(f'[ThumbnailCache] Thumbnail {f_name} has been stored for {storageTime}s') + + if storageTime > ThumbnailCache.storageTimeLimit * 3600 * 24: + # Mark as removable if storage time exceeds limit + logging.debug(f'[ThumbnailCache] {f_name} exceeded storage time limit') + toRemove.append(pathname) + else: + # Store path and last usage time for potentially sorting and removing later + remaining.append((pathname, lastUsage)) + + # Remove all thumbnails marked as removable + for path in toRemove: + logging.debug(f'[ThumbnailCache] Remove {path}') + try: + os.remove(path) + except FileNotFoundError: + logging.error(f'[ThumbnailCache] Tried to remove {path} but this file does not exist') + + # Check if number of thumbnails on disk exceeds limit + if len(remaining) > ThumbnailCache.maxThumbnailsOnDisk: + nbToRemove = len(remaining) - ThumbnailCache.maxThumbnailsOnDisk + logging.debug( + f'[ThumbnailCache] Too many thumbnails: {len(remaining)} remaining, {nbToRemove} will be removed') + + # Sort by last usage order and remove excess + remaining.sort(key=lambda elt: elt[1]) + for i in range(nbToRemove): + path = remaining[i][0] + logging.debug(f'[ThumbnailCache] Remove {path}') + try: + os.remove(path) + except FileNotFoundError: + logging.error(f'[ThumbnailCache] Tried to remove {path} but this file does not exist') + @staticmethod def thumbnailPath(imgPath): """Use SHA1 hashing to associate a unique thumbnail to an image. @@ -127,8 +193,12 @@ def thumbnail(self, imgSource, callerID): return source # Thumbnail does not exist - # create it in a worker thread to avoid UI freeze - ThumbnailCache.pool.apply_async(self.createThumbnail, args=(imgSource,callerID,)) + # Create request and start a thread if needed + ThumbnailCache.requests.append((imgSource, callerID)) + if ThumbnailCache.activeWorkerThreads < ThumbnailCache.maxWorkerThreads: + thread = Thread(target=self.handleRequestsAsync) + thread.start() + return None def createThumbnail(self, imgSource, callerID): @@ -165,65 +235,19 @@ def createThumbnail(self, imgSource, callerID): # Notify listeners self.thumbnailCreated.emit(imgSource, callerID) - @staticmethod - def clean(): - """Scan the thumbnail directory and: - 1. remove all thumbnails that have not been used for more than storageTimeLimit days - 2. ensure that the number of thumbnails on disk does not exceed maxThumbnailsOnDisk. - """ - # Check if thumbnail directory exists - if not os.path.exists(ThumbnailCache.thumbnailDir): - logging.debug('[ThumbnailCache] Thumbnail directory does not exist yet.') - return - - # Get current time - now = time.time() - - # Scan thumbnail directory and gather all thumbnails to remove - toRemove = [] - remaining = [] - for f_name in os.listdir(ThumbnailCache.thumbnailDir): - pathname = os.path.join(ThumbnailCache.thumbnailDir, f_name) - - # System call to get current item info - f_stat = os.stat(pathname, follow_symlinks=False) - - # Check if this is a regular file - if not stat.S_ISREG(f_stat.st_mode): - continue - - # Compute storage duration since last usage of thumbnail - lastUsage = f_stat.st_mtime - storageTime = now - lastUsage - logging.debug(f'[ThumbnailCache] Thumbnail {f_name} has been stored for {storageTime}s') - - if storageTime > ThumbnailCache.storageTimeLimit * 3600 * 24: - # Mark as removable if storage time exceeds limit - logging.debug(f'[ThumbnailCache] {f_name} exceeded storage time limit') - toRemove.append(pathname) - else: - # Store path and last usage time for potentially sorting and removing later - remaining.append((pathname, lastUsage)) + def handleRequestsAsync(self): + """Process thumbnail creation requests in LIFO order. - # Remove all thumbnails marked as removable - for path in toRemove: - logging.debug(f'[ThumbnailCache] Remove {path}') - try: - os.remove(path) - except FileNotFoundError: - logging.error(f'[ThumbnailCache] Tried to remove {path} but this file does not exist') - - # Check if number of thumbnails on disk exceeds limit - if len(remaining) > ThumbnailCache.maxThumbnailsOnDisk: - nbToRemove = len(remaining) - ThumbnailCache.maxThumbnailsOnDisk - logging.debug(f'[ThumbnailCache] Too many thumbnails: {len(remaining)} remaining, {nbToRemove} will be removed') - - # Sort by last usage order and remove excess - remaining.sort(key=lambda elt: elt[1]) - for i in range(nbToRemove): - path = remaining[i][0] - logging.debug(f'[ThumbnailCache] Remove {path}') - try: - os.remove(path) - except FileNotFoundError: - logging.error(f'[ThumbnailCache] Tried to remove {path} but this file does not exist') + This method is only meant to be called by worker threads, + hence it also takes care of registering/unregistering the calling thread. + """ + # Register worker thread + ThumbnailCache.activeWorkerThreads += 1 + try: + while True: + req = ThumbnailCache.requests.pop() + self.createThumbnail(req[0], req[1]) + except IndexError: + # No more request to process + # Unregister worker thread + ThumbnailCache.activeWorkerThreads -= 1 From 3f96778f2aaa66a59b7b46ae4aee3569e7b53acb Mon Sep 17 00:00:00 2001 From: mugulmd Date: Tue, 17 Jan 2023 03:09:39 -0800 Subject: [PATCH 21/29] [ui] ThumbnailCache: check if thumbnail has been created by another thread --- meshroom/ui/components/thumbnail.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index 4575d3c258..c809f01dc5 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -165,6 +165,22 @@ def thumbnailPath(imgPath): path = os.path.join(ThumbnailCache.thumbnailDir, f'{digest}.jpg') return path + @staticmethod + def checkThumbnail(path): + """Check if a thumbnail already exists on disk, and if so update its last modification time. + + Args: + path (str): filepath to the thumbnail + + Returns: + (bool): whether the thumbnail exists on disk or not + """ + if os.path.exists(path): + # Update last modification time + Path(path).touch(exist_ok=True) + return True + return False + @Slot(QUrl, int, result=QUrl) def thumbnail(self, imgSource, callerID): """Retrieve the filepath of the thumbnail corresponding to a given image. @@ -187,9 +203,7 @@ def thumbnail(self, imgSource, callerID): source = QUrl.fromLocalFile(path) # Check if thumbnail already exists - if os.path.exists(path): - # Update last modification time - Path(path).touch(exist_ok=True) + if ThumbnailCache.checkThumbnail(path): return source # Thumbnail does not exist @@ -210,6 +224,12 @@ def createThumbnail(self, imgSource, callerID): """ imgPath = imgSource.toLocalFile() path = ThumbnailCache.thumbnailPath(imgPath) + + # Check if thumbnail already exists (it may have been created by another thread) + if ThumbnailCache.checkThumbnail(path): + self.thumbnailCreated.emit(imgSource, callerID) + return + logging.debug(f'[ThumbnailCache] Creating thumbnail {path} for image {imgPath}') # Initialize image reader object From ae3184deccb8c4ab9da14e4818b95ec8adaa7b32 Mon Sep 17 00:00:00 2001 From: mugulmd Date: Tue, 17 Jan 2023 07:00:17 -0800 Subject: [PATCH 22/29] [ui] send thumbnail request with timer to avoid deadlocks --- .../ui/qml/ImageGallery/ImageDelegate.qml | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml index 734b8f71d4..ee7e33c164 100644 --- a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml +++ b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml @@ -11,7 +11,7 @@ Item { id: root property variant viewpoint - property int cellID + property int cellID: -1 property bool isCurrentItem: false property alias source: _viewpoint.source property alias metadata: _viewpoint.metadata @@ -35,12 +35,28 @@ Item { // update thumbnail location // can be called from the GridView when a new thumbnail has been written on disk function updateThumbnail() { - thumbnail.source = ThumbnailCache.thumbnail(root.source, cellID); + thumbnail.source = ThumbnailCache.thumbnail(root.source, root.cellID); } onSourceChanged: { updateThumbnail(); } + // Send a new request every 2 seconds until thumbnail is loaded + // This is meant to avoid deadlocks in case there is a synchronization problem + Timer { + interval: 2000 + repeat: true + running: true + onTriggered: { + if (thumbnail.status == Image.Null) { + updateThumbnail(); + } + else { + running = false; + } + } + } + MouseArea { id: imageMA anchors.fill: parent @@ -96,7 +112,7 @@ Item { } BusyIndicator { anchors.centerIn: parent - running: Filepath.urlToString(thumbnail.source).length === 0 + running: thumbnail.status != Image.Ready } } // Image basename From dd235dcbf18d1be5b6b25a2b55a7752c6631fba7 Mon Sep 17 00:00:00 2001 From: mugulmd Date: Tue, 17 Jan 2023 07:11:32 -0800 Subject: [PATCH 23/29] [ui] clear thumbnail requests when changing viewpoints --- meshroom/ui/components/thumbnail.py | 8 ++++++++ meshroom/ui/qml/ImageGallery/ImageGallery.qml | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index c809f01dc5..e9ec5fe646 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -271,3 +271,11 @@ def handleRequestsAsync(self): # No more request to process # Unregister worker thread ThumbnailCache.activeWorkerThreads -= 1 + + @Slot() + def clearRequests(self): + """Clear all pending thumbnail creation requests. + + Requests already under treatment by a worker thread will still be completed. + """ + ThumbnailCache.requests.clear() diff --git a/meshroom/ui/qml/ImageGallery/ImageGallery.qml b/meshroom/ui/qml/ImageGallery/ImageGallery.qml index 1ded88b08c..d710375be3 100644 --- a/meshroom/ui/qml/ImageGallery/ImageGallery.qml +++ b/meshroom/ui/qml/ImageGallery/ImageGallery.qml @@ -39,6 +39,10 @@ Panel { property variant viewpoints: currentCameraInit ? currentCameraInit.attribute('viewpoints').value : undefined property variant intrinsics: currentCameraInit ? currentCameraInit.attribute('intrinsics').value : undefined property bool readOnly: root.readOnly || displayHDR.checked + + onViewpointsChanged: { + ThumbnailCache.clearRequests(); + } } property variant parsedIntrinsic From 2f72f9e7d1ecfaef56c9f015febcf93d31ce6afa Mon Sep 17 00:00:00 2001 From: mugulmd Date: Mon, 23 Jan 2023 06:28:30 -0800 Subject: [PATCH 24/29] [ui] ImageDelegate: increase thumbnail timer to 5s --- meshroom/ui/qml/ImageGallery/ImageDelegate.qml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml index ee7e33c164..7c7a74028d 100644 --- a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml +++ b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml @@ -41,10 +41,10 @@ Item { updateThumbnail(); } - // Send a new request every 2 seconds until thumbnail is loaded + // Send a new request every 5 seconds until thumbnail is loaded // This is meant to avoid deadlocks in case there is a synchronization problem Timer { - interval: 2000 + interval: 5000 repeat: true running: true onTriggered: { From c48233141a8d272ef0a2777b1428861979ccadd5 Mon Sep 17 00:00:00 2001 From: mugulmd Date: Mon, 23 Jan 2023 06:29:15 -0800 Subject: [PATCH 25/29] [ui] ThumbnailCache: asynchronous cache cleaning --- meshroom/ui/components/thumbnail.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index e9ec5fe646..6826837c6a 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -26,6 +26,7 @@ class ThumbnailCache(QObject): i.e. scanning this directory and removing thumbnails that have not been used for too long. This operation also ensures that the number of thumbnails on disk does not exceed a certain limit, by removing thumbnails if necessary (from least recently used to most recently used). + Since this operation is done at application startup, it is also performed asynchronously. The default time limit is 90 days, and can be overriden with the MESHROOM_THUMBNAIL_TIME_LIMIT environment variable. @@ -55,10 +56,11 @@ class ThumbnailCache(QObject): # - an identifier for the caller, e.g. the component that sent the request (useful for faster dispatch in QML) thumbnailCreated = Signal(QUrl, int) - # Threads info and LIFO structure for running createThumbnail asynchronously + # Threads info and LIFO structure for running clean and createThumbnail asynchronously maxWorkerThreads = 3 activeWorkerThreads = 0 requests = [] + cleaningThread = None @staticmethod def initialize(): @@ -82,7 +84,9 @@ def initialize(): ThumbnailCache.maxThumbnailsOnDisk = int(maxOnDisk) # Clean thumbnail directory - ThumbnailCache.clean() + # This is performed asynchronously to avoid freezing the app at startup + ThumbnailCache.cleaningThread = Thread(target=ThumbnailCache.clean) + ThumbnailCache.cleaningThread.start() # Make sure the thumbnail directory exists before writing into it os.makedirs(ThumbnailCache.thumbnailDir, exist_ok=True) @@ -260,9 +264,18 @@ def handleRequestsAsync(self): This method is only meant to be called by worker threads, hence it also takes care of registering/unregistering the calling thread. + + Note: this operation waits for the cleaning process to finish before starting, + in order to avoid synchronization issues. """ # Register worker thread ThumbnailCache.activeWorkerThreads += 1 + + # Wait for cleaning thread to finish + if ThumbnailCache.cleaningThread is not None and ThumbnailCache.cleaningThread.is_alive(): + ThumbnailCache.cleaningThread.join() + + # Handle requests until the requests stack is empty try: while True: req = ThumbnailCache.requests.pop() From 9b0be36278af10f7dd7d7095fecb65c1ea3aa1c6 Mon Sep 17 00:00:00 2001 From: mugulmd Date: Thu, 26 Jan 2023 05:10:35 -0800 Subject: [PATCH 26/29] [ui] improve thumbnails quality --- meshroom/ui/components/thumbnail.py | 6 ++++-- meshroom/ui/qml/ImageGallery/ImageDelegate.qml | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index 6826837c6a..186dd9b114 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -42,7 +42,7 @@ class ThumbnailCache(QObject): thumbnailDir = '' # Thumbnail dimensions limit (the actual dimensions of a thumbnail will depend on the aspect ratio) - thumbnailSize = QSize(100, 100) + thumbnailSize = QSize(256, 256) # Time limit for thumbnail storage on disk, expressed in days storageTimeLimit = 90 @@ -248,7 +248,9 @@ def createThumbnail(self, imgSource, callerID): return # Scale image while preserving aspect ratio - thumbnail = img.scaled(ThumbnailCache.thumbnailSize, aspectMode=Qt.KeepAspectRatio) + thumbnail = img.scaled(ThumbnailCache.thumbnailSize, + aspectMode=Qt.KeepAspectRatio, + mode=Qt.SmoothTransformation) # Write thumbnail to disk and check for potential errors writer = QImageWriter(path) diff --git a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml index 7c7a74028d..2cd5cc449a 100644 --- a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml +++ b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml @@ -109,6 +109,7 @@ Item { asynchronous: true autoTransform: true fillMode: Image.PreserveAspectFit + sourceSize: Qt.size(100, 100) } BusyIndicator { anchors.centerIn: parent From db8b3873e3957f8c3ec7ab1bbbce3b3871032c1f Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Wed, 8 Feb 2023 18:20:54 +0100 Subject: [PATCH 27/29] [ui] ImageGallery: do not reduce thumbnail resolution As we rely on the thumbnails cache, the images are already downscaled at a small resolution. So there is no more need to force a max resolution on the Qml Image. --- meshroom/ui/qml/ImageGallery/ImageDelegate.qml | 1 - 1 file changed, 1 deletion(-) diff --git a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml index 2cd5cc449a..7c7a74028d 100644 --- a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml +++ b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml @@ -109,7 +109,6 @@ Item { asynchronous: true autoTransform: true fillMode: Image.PreserveAspectFit - sourceSize: Qt.size(100, 100) } BusyIndicator { anchors.centerIn: parent From db8d00e901ac8e3a5718c3610269551c64b1cc86 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Wed, 8 Feb 2023 21:50:04 +0100 Subject: [PATCH 28/29] [ui] ImageGallery thumbnails: no need for caching and expensive smooth resizing --- meshroom/ui/qml/ImageGallery/ImageDelegate.qml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml index 7c7a74028d..27c34eaa8b 100644 --- a/meshroom/ui/qml/ImageGallery/ImageDelegate.qml +++ b/meshroom/ui/qml/ImageGallery/ImageDelegate.qml @@ -109,6 +109,8 @@ Item { asynchronous: true autoTransform: true fillMode: Image.PreserveAspectFit + smooth: false + cache: false } BusyIndicator { anchors.centerIn: parent From c758bf043bad81577ba0064f436b634de88619e3 Mon Sep 17 00:00:00 2001 From: mugulmd Date: Thu, 9 Feb 2023 01:50:38 -0800 Subject: [PATCH 29/29] [ui] thumbnail cache: use thread pool to manage thread lifetime automatically --- meshroom/ui/components/thumbnail.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index 186dd9b114..bcf9194bb3 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -10,6 +10,7 @@ import time import logging from threading import Thread +from multiprocessing.pool import ThreadPool class ThumbnailCache(QObject): @@ -57,10 +58,9 @@ class ThumbnailCache(QObject): thumbnailCreated = Signal(QUrl, int) # Threads info and LIFO structure for running clean and createThumbnail asynchronously - maxWorkerThreads = 3 - activeWorkerThreads = 0 requests = [] cleaningThread = None + workerThreads = ThreadPool(processes=3) @staticmethod def initialize(): @@ -211,11 +211,9 @@ def thumbnail(self, imgSource, callerID): return source # Thumbnail does not exist - # Create request and start a thread if needed + # Create request and submit to worker threads ThumbnailCache.requests.append((imgSource, callerID)) - if ThumbnailCache.activeWorkerThreads < ThumbnailCache.maxWorkerThreads: - thread = Thread(target=self.handleRequestsAsync) - thread.start() + ThumbnailCache.workerThreads.apply_async(func=self.handleRequestsAsync) return None @@ -264,15 +262,9 @@ def createThumbnail(self, imgSource, callerID): def handleRequestsAsync(self): """Process thumbnail creation requests in LIFO order. - This method is only meant to be called by worker threads, - hence it also takes care of registering/unregistering the calling thread. - Note: this operation waits for the cleaning process to finish before starting, in order to avoid synchronization issues. """ - # Register worker thread - ThumbnailCache.activeWorkerThreads += 1 - # Wait for cleaning thread to finish if ThumbnailCache.cleaningThread is not None and ThumbnailCache.cleaningThread.is_alive(): ThumbnailCache.cleaningThread.join() @@ -284,8 +276,7 @@ def handleRequestsAsync(self): self.createThumbnail(req[0], req[1]) except IndexError: # No more request to process - # Unregister worker thread - ThumbnailCache.activeWorkerThreads -= 1 + return @Slot() def clearRequests(self):