From 5e6c74445ef6da7cc1239bb081c5cb9855b721b6 Mon Sep 17 00:00:00 2001 From: Lucas Cimon <925560+Lucas-C@users.noreply.github.com> Date: Mon, 13 Mar 2023 20:39:14 +0100 Subject: [PATCH] More memory-tracking tools --- .../continuous-integration-workflow.yml | 2 +- fpdf/__init__.py | 2 +- fpdf/image_parsing.py | 301 +++--------------- fpdf/util.py | 140 +++++++- test/conftest.py | 59 +++- test/embed_file_all_optionals.pdf | Bin 1307 -> 1301 bytes test/embed_file_self.pdf | Bin 1148 -> 1140 bytes test/file_attachment_annotation.pdf | Bin 1321 -> 1313 bytes test/image/image_types/test_insert_images.py | 10 +- test/image/png_images/test_png_file.py | 10 +- test/image/test_load_image.py | 6 +- test/image/test_oversized.py | 18 +- test/requirements.txt | 1 - test/test_perfs.py | 6 +- 14 files changed, 239 insertions(+), 316 deletions(-) diff --git a/.github/workflows/continuous-integration-workflow.yml b/.github/workflows/continuous-integration-workflow.yml index 3b11a973a..489014481 100644 --- a/.github/workflows/continuous-integration-workflow.yml +++ b/.github/workflows/continuous-integration-workflow.yml @@ -58,7 +58,7 @@ jobs: # Ensuring there is no `generate=True` left remaining in calls to assert_pdf_equal: grep -IRF generate=True test/ && exit 1 # Executing all tests: - pytest -vv --final-rss-usage + pytest -vv --final-memory-usage - name: Uploading coverage report to codecov.io ☑ if: matrix.python-version == '3.10' && matrix.platform == 'ubuntu-latest' run: bash <(curl -s https://codecov.io/bash) diff --git a/fpdf/__init__.py b/fpdf/__init__.py index 836dfab48..4faf9fdee 100644 --- a/fpdf/__init__.py +++ b/fpdf/__init__.py @@ -12,7 +12,6 @@ from .html import HTMLMixin, HTML2FPDF from .prefs import ViewerPreferences from .template import Template, FlexTemplate -from . import svg from .deprecation import WarnOnDeprecatedModuleAttributes FPDF_VERSION = _FPDF_VERSION @@ -37,6 +36,7 @@ "__license__", # Classes "FPDF", + "FPDFException", "Align", "XPos", "YPos", diff --git a/fpdf/image_parsing.py b/fpdf/image_parsing.py index 7355977db..dc4c1c6d6 100644 --- a/fpdf/image_parsing.py +++ b/fpdf/image_parsing.py @@ -24,6 +24,37 @@ LOGGER = logging.getLogger(__name__) SUPPORTED_IMAGE_FILTERS = ("AUTO", "FlateDecode", "DCTDecode", "JPXDecode") +# fmt: off +TIFFBitRevTable = [ + 0x00, 0x80, 0x40, 0xC0, 0x20, 0xA0, 0x60, 0xE0, 0x10, 0x90, + 0x50, 0xD0, 0x30, 0xB0, 0x70, 0xF0, 0x08, 0x88, 0x48, 0xC8, + 0x28, 0xA8, 0x68, 0xE8, 0x18, 0x98, 0x58, 0xD8, 0x38, 0xB8, + 0x78, 0xF8, 0x04, 0x84, 0x44, 0xC4, 0x24, 0xA4, 0x64, 0xE4, + 0x14, 0x94, 0x54, 0xD4, 0x34, 0xB4, 0x74, 0xF4, 0x0C, 0x8C, + 0x4C, 0xCC, 0x2C, 0xAC, 0x6C, 0xEC, 0x1C, 0x9C, 0x5C, 0xDC, + 0x3C, 0xBC, 0x7C, 0xFC, 0x02, 0x82, 0x42, 0xC2, 0x22, 0xA2, + 0x62, 0xE2, 0x12, 0x92, 0x52, 0xD2, 0x32, 0xB2, 0x72, 0xF2, + 0x0A, 0x8A, 0x4A, 0xCA, 0x2A, 0xAA, 0x6A, 0xEA, 0x1A, 0x9A, + 0x5A, 0xDA, 0x3A, 0xBA, 0x7A, 0xFA, 0x06, 0x86, 0x46, 0xC6, + 0x26, 0xA6, 0x66, 0xE6, 0x16, 0x96, 0x56, 0xD6, 0x36, 0xB6, + 0x76, 0xF6, 0x0E, 0x8E, 0x4E, 0xCE, 0x2E, 0xAE, 0x6E, 0xEE, + 0x1E, 0x9E, 0x5E, 0xDE, 0x3E, 0xBE, 0x7E, 0xFE, 0x01, 0x81, + 0x41, 0xC1, 0x21, 0xA1, 0x61, 0xE1, 0x11, 0x91, 0x51, 0xD1, + 0x31, 0xB1, 0x71, 0xF1, 0x09, 0x89, 0x49, 0xC9, 0x29, 0xA9, + 0x69, 0xE9, 0x19, 0x99, 0x59, 0xD9, 0x39, 0xB9, 0x79, 0xF9, + 0x05, 0x85, 0x45, 0xC5, 0x25, 0xA5, 0x65, 0xE5, 0x15, 0x95, + 0x55, 0xD5, 0x35, 0xB5, 0x75, 0xF5, 0x0D, 0x8D, 0x4D, 0xCD, + 0x2D, 0xAD, 0x6D, 0xED, 0x1D, 0x9D, 0x5D, 0xDD, 0x3D, 0xBD, + 0x7D, 0xFD, 0x03, 0x83, 0x43, 0xC3, 0x23, 0xA3, 0x63, 0xE3, + 0x13, 0x93, 0x53, 0xD3, 0x33, 0xB3, 0x73, 0xF3, 0x0B, 0x8B, + 0x4B, 0xCB, 0x2B, 0xAB, 0x6B, 0xEB, 0x1B, 0x9B, 0x5B, 0xDB, + 0x3B, 0xBB, 0x7B, 0xFB, 0x07, 0x87, 0x47, 0xC7, 0x27, 0xA7, + 0x67, 0xE7, 0x17, 0x97, 0x57, 0xD7, 0x37, 0xB7, 0x77, 0xF7, + 0x0F, 0x8F, 0x4F, 0xCF, 0x2F, 0xAF, 0x6F, 0xEF, 0x1F, 0x9F, + 0x5F, 0xDF, 0x3F, 0xBF, 0x7F, 0xFF, +] +# fmt: on + def load_image(filename): """ @@ -54,276 +85,16 @@ def _decode_base64_image(base64Image): return BytesIO(decodedData) -TIFFBitRevTable = [ - 0x00, - 0x80, - 0x40, - 0xC0, - 0x20, - 0xA0, - 0x60, - 0xE0, - 0x10, - 0x90, - 0x50, - 0xD0, - 0x30, - 0xB0, - 0x70, - 0xF0, - 0x08, - 0x88, - 0x48, - 0xC8, - 0x28, - 0xA8, - 0x68, - 0xE8, - 0x18, - 0x98, - 0x58, - 0xD8, - 0x38, - 0xB8, - 0x78, - 0xF8, - 0x04, - 0x84, - 0x44, - 0xC4, - 0x24, - 0xA4, - 0x64, - 0xE4, - 0x14, - 0x94, - 0x54, - 0xD4, - 0x34, - 0xB4, - 0x74, - 0xF4, - 0x0C, - 0x8C, - 0x4C, - 0xCC, - 0x2C, - 0xAC, - 0x6C, - 0xEC, - 0x1C, - 0x9C, - 0x5C, - 0xDC, - 0x3C, - 0xBC, - 0x7C, - 0xFC, - 0x02, - 0x82, - 0x42, - 0xC2, - 0x22, - 0xA2, - 0x62, - 0xE2, - 0x12, - 0x92, - 0x52, - 0xD2, - 0x32, - 0xB2, - 0x72, - 0xF2, - 0x0A, - 0x8A, - 0x4A, - 0xCA, - 0x2A, - 0xAA, - 0x6A, - 0xEA, - 0x1A, - 0x9A, - 0x5A, - 0xDA, - 0x3A, - 0xBA, - 0x7A, - 0xFA, - 0x06, - 0x86, - 0x46, - 0xC6, - 0x26, - 0xA6, - 0x66, - 0xE6, - 0x16, - 0x96, - 0x56, - 0xD6, - 0x36, - 0xB6, - 0x76, - 0xF6, - 0x0E, - 0x8E, - 0x4E, - 0xCE, - 0x2E, - 0xAE, - 0x6E, - 0xEE, - 0x1E, - 0x9E, - 0x5E, - 0xDE, - 0x3E, - 0xBE, - 0x7E, - 0xFE, - 0x01, - 0x81, - 0x41, - 0xC1, - 0x21, - 0xA1, - 0x61, - 0xE1, - 0x11, - 0x91, - 0x51, - 0xD1, - 0x31, - 0xB1, - 0x71, - 0xF1, - 0x09, - 0x89, - 0x49, - 0xC9, - 0x29, - 0xA9, - 0x69, - 0xE9, - 0x19, - 0x99, - 0x59, - 0xD9, - 0x39, - 0xB9, - 0x79, - 0xF9, - 0x05, - 0x85, - 0x45, - 0xC5, - 0x25, - 0xA5, - 0x65, - 0xE5, - 0x15, - 0x95, - 0x55, - 0xD5, - 0x35, - 0xB5, - 0x75, - 0xF5, - 0x0D, - 0x8D, - 0x4D, - 0xCD, - 0x2D, - 0xAD, - 0x6D, - 0xED, - 0x1D, - 0x9D, - 0x5D, - 0xDD, - 0x3D, - 0xBD, - 0x7D, - 0xFD, - 0x03, - 0x83, - 0x43, - 0xC3, - 0x23, - 0xA3, - 0x63, - 0xE3, - 0x13, - 0x93, - 0x53, - 0xD3, - 0x33, - 0xB3, - 0x73, - 0xF3, - 0x0B, - 0x8B, - 0x4B, - 0xCB, - 0x2B, - 0xAB, - 0x6B, - 0xEB, - 0x1B, - 0x9B, - 0x5B, - 0xDB, - 0x3B, - 0xBB, - 0x7B, - 0xFB, - 0x07, - 0x87, - 0x47, - 0xC7, - 0x27, - 0xA7, - 0x67, - 0xE7, - 0x17, - 0x97, - 0x57, - 0xD7, - 0x37, - 0xB7, - 0x77, - 0xF7, - 0x0F, - 0x8F, - 0x4F, - 0xCF, - 0x2F, - 0xAF, - 0x6F, - 0xEF, - 0x1F, - 0x9F, - 0x5F, - 0xDF, - 0x3F, - 0xBF, - 0x7F, - 0xFF, -] - - def is_iccp_valid(iccp, filename): "Checks the validity of an ICC profile" try: profile = ImageCms.getOpenProfile(BytesIO(iccp)) except ImageCms.PyCMSError: - LOGGER.warning("Invalid ICC Profile in file %s", filename) + LOGGER.info("Invalid ICC Profile in file %s", filename) return False color_space = profile.profile.xcolor_space.strip() if color_space not in ("GRAY", "RGB"): - LOGGER.warning( + LOGGER.info( "Unsupported color space %s in ICC Profile of file %s - cf. issue #711", color_space, filename, @@ -342,13 +113,16 @@ def get_img_info(filename, img=None, image_filter="AUTO", dims=None): if Image is None: raise EnvironmentError("Pillow not available - fpdf2 cannot insert images") + is_pil_img = True img_raw_data = None if not img or isinstance(img, (Path, str)): img_raw_data = load_image(filename) img = Image.open(img_raw_data) + is_pil_img = False elif not isinstance(img, Image.Image): img_raw_data = img img = Image.open(img_raw_data) + is_pil_img = False img_altered = False if dims: @@ -502,6 +276,9 @@ def get_img_info(filename, img=None, image_filter="AUTO", dims=None): if img.mode == "1": dp = f"/BlackIs1 true /Columns {w} /K -1 /Rows {h}" + if not is_pil_img: + img.close() + info.update( { "w": w, diff --git a/fpdf/util.py b/fpdf/util.py index 29dde5565..e78c2b782 100644 --- a/fpdf/util.py +++ b/fpdf/util.py @@ -1,6 +1,11 @@ -import locale +import gc, os, warnings from datetime import datetime, timezone -from typing import Union, Iterable +from numbers import Number +from tracemalloc import get_traced_memory, is_tracing +from typing import Iterable, Tuple, Union + +# default block size from src/libImaging/Storage.c: +PIL_MEM_BLOCK_SIZE_IN_MIB = 16 def buffer_subst(buffer, placeholder, value): @@ -57,7 +62,7 @@ def b(s): raise ValueError(f"Invalid input: {s}") -def get_scale_factor(unit: Union[str, float, int]) -> float: +def get_scale_factor(unit: Union[str, Number]) -> float: """ Get how many pts are in a unit. (k) @@ -68,7 +73,7 @@ def get_scale_factor(unit: Union[str, float, int]) -> float: Raises: ValueError """ - if isinstance(unit, (int, float)): + if isinstance(unit, Number): return float(unit) if unit == "pt": @@ -84,8 +89,8 @@ def get_scale_factor(unit: Union[str, float, int]) -> float: def convert_unit( to_convert: Union[float, int, Iterable[Union[float, int, Iterable]]], - old_unit: Union[str, float, int], - new_unit: Union[str, float, int], + old_unit: Union[str, Number], + new_unit: Union[str, Number], ) -> Union[float, tuple]: """ Convert a number or sequence of numbers from one unit to another. @@ -107,14 +112,115 @@ def convert_unit( return to_convert / unit_conversion_factor -def dochecks(): - # Check for locale-related bug - # if (1.1==1): - # raise FPDFException("Don\'t alter the locale before including class file") - # Check for decimal separator - if f"{1.0:.1f}" != "1.0": - locale.setlocale(locale.LC_NUMERIC, "C") - - -# Moved here from FPDF#__init__ -dochecks() +################################################################################ +################### Utility functions to track memory usage #################### +################################################################################ + + +def print_mem_usage(prefix): + print(get_mem_usage(prefix)) + + +def get_mem_usage(prefix) -> str: + _collected_count = gc.collect() + rss = get_process_rss() + # heap_size, stack_size = get_process_heap_and_stack_sizes() + # objs_size_sum = get_gc_managed_objs_total_size() + pillow = get_pillow_allocated_memory() + if is_tracing(): + malloc_stats = get_tracemalloc_traced_memory() + else: + malloc_stats = get_pymalloc_allocated_over_total_size() + return f"{prefix:<40} Malloc stats: {malloc_stats} | Pillow: {pillow} | Process RSS: {rss}" + + +def get_process_rss() -> str: + rss_as_mib = get_process_rss_as_mib() + if rss_as_mib: + return f"{rss_as_mib:.1f} MiB" + return "" + + +def get_process_rss_as_mib() -> Union[Number, None]: + "Inspired by psutil source code" + pid = os.getpid() + try: + with open(f"/proc/{pid}/statm", encoding="utf8") as statm: + return ( + int(statm.readline().split()[1]) + * os.sysconf("SC_PAGE_SIZE") + / 1024 + / 1024 + ) + except FileNotFoundError: # This file only exists under Linux + return None + + +def get_process_heap_and_stack_sizes() -> Tuple[str]: + heap_size_in_mib, stack_size_in_mib = "", "" + pid = os.getpid() + try: + with open(f"/proc/{pid}/maps", encoding="utf8") as maps_file: + maps_lines = list(maps_file) + except FileNotFoundError: # This file only exists under Linux + return heap_size_in_mib, stack_size_in_mib + for line in maps_lines: + line = line.split() + addr_range, path = line[0], line[-1] + addr_start, addr_end = addr_range.split("-") + addr_start, addr_end = int(addr_start, 16), int(addr_end, 16) + size = addr_end - addr_start + if path == "[heap]": + heap_size_in_mib = f"{size / 1024 / 1024:.1f} MiB" + elif path == "[stack]": + stack_size_in_mib = f"{size / 1024 / 1024:.1f} MiB" + return heap_size_in_mib, stack_size_in_mib + + +def get_pymalloc_allocated_over_total_size() -> Tuple[str]: + """ + Get PyMalloc stats from sys._debugmallocstats() + From experiments, not very reliable + """ + try: + # pylint: disable=import-outside-toplevel + from pymemtrace.debug_malloc_stats import get_debugmallocstats + + allocated, total = -1, -1 + for line in get_debugmallocstats().decode().splitlines(): + if line.startswith("Total"): + total = int(line.split()[-1].replace(",", "")) + elif line.startswith("# bytes in allocated blocks"): + allocated = int(line.split()[-1].replace(",", "")) + return f"{allocated / 1024 / 1024:.1f} / {total / 1024 / 1024:.1f} MiB" + except ImportError: + warnings.warn("pymemtrace could not be imported - Run: pip install pymemtrace") + return "" + + +def get_gc_managed_objs_total_size() -> str: + "From experiments, not very reliable" + try: + # pylint: disable=import-outside-toplevel + from pympler.muppy import get_objects, getsizeof + + objs_total_size = sum(getsizeof(obj) for obj in get_objects()) + return f"{objs_total_size / 1024 / 1024:.1f} MiB" + except ImportError: + warnings.warn("pympler could not be imported - Run: pip install pympler") + return "" + + +def get_tracemalloc_traced_memory() -> str: + "Requires python -X tracemalloc" + current, peak = get_traced_memory() + return f"{current / 1024 / 1024:.1f} (peak={peak / 1024 / 1024:.1f}) MiB" + + +def get_pillow_allocated_memory() -> str: + # pylint: disable=c-extension-no-member,import-outside-toplevel + from PIL import Image + + stats = Image.core.get_stats() + blocks_in_use = stats["allocated_blocks"] - stats["freed_blocks"] + return f"{blocks_in_use * PIL_MEM_BLOCK_SIZE_IN_MIB:.1f} MiB" diff --git a/test/conftest.py b/test/conftest.py index f43c6686c..7166bbcc7 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,21 +1,27 @@ +# pylint: disable=import-outside-toplevel from contextlib import contextmanager from datetime import datetime, timezone from time import perf_counter from types import SimpleNamespace +import functools +import gc import hashlib +import linecache import pathlib import signal import shutil import sys +import tracemalloc import warnings -import gc, linecache, tracemalloc + from subprocess import check_output, CalledProcessError, PIPE -from psutil import Process # transitive dependency of memunit import pytest +from fpdf.util import get_process_rss_as_mib, print_mem_usage from fpdf.template import Template + QPDF_AVAILABLE = bool(shutil.which("qpdf")) if not QPDF_AVAILABLE: warnings.warn( @@ -255,13 +261,32 @@ def handler(_, __): signal.signal(signal.SIGALRM, signal.SIG_DFL) +def ensure_rss_memory_below(max_in_mib): + def actual_decorator(test_func): + @functools.wraps(test_func) + def wrapper(*args, **kwargs): + test_func(*args, **kwargs) + rss_in_mib = get_process_rss_as_mib() + if rss_in_mib: + assert rss_in_mib < max_in_mib + + return wrapper + + return actual_decorator + + # Enabling this check creates an increase in memory usage, # so we require an opt-in through a CLI argument: def pytest_addoption(parser): parser.addoption( - "--final-rss-usage", + "--final-memory-usage", + action="store_true", + help="At the end of the tests execution, display the current memory usage", + ) + parser.addoption( + "--pympler-summary", action="store_true", - help="At the end of the tests execution, display the current RSS memory usage", + help="At the end of the tests execution, display a summary of objects allocated in memory", ) parser.addoption( "--trace-malloc", @@ -271,14 +296,30 @@ def pytest_addoption(parser): @pytest.fixture(scope="session", autouse=True) -def final_rss_usage(request): +def final_memory_usage(request): yield - if request.config.getoption("final_rss_usage"): - rss_in_mib = Process().memory_info().rss / 1024 / 1024 + if request.config.getoption("final_memory_usage"): + gc.collect() capmanager = request.config.pluginmanager.getplugin("capturemanager") with capmanager.global_and_fixture_disabled(): print("\n") - print(f"[psutil] Final process RSS memory usage: {rss_in_mib:.1f} MiB") + print_mem_usage("Final memory usage:") + + +@pytest.fixture(scope="session", autouse=True) +def pympler_summary(request): + yield + if request.config.getoption("pympler_summary"): + # pylint: disable=import-error + from pympler.muppy import get_objects + from pympler.summary import print_, summarize + + gc.collect() + all_objects = get_objects() + capmanager = request.config.pluginmanager.getplugin("capturemanager") + with capmanager.global_and_fixture_disabled(): + print("\n[pympler/muppy] biggest objects summary:") + print_(summarize(all_objects)) @pytest.fixture(scope="session", autouse=True) @@ -286,7 +327,6 @@ def trace_malloc(request): if not request.config.getoption("trace_malloc"): yield return - capmanager = request.config.pluginmanager.getplugin("capturemanager") gc.collect() # Top-10 recipe from: https://docs.python.org/3/library/tracemalloc.html#display-the-top-10 tracemalloc.start() @@ -305,6 +345,7 @@ def trace_malloc(request): ) ) top_stats = snapshot2.compare_to(snapshot1, "lineno") + capmanager = request.config.pluginmanager.getplugin("capturemanager") with capmanager.global_and_fixture_disabled(): print("[tracemalloc] Top 10 differences:") for stat in top_stats[:10]: diff --git a/test/embed_file_all_optionals.pdf b/test/embed_file_all_optionals.pdf index d408ff960584f4fb91f526190786f2d2e8dfeb09..e16df62cac2c0f77d06351b860449186e6575725 100644 GIT binary patch delta 253 zcmbQuHI-{a5|fA#mwrHEQDSbff{l%SaAsAif|=#yGNy0^6H8MA3rkCrq*Qa`vmEGnre SN=@Ukv^3;WRdw}u;{pH~;7k4h delta 263 zcmbQrHJfWg5|fApmwrHEQDSbff{l%SaAsAig1PzRHO!im=QHUkn3^RSrJ7kLnkOfl zrJ1H#09nSVW~QczNhT&1DM^Ww-!iE>o^a$lpuoX;(YE~MLXm$>AqzrHUcSAp7qj=| zl}C#_w~E?v>PhT!V4MB#?d!}}_MICoEv}slaXzzTcQMBc)|9O>_cr@7%P@*sDj0x( zLY@K_m|2yRj>S60#nRH%z}dyn(Zs;e*xbd?z{SMH*}~Dqz}3~r)zQhw&W50h dSS~v|uHur!qLPZD)HE(jOLHz&RabvEE&x9oQ^5cL diff --git a/test/embed_file_self.pdf b/test/embed_file_self.pdf index a34583a5d4520577986e5744088ef68876e30aac..d4b979bcda8624ca7b505aa563615a149ae64179 100644 GIT binary patch delta 158 zcmeyv@r7f9A(NhHz{0N1Q1_5c6? delta 166 zcmeyu@rPrBA(N=NIhTGwVo_plv4V|_esE@0s)D)sWH%;r_T1Fm(!9))%{5HkjADic z3I-sckf*=}W*8Wl1J&JSwvKT&wRCecG_^1^G&6FuFn2U?aWu1Vu{1JwF)+6@F|f3= gA*dpj%g&CgxFoTtq@pM_jmyH=f=gA^)!&T^0F3b}egFUf diff --git a/test/file_attachment_annotation.pdf b/test/file_attachment_annotation.pdf index ce7e38d0ff842ad1695a582db2444e03af9ec5f8..029f319d3a8c6e713b4eec2647d5647be61f2bd8 100644 GIT binary patch delta 159 zcmZ3Vf%LY@K_ zm|1_nl4s;aL3Zd?En+9)yr delta 167 zcmZ3;wUTSYT_#a;b1waW#G=I9Vg(x;{ou^1R0VVM$)A|a*>h8KOY<^IHfu0@Gm2Rl zDHwo&LY@K_m|?|$kox%#Msi&#nsi(+`z!q i&W50hSS~v|uHur!qLPZD)HE(b149EYRaIAiH!c7JOe=-} diff --git a/test/image/image_types/test_insert_images.py b/test/image/image_types/test_insert_images.py index 559f88a08..0cf7351ba 100644 --- a/test/image/image_types/test_insert_images.py +++ b/test/image/image_types/test_insert_images.py @@ -1,4 +1,5 @@ import io +import logging import sys from pathlib import Path @@ -133,10 +134,11 @@ def test_insert_jpg_icc(tmp_path): def test_insert_jpg_invalid_icc(caplog, tmp_path): - pdf = fpdf.FPDF() - pdf.add_page(format=(448, 498)) - pdf.set_margin(0) - pdf.image(HERE / "insert_images_insert_jpg_icc_invalid.jpg", x=0, y=0, h=498) + with caplog.at_level(logging.INFO): + pdf = fpdf.FPDF() + pdf.add_page(format=(448, 498)) + pdf.set_margin(0) + pdf.image(HERE / "insert_images_insert_jpg_icc_invalid.jpg", x=0, y=0, h=498) assert "Invalid ICC Profile in file" in caplog.text assert_pdf_equal(pdf, HERE / "image_types_insert_jpg_icc_invalid.pdf", tmp_path) diff --git a/test/image/png_images/test_png_file.py b/test/image/png_images/test_png_file.py index fc3557373..a4ede3d50 100644 --- a/test/image/png_images/test_png_file.py +++ b/test/image/png_images/test_png_file.py @@ -1,3 +1,4 @@ +import logging from pathlib import Path from fpdf import FPDF @@ -8,10 +9,11 @@ def test_insert_png_files(caplog, tmp_path): - pdf = FPDF() - for path in sorted(HERE.glob("*.png")): - pdf.add_page() - pdf.image(str(path), x=0, y=0, w=0, h=0) + with caplog.at_level(logging.INFO): + pdf = FPDF() + for png_path in sorted(HERE.glob("*.png")): + pdf.add_page() + pdf.image(png_path, x=0, y=0, w=0, h=0) # Note: 7 of those images have an ICC profile, and there are only 5 distinct ICC profiles among them assert_pdf_equal(pdf, HERE / "image_png_insert_png_files.pdf", tmp_path) diff --git a/test/image/test_load_image.py b/test/image/test_load_image.py index fe0ba8e94..d73ff2265 100644 --- a/test/image/test_load_image.py +++ b/test/image/test_load_image.py @@ -2,11 +2,11 @@ from glob import glob from pathlib import Path -import memunit, pytest +import pytest import fpdf -from test.conftest import assert_pdf_equal, time_execution +from test.conftest import assert_pdf_equal, ensure_rss_memory_below, time_execution HERE = Path(__file__).resolve().parent @@ -42,7 +42,7 @@ def test_load_invalid_base64_data(): # ensure memory usage does not get too high - this value depends on Python version: -@memunit.assert_lt_mb(200) +@ensure_rss_memory_below(max_in_mib=185) def test_share_images_cache(tmp_path): images_cache = {} icc_profiles_cache = {} diff --git a/test/image/test_oversized.py b/test/image/test_oversized.py index b7107d4cd..eeec3eb11 100644 --- a/test/image/test_oversized.py +++ b/test/image/test_oversized.py @@ -1,15 +1,13 @@ import logging from pathlib import Path -import memunit - import fpdf -from test.conftest import assert_pdf_equal +from test.conftest import assert_pdf_equal, ensure_rss_memory_below HERE = Path(__file__).resolve().parent IMAGE_PATH = HERE / "png_images/6c853ed9dacd5716bc54eb59cec30889.png" -MAX_MEMORY_MB = 200 # memory usage depends on Python version +MAX_MEMORY_MB = 185 # memory usage depends on Python version def test_oversized_images_warn(caplog): @@ -20,7 +18,7 @@ def test_oversized_images_warn(caplog): assert "OVERSIZED" in caplog.text -@memunit.assert_lt_mb(MAX_MEMORY_MB) +@ensure_rss_memory_below(max_in_mib=MAX_MEMORY_MB) def test_oversized_images_downscale_simple(caplog, tmp_path): caplog.set_level(logging.DEBUG) pdf = fpdf.FPDF() @@ -34,7 +32,7 @@ def test_oversized_images_downscale_simple(caplog, tmp_path): assert_pdf_equal(pdf, HERE / "oversized_images_downscale_simple.pdf", tmp_path) -@memunit.assert_lt_mb(MAX_MEMORY_MB) +@ensure_rss_memory_below(max_in_mib=MAX_MEMORY_MB) def test_oversized_images_downscale_twice(tmp_path): pdf = fpdf.FPDF() pdf.oversized_images = "DOWNSCALE" @@ -47,7 +45,7 @@ def test_oversized_images_downscale_twice(tmp_path): assert_pdf_equal(pdf, HERE / "oversized_images_downscale_twice.pdf", tmp_path) -@memunit.assert_lt_mb(MAX_MEMORY_MB) +@ensure_rss_memory_below(max_in_mib=MAX_MEMORY_MB) def test_oversized_images_downscaled_and_highres(): pdf = fpdf.FPDF() pdf.oversized_images = "DOWNSCALE" @@ -58,7 +56,7 @@ def test_oversized_images_downscaled_and_highres(): # Not calling assert_pdf_equal to avoid storing a large binary (1.4M) in this git repo -@memunit.assert_lt_mb(MAX_MEMORY_MB) +@ensure_rss_memory_below(max_in_mib=MAX_MEMORY_MB) def test_oversized_images_highres_and_downscaled(): pdf = fpdf.FPDF() pdf.oversized_images = "DOWNSCALE" @@ -69,7 +67,7 @@ def test_oversized_images_highres_and_downscaled(): # Not calling assert_pdf_equal to avoid storing a large binary (1.4M) in this git repo -@memunit.assert_lt_mb(MAX_MEMORY_MB) +@ensure_rss_memory_below(max_in_mib=MAX_MEMORY_MB) def test_oversized_images_downscale_biggest_1st(tmp_path): pdf = fpdf.FPDF() pdf.oversized_images = "DOWNSCALE" @@ -82,7 +80,7 @@ def test_oversized_images_downscale_biggest_1st(tmp_path): assert_pdf_equal(pdf, HERE / "oversized_images_downscale_biggest_1st.pdf", tmp_path) -@memunit.assert_lt_mb(MAX_MEMORY_MB) +@ensure_rss_memory_below(max_in_mib=MAX_MEMORY_MB) def test_oversized_images_downscale_biggest_2nd(caplog, tmp_path): caplog.set_level(logging.DEBUG) pdf = fpdf.FPDF() diff --git a/test/requirements.txt b/test/requirements.txt index 714abc439..17d3230af 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,7 +1,6 @@ bandit black endesive -memunit pylint pytest pytest-cov diff --git a/test/test_perfs.py b/test/test_perfs.py index ec80f194d..5c2323e3b 100644 --- a/test/test_perfs.py +++ b/test/test_perfs.py @@ -1,17 +1,15 @@ from pathlib import Path import pytest +from test.conftest import ensure_rss_memory_below from fpdf import FPDF HERE = Path(__file__).resolve().parent +@ensure_rss_memory_below(max_in_mib=175) @pytest.mark.timeout(40) -# Note: this does not combine well with memunit. -# memory_profiler.MemTimer does not terminate properly when a signal is raised by pytest-timeout, -# and as a consequence multiprocessing.util._exit_function becomes blocking at the end of Pytest execution, -# (on line "calling join() for process MemTimer-1") def test_intense_image_rendering(): png_file_paths = [] for png_file_path in (HERE / "image/png_images/").glob("*.png"):