From 7ad3b076b928ca8541d7f5e13f88163278ee7350 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 3 Sep 2024 12:05:41 +0700 Subject: [PATCH] ensure qp is clamped to a valid range --- xpra/codecs/nvidia/nvenc/encoder.pyx | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/xpra/codecs/nvidia/nvenc/encoder.pyx b/xpra/codecs/nvidia/nvenc/encoder.pyx index e532f14ef3..eeabd2ec1c 100644 --- a/xpra/codecs/nvidia/nvenc/encoder.pyx +++ b/xpra/codecs/nvidia/nvenc/encoder.pyx @@ -2043,9 +2043,12 @@ cdef class Encoder: rc.rateControlMode = NV_ENC_PARAMS_RC_VBR #rc.zeroReorderDelay = 1 #zero-latency QP_MAX_VALUE = 51 #255 for AV1! - qpmin = QP_MAX_VALUE-min(QP_MAX_VALUE, int(QP_MAX_VALUE*(self.quality-10)//100)) - qpmax = QP_MAX_VALUE-max(0, int(QP_MAX_VALUE*(self.quality+10)//100)) - qp = min(QP_MAX_VALUE, max(0, (qpmin + qpmax)//2)) + + def qp(pct: float) -> int: + return QP_MAX_VALUE-max(0, min(QP_MAX_VALUE, round(QP_MAX_VALUE * pct))) + qpmin = qp(self.quality-10) + qpmax = qp(self.quality+10) + qp = min(QP_MAX_VALUE, max(0, round((qpmin + qpmax)//2))) rc.enableMinQP = 1 rc.enableMaxQP = 1 rc.minQP.qpInterB = qpmin