ensure qp is clamped to a valid range

Xpra-org · Sep 3, 2024 · 7ad3b07 · 7ad3b07
1 parent d81c793
commit 7ad3b07
Showing 1 changed file with 6 additions and 3 deletions.
diff --git a/xpra/codecs/nvidia/nvenc/encoder.pyx b/xpra/codecs/nvidia/nvenc/encoder.pyx
@@ -2043,9 +2043,12 @@ cdef class Encoder:
         rc.rateControlMode = NV_ENC_PARAMS_RC_VBR
         #rc.zeroReorderDelay = 1       #zero-latency
         QP_MAX_VALUE = 51       #255 for AV1!
-        qpmin = QP_MAX_VALUE-min(QP_MAX_VALUE, int(QP_MAX_VALUE*(self.quality-10)//100))
-        qpmax = QP_MAX_VALUE-max(0, int(QP_MAX_VALUE*(self.quality+10)//100))
-        qp = min(QP_MAX_VALUE, max(0, (qpmin + qpmax)//2))
+
+        def qp(pct: float) -> int:
+            return QP_MAX_VALUE-max(0, min(QP_MAX_VALUE, round(QP_MAX_VALUE * pct)))
+        qpmin = qp(self.quality-10)
+        qpmax = qp(self.quality+10)
+        qp = min(QP_MAX_VALUE, max(0, round((qpmin + qpmax)//2)))
         rc.enableMinQP = 1
         rc.enableMaxQP = 1
         rc.minQP.qpInterB = qpmin