Make temperature an adaptive counter like the rest

Author: gvanrossum

Include/cpython/optimizer.h

@@ -89,7 +89,7 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst)
 
 typedef struct _exit_data {
     uint32_t target;
-    int16_t temperature;
+    uint16_t temperature;
     const struct _PyExecutorObject *executor;
 } _PyExitData;
 
@@ -115,11 +115,10 @@ typedef int (*optimize_func)(
 struct _PyOptimizerObject {
     PyObject_HEAD
     optimize_func optimize;
-    /* These thresholds are treated as signed so do not exceed INT16_MAX
-     * Use INT16_MAX to indicate that the optimizer should never be called */
+    /* Initial values for adaptive-style counters */
+    uint16_t backedge_threshold;
     uint16_t resume_threshold;
     uint16_t side_threshold;
-    uint16_t backedge_threshold;
     /* Data needed by the optimizer goes here, but is opaque to the VM */
 };
 

Include/internal/pycore_interp.h

@@ -239,10 +239,9 @@ struct _is {
     _PyOptimizerObject *optimizer;
     _PyExecutorObject *executor_list_head;
 
-    /* These two values are shifted and offset to speed up check in JUMP_BACKWARD */
-    uint32_t optimizer_resume_threshold;
+    /* These three values are shifted and offset to speed up check in JUMP_BACKWARD */
     uint32_t optimizer_backedge_threshold;
-
+    uint32_t optimizer_resume_threshold;
     uint16_t optimizer_side_threshold;
 
     _rare_events rare_events;

Python/bytecodes.c

@@ -4157,21 +4157,21 @@ dummy_func(
         tier2 op(_COLD_EXIT, (--)) {
             _PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor;
             _PyExitData *exit = &previous->exits[oparg];
-            exit->temperature++;
             PyCodeObject *code = _PyFrame_GetCode(frame);
             _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target;
-            if (exit->temperature < (int32_t)tstate->interp->optimizer_side_threshold) {
+            if (!ADAPTIVE_COUNTER_IS_ZERO(exit->temperature)) {
+                DECREMENT_ADAPTIVE_COUNTER(exit->temperature);
                 GOTO_TIER_ONE(target);
             }
             _PyExecutorObject *executor;
             if (target->op.code == ENTER_EXECUTOR) {
                 executor = code->co_executors->executors[target->op.arg];
                 Py_INCREF(executor);
-            } else {
+            }
+            else {
                 int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor);
                 if (optimized <= 0) {
-                    int32_t new_temp = -1 * tstate->interp->optimizer_side_threshold;
-                    exit->temperature = (new_temp < INT16_MIN) ? INT16_MIN : new_temp;
+                    exit->temperature = adaptive_counter_backoff(exit->temperature);
                     if (optimized < 0) {
                         Py_DECREF(previous);
                         tstate->previous_executor = Py_None;

Python/executor_cases.c.h

@@ -125,8 +125,8 @@ PyTypeObject _PyDefaultOptimizer_Type = {
 static _PyOptimizerObject _PyOptimizer_Default = {
     PyObject_HEAD_INIT(&_PyDefaultOptimizer_Type)
     .optimize = never_optimize,
-    .resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD,
     .backedge_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD,
+    .resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD,
     .side_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD,
 };
 
@@ -189,10 +189,11 @@ _Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer)
     interp->optimizer = optimizer;
     interp->optimizer_backedge_threshold = shift_and_offset_threshold(optimizer->backedge_threshold);
     interp->optimizer_resume_threshold = shift_and_offset_threshold(optimizer->resume_threshold);
-    interp->optimizer_side_threshold = optimizer->side_threshold;
+    interp->optimizer_side_threshold = shift_and_offset_threshold(optimizer->side_threshold);
     if (optimizer == &_PyOptimizer_Default) {
         assert(interp->optimizer_backedge_threshold == OPTIMIZER_UNREACHABLE_THRESHOLD);
         assert(interp->optimizer_resume_threshold == OPTIMIZER_UNREACHABLE_THRESHOLD);
+        assert(interp->optimizer_side_threshold == OPTIMIZER_UNREACHABLE_THRESHOLD);
     }
     return old;
 }
@@ -1103,10 +1104,12 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil
     }
 
     /* Initialize exits */
+    PyInterpreterState *interp = _PyInterpreterState_GET();
     assert(exit_count < COLD_EXIT_COUNT);
     for (int i = 0; i < exit_count; i++) {
         executor->exits[i].executor = &COLD_EXITS[i];
-        executor->exits[i].temperature = 0;
+        executor->exits[i].temperature =
+            adaptive_counter_bits(interp->optimizer_side_threshold, 4);  // TODO: Constantify
     }
     int next_exit = exit_count-1;
     _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length];
@@ -1288,11 +1291,11 @@ PyUnstable_Optimizer_NewUOpOptimizer(void)
         return NULL;
     }
     opt->optimize = uop_optimize;
-    opt->resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD;
     // Need a few iterations to settle specializations,
     // and to ammortize the cost of optimization.
-    opt->side_threshold = 16;
     opt->backedge_threshold = 16;
+    opt->resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD;
+    opt->side_threshold = 16;
     return (PyObject *)opt;
 }
 
@@ -1382,9 +1385,9 @@ PyUnstable_Optimizer_NewCounter(void)
         return NULL;
     }
     opt->base.optimize = counter_optimize;
+    opt->base.backedge_threshold = 0;
     opt->base.resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD;
     opt->base.side_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD;
-    opt->base.backedge_threshold = 0;
     opt->count = 0;
     return (PyObject *)opt;
 }
@@ -1551,7 +1554,7 @@ _Py_ExecutorClear(_PyExecutorObject *executor)
     for (uint32_t i = 0; i < executor->exit_count; i++) {
         Py_DECREF(executor->exits[i].executor);
         executor->exits[i].executor = &COLD_EXITS[i];
-        executor->exits[i].temperature = INT16_MIN;
+        executor->exits[i].temperature = OPTIMIZER_UNREACHABLE_THRESHOLD;
     }
     _Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index];
     assert(instruction->op.code == ENTER_EXECUTOR);