Make _Py_BackoffCounter a member of _Py_CODEUNIT

This changes a lot of things but the end result is arguably better.
python · gvanrossum · Apr 4, 2024 · Mar 22, 2024 · Mar 22, 2024 · Mar 22, 2024
commit 3fee35f0e749feb27e104e8f212bed20c390149e
diff --git a/Include/cpython/code.h b/Include/cpython/code.h
@@ -24,6 +24,16 @@ typedef struct _Py_GlobalMonitors {
     uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS];
 } _Py_GlobalMonitors;
 
+typedef struct {
+    union {
+        struct {
+            uint16_t backoff : 4;
+            uint16_t value : 12;
+        };
+        uint16_t as_counter;  // For printf("%#x", ...)
+    };
+} _Py_BackoffCounter;
+
 /* Each instruction in a code object is a fixed-width value,
  * currently 2 bytes: 1-byte opcode + 1-byte oparg.  The EXTENDED_ARG
  * opcode allows for larger values but the current limit is 3 uses
@@ -39,6 +49,7 @@ typedef union {
         uint8_t code;
         uint8_t arg;
     } op;
+    _Py_BackoffCounter counter;  // First cache entry of specializable op
 } _Py_CODEUNIT;
 
 

diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h
@@ -89,7 +89,7 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst)
 
 typedef struct _exit_data {
     uint32_t target;
-    uint16_t temperature;
+    _Py_BackoffCounter temperature;
     const struct _PyExecutorObject *executor;
 } _PyExitData;
 

diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h
@@ -31,43 +31,30 @@ extern "C" {
    There is an exceptional value which must not be updated, 0xFFFF.
 */
 
-typedef struct {
-    union {
-        uint16_t counter;
-        struct {
-            uint16_t backoff : 4;
-            uint16_t value : 12;
-        };
-    };
-} backoff_counter_t;
-
-static_assert(sizeof(backoff_counter_t) == sizeof(_Py_CODEUNIT),
-    "backoff counter size should be the same size as a code unit");
-
 #define UNREACHABLE_BACKOFF 0xFFFF
 
 static inline bool
-is_unreachable_backoff_counter(backoff_counter_t counter)
+is_unreachable_backoff_counter(_Py_BackoffCounter counter)
 {
-    return counter.counter == UNREACHABLE_BACKOFF;
+    return counter.as_counter == UNREACHABLE_BACKOFF;
 }
 
-static inline backoff_counter_t
+static inline _Py_BackoffCounter
 make_backoff_counter(uint16_t value, uint16_t backoff)
 {
     assert(backoff <= 15);
     assert(value <= 0xFFF);
-    return (backoff_counter_t){.value = value, .backoff = backoff};
+    return (_Py_BackoffCounter){.value = value, .backoff = backoff};
 }
 
-static inline backoff_counter_t
+static inline _Py_BackoffCounter
 forge_backoff_counter(uint16_t counter)
 {
-    return (backoff_counter_t){.counter = counter};
+    return (_Py_BackoffCounter){.as_counter = counter};
 }
 
-static inline backoff_counter_t
-restart_backoff_counter(backoff_counter_t counter)
+static inline _Py_BackoffCounter
+restart_backoff_counter(_Py_BackoffCounter counter)
 {
     assert(!is_unreachable_backoff_counter(counter));
     if (counter.backoff < 12) {
@@ -78,14 +65,14 @@ restart_backoff_counter(backoff_counter_t counter)
     }
 }
 
-static inline backoff_counter_t
-pause_backoff_counter(backoff_counter_t counter)
+static inline _Py_BackoffCounter
+pause_backoff_counter(_Py_BackoffCounter counter)
 {
     return make_backoff_counter(counter.value | 1, counter.backoff);
 }
 
-static inline backoff_counter_t
-advance_backoff_counter(backoff_counter_t counter)
+static inline _Py_BackoffCounter
+advance_backoff_counter(_Py_BackoffCounter counter)
 {
     if (!is_unreachable_backoff_counter(counter)) {
         return make_backoff_counter((counter.value - 1) & 0xFFF, counter.backoff);
@@ -96,16 +83,16 @@ advance_backoff_counter(backoff_counter_t counter)
 }
 
 static inline bool
-backoff_counter_triggers(backoff_counter_t counter)
+backoff_counter_triggers(_Py_BackoffCounter counter)
 {
     return counter.value == 0;
 }
 
-static inline uint16_t
+static inline _Py_BackoffCounter
 initial_backoff_counter(void)
 {
     // Backoff sequence 16, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096
-    return make_backoff_counter(16, 3).counter;
+    return make_backoff_counter(16, 3);
 }
 
 #ifdef __cplusplus

diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
@@ -31,7 +31,7 @@ extern "C" {
 #define CACHE_ENTRIES(cache) (sizeof(cache)/sizeof(_Py_CODEUNIT))
 
 typedef struct {
-    uint16_t counter;
+    _Py_BackoffCounter counter;
     uint16_t module_keys_version;
     uint16_t builtin_keys_version;
     uint16_t index;
@@ -40,44 +40,44 @@ typedef struct {
 #define INLINE_CACHE_ENTRIES_LOAD_GLOBAL CACHE_ENTRIES(_PyLoadGlobalCache)
 
 typedef struct {
-    uint16_t counter;
+    _Py_BackoffCounter counter;
 } _PyBinaryOpCache;
 
 #define INLINE_CACHE_ENTRIES_BINARY_OP CACHE_ENTRIES(_PyBinaryOpCache)
 
 typedef struct {
-    uint16_t counter;
+    _Py_BackoffCounter counter;
 } _PyUnpackSequenceCache;
 
    #define INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE \
     CACHE_ENTRIES(_PyUnpackSequenceCache)
 
 typedef struct {
-    uint16_t counter;
+    _Py_BackoffCounter counter;
 } _PyCompareOpCache;
 
 #define INLINE_CACHE_ENTRIES_COMPARE_OP CACHE_ENTRIES(_PyCompareOpCache)
 
 typedef struct {
-    uint16_t counter;
+    _Py_BackoffCounter counter;
 } _PyBinarySubscrCache;
 
 #define INLINE_CACHE_ENTRIES_BINARY_SUBSCR CACHE_ENTRIES(_PyBinarySubscrCache)
 
 typedef struct {
-    uint16_t counter;
+    _Py_BackoffCounter counter;
 } _PySuperAttrCache;
 
 #define INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR CACHE_ENTRIES(_PySuperAttrCache)
 
 typedef struct {
-    uint16_t counter;
+    _Py_BackoffCounter counter;
     uint16_t version[2];
     uint16_t index;
 } _PyAttrCache;
 
 typedef struct {
-    uint16_t counter;
+    _Py_BackoffCounter counter;
     uint16_t type_version[2];
     union {
         uint16_t keys_version[2];
@@ -93,39 +93,39 @@ typedef struct {
 #define INLINE_CACHE_ENTRIES_STORE_ATTR CACHE_ENTRIES(_PyAttrCache)
 
 typedef struct {
-    uint16_t counter;
+    _Py_BackoffCounter counter;
     uint16_t func_version[2];
 } _PyCallCache;
 
 #define INLINE_CACHE_ENTRIES_CALL CACHE_ENTRIES(_PyCallCache)
 
 typedef struct {
-    uint16_t counter;
+    _Py_BackoffCounter counter;
 } _PyStoreSubscrCache;
 
 #define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache)
 
 typedef struct {
-    uint16_t counter;
+    _Py_BackoffCounter counter;
 } _PyForIterCache;
 
 #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache)
 
 typedef struct {
-    uint16_t counter;
+    _Py_BackoffCounter counter;
 } _PySendCache;
 
 #define INLINE_CACHE_ENTRIES_SEND CACHE_ENTRIES(_PySendCache)
 
 typedef struct {
-    uint16_t counter;
+    _Py_BackoffCounter counter;
     uint16_t version[2];
 } _PyToBoolCache;
 
 #define INLINE_CACHE_ENTRIES_TO_BOOL CACHE_ENTRIES(_PyToBoolCache)
 
 typedef struct {
-    uint16_t counter;
+    _Py_BackoffCounter counter;
 } _PyContainsOpCache;
 
 #define INLINE_CACHE_ENTRIES_CONTAINS_OP CACHE_ENTRIES(_PyContainsOpCache)
@@ -476,26 +476,26 @@ write_location_entry_start(uint8_t *ptr, int code, int length)
 #define ADAPTIVE_COOLDOWN_VALUE 52
 #define ADAPTIVE_COOLDOWN_BACKOFF 0
 
-static inline uint16_t
+static inline _Py_BackoffCounter
 adaptive_counter_bits(uint16_t value, uint16_t backoff) {
-    return make_backoff_counter(value, backoff).counter;
+    return make_backoff_counter(value, backoff);
 }
 
-static inline uint16_t
+static inline _Py_BackoffCounter
 adaptive_counter_warmup(void) {
     return adaptive_counter_bits(ADAPTIVE_WARMUP_VALUE,
                                  ADAPTIVE_WARMUP_BACKOFF);
 }
 
-static inline uint16_t
+static inline _Py_BackoffCounter
 adaptive_counter_cooldown(void) {
     return adaptive_counter_bits(ADAPTIVE_COOLDOWN_VALUE,
                                  ADAPTIVE_COOLDOWN_BACKOFF);
 }
 
-static inline uint16_t
-adaptive_counter_backoff(uint16_t counter) {
-    return restart_backoff_counter(forge_backoff_counter(counter)).counter;
+static inline _Py_BackoffCounter
+adaptive_counter_backoff(_Py_BackoffCounter counter) {
+    return restart_backoff_counter(counter);
 }