Skip to content

Commit 7820a58

Browse files
authored
bpo-46841: Use inline caching for COMPARE_OP (GH-31622)
1 parent df9f759 commit 7820a58

File tree

9 files changed

+179
-163
lines changed

9 files changed

+179
-163
lines changed

Include/internal/pycore_code.h

+12-4
Original file line numberDiff line numberDiff line change
@@ -77,13 +77,20 @@ typedef struct {
7777
} _PyBinaryOpCache;
7878

7979
#define INLINE_CACHE_ENTRIES_BINARY_OP CACHE_ENTRIES(_PyBinaryOpCache)
80+
8081
typedef struct {
8182
_Py_CODEUNIT counter;
8283
} _PyUnpackSequenceCache;
8384

84-
8585
#define INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE \
86-
(sizeof(_PyUnpackSequenceCache) / sizeof(_Py_CODEUNIT))
86+
CACHE_ENTRIES(_PyUnpackSequenceCache)
87+
88+
typedef struct {
89+
_Py_CODEUNIT counter;
90+
_Py_CODEUNIT mask;
91+
} _PyCompareOpCache;
92+
93+
#define INLINE_CACHE_ENTRIES_COMPARE_OP CACHE_ENTRIES(_PyCompareOpCache)
8794

8895
/* Maximum size of code to quicken, in code units. */
8996
#define MAX_SIZE_TO_QUICKEN 5000
@@ -323,8 +330,9 @@ extern int _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int narg
323330
extern int _Py_Specialize_Precall(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
324331
PyObject *kwnames, SpecializedCacheEntry *cache, PyObject *builtins);
325332
extern void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
326-
int oparg);
327-
extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, SpecializedCacheEntry *cache);
333+
int oparg);
334+
extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs,
335+
_Py_CODEUNIT *instr, int oparg);
328336
extern void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr,
329337
int oparg);
330338

Include/opcode.h

+1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Lib/importlib/_bootstrap_external.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -389,6 +389,7 @@ def _write_atomic(path, data, mode=0o666):
389389
# Python 3.11a5 3480 (New CALL opcodes, second iteration)
390390
# Python 3.11a5 3481 (Use inline cache for BINARY_OP)
391391
# Python 3.11a5 3482 (Use inline caching for UNPACK_SEQUENCE and LOAD_GLOBAL)
392+
# Python 3.11a5 3483 (Use inline caching for COMPARE_OP)
392393

393394
# Python 3.12 will start with magic number 3500
394395

@@ -403,7 +404,7 @@ def _write_atomic(path, data, mode=0o666):
403404
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
404405
# in PC/launcher.c must also be updated.
405406

406-
MAGIC_NUMBER = (3482).to_bytes(2, 'little') + b'\r\n'
407+
MAGIC_NUMBER = (3483).to_bytes(2, 'little') + b'\r\n'
407408
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
408409

409410
_PYCACHE = '__pycache__'

Lib/opcode.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ def jabs_op(name, op, entries=0):
125125
def_op('BUILD_SET', 104) # Number of set items
126126
def_op('BUILD_MAP', 105) # Number of dict entries
127127
name_op('LOAD_ATTR', 106) # Index in name list
128-
def_op('COMPARE_OP', 107) # Comparison operator
128+
def_op('COMPARE_OP', 107, 2) # Comparison operator
129129
hascompare.append(107)
130130
name_op('IMPORT_NAME', 108) # Index in name list
131131
name_op('IMPORT_FROM', 109) # Index in name list

Lib/test/test_compile.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -1002,7 +1002,9 @@ def if_else_break():
10021002
'JUMP_FORWARD',
10031003
)
10041004

1005-
for line, instr in enumerate(dis.Bytecode(if_else_break)):
1005+
for line, instr in enumerate(
1006+
dis.Bytecode(if_else_break, show_caches=True)
1007+
):
10061008
if instr.opname == 'JUMP_FORWARD':
10071009
self.assertNotEqual(instr.arg, 0)
10081010
elif instr.opname in HANDLED_JUMPS:

Lib/test/test_dis.py

+123-123
Large diffs are not rendered by default.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Use inline caching for :opcode:`COMPARE_OP`.

Python/ceval.c

+18-15
Original file line numberDiff line numberDiff line change
@@ -3671,36 +3671,37 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
36713671
SET_TOP(res);
36723672
Py_DECREF(left);
36733673
Py_DECREF(right);
3674-
if (res == NULL)
3674+
if (res == NULL) {
36753675
goto error;
3676+
}
3677+
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
36763678
PREDICT(POP_JUMP_IF_FALSE);
36773679
PREDICT(POP_JUMP_IF_TRUE);
36783680
DISPATCH();
36793681
}
36803682

36813683
TARGET(COMPARE_OP_ADAPTIVE) {
36823684
assert(cframe.use_tracing == 0);
3683-
SpecializedCacheEntry *cache = GET_CACHE();
3684-
if (cache->adaptive.counter == 0) {
3685+
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
3686+
if (cache->counter == 0) {
36853687
PyObject *right = TOP();
36863688
PyObject *left = SECOND();
36873689
next_instr--;
3688-
_Py_Specialize_CompareOp(left, right, next_instr, cache);
3690+
_Py_Specialize_CompareOp(left, right, next_instr, oparg);
36893691
DISPATCH();
36903692
}
36913693
else {
36923694
STAT_INC(COMPARE_OP, deferred);
3693-
cache->adaptive.counter--;
3694-
oparg = cache->adaptive.original_oparg;
3695+
cache->counter--;
36953696
JUMP_TO_INSTRUCTION(COMPARE_OP);
36963697
}
36973698
}
36983699

36993700
TARGET(COMPARE_OP_FLOAT_JUMP) {
37003701
assert(cframe.use_tracing == 0);
37013702
// Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false)
3702-
SpecializedCacheEntry *caches = GET_CACHE();
3703-
int when_to_jump_mask = caches[0].adaptive.index;
3703+
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
3704+
int when_to_jump_mask = cache->mask;
37043705
PyObject *right = TOP();
37053706
PyObject *left = SECOND();
37063707
DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP);
@@ -3711,6 +3712,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
37113712
DEOPT_IF(isnan(dleft), COMPARE_OP);
37123713
DEOPT_IF(isnan(dright), COMPARE_OP);
37133714
STAT_INC(COMPARE_OP, hit);
3715+
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
37143716
NEXTOPARG();
37153717
STACK_SHRINK(2);
37163718
Py_DECREF(left);
@@ -3731,8 +3733,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
37313733
TARGET(COMPARE_OP_INT_JUMP) {
37323734
assert(cframe.use_tracing == 0);
37333735
// Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false)
3734-
SpecializedCacheEntry *caches = GET_CACHE();
3735-
int when_to_jump_mask = caches[0].adaptive.index;
3736+
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
3737+
int when_to_jump_mask = cache->mask;
37363738
PyObject *right = TOP();
37373739
PyObject *left = SECOND();
37383740
DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP);
@@ -3744,6 +3746,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
37443746
Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0];
37453747
Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0];
37463748
int sign = (ileft > iright) - (ileft < iright);
3749+
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
37473750
NEXTOPARG();
37483751
STACK_SHRINK(2);
37493752
Py_DECREF(left);
@@ -3764,8 +3767,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
37643767
TARGET(COMPARE_OP_STR_JUMP) {
37653768
assert(cframe.use_tracing == 0);
37663769
// Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false)
3767-
SpecializedCacheEntry *caches = GET_CACHE();
3768-
int invert = caches[0].adaptive.index;
3770+
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
3771+
int invert = cache->mask;
37693772
PyObject *right = TOP();
37703773
PyObject *left = SECOND();
37713774
DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP);
@@ -3775,8 +3778,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
37753778
if (res < 0) {
37763779
goto error;
37773780
}
3778-
assert(caches[0].adaptive.original_oparg == Py_EQ ||
3779-
caches[0].adaptive.original_oparg == Py_NE);
3781+
assert(oparg == Py_EQ || oparg == Py_NE);
3782+
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
37803783
NEXTOPARG();
37813784
assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE);
37823785
STACK_SHRINK(2);
@@ -5601,7 +5604,7 @@ MISS_WITH_CACHE(LOAD_METHOD)
56015604
MISS_WITH_CACHE(PRECALL)
56025605
MISS_WITH_CACHE(CALL)
56035606
MISS_WITH_INLINE_CACHE(BINARY_OP)
5604-
MISS_WITH_CACHE(COMPARE_OP)
5607+
MISS_WITH_INLINE_CACHE(COMPARE_OP)
56055608
MISS_WITH_CACHE(BINARY_SUBSCR)
56065609
MISS_WITH_INLINE_CACHE(UNPACK_SEQUENCE)
56075610
MISS_WITH_OPARG_COUNTER(STORE_SUBSCR)

Python/specialize.c

+18-18
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,6 @@ static uint8_t cache_requirements[256] = {
6565
[CALL] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */
6666
[PRECALL] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */
6767
[STORE_ATTR] = 1, // _PyAdaptiveEntry
68-
[COMPARE_OP] = 1, /* _PyAdaptiveEntry */
6968
};
7069

7170
Py_ssize_t _Py_QuickenedCount = 0;
@@ -2057,26 +2056,27 @@ static int compare_masks[] = {
20572056
};
20582057

20592058
void
2060-
_Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs,
2061-
_Py_CODEUNIT *instr, SpecializedCacheEntry *cache)
2059+
_Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
2060+
int oparg)
20622061
{
2063-
_PyAdaptiveEntry *adaptive = &cache->adaptive;
2064-
int op = adaptive->original_oparg;
2065-
int next_opcode = _Py_OPCODE(instr[1]);
2062+
assert(_PyOpcode_InlineCacheEntries[COMPARE_OP] ==
2063+
INLINE_CACHE_ENTRIES_COMPARE_OP);
2064+
_PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1);
2065+
int next_opcode = _Py_OPCODE(instr[INLINE_CACHE_ENTRIES_COMPARE_OP + 1]);
20662066
if (next_opcode != POP_JUMP_IF_FALSE && next_opcode != POP_JUMP_IF_TRUE) {
20672067
// Can't ever combine, so don't don't bother being adaptive (unless
20682068
// we're collecting stats, where it's more important to get accurate hit
20692069
// counts for the unadaptive version and each of the different failure
20702070
// types):
20712071
#ifndef Py_STATS
2072-
*instr = _Py_MAKECODEUNIT(COMPARE_OP, adaptive->original_oparg);
2072+
*instr = _Py_MAKECODEUNIT(COMPARE_OP, oparg);
20732073
return;
20742074
#endif
20752075
SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_NOT_FOLLOWED_BY_COND_JUMP);
20762076
goto failure;
20772077
}
2078-
assert(op <= Py_GE);
2079-
int when_to_jump_mask = compare_masks[op];
2078+
assert(oparg <= Py_GE);
2079+
int when_to_jump_mask = compare_masks[oparg];
20802080
if (next_opcode == POP_JUMP_IF_FALSE) {
20812081
when_to_jump_mask = (1 | 2 | 4) & ~when_to_jump_mask;
20822082
}
@@ -2085,14 +2085,14 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs,
20852085
goto failure;
20862086
}
20872087
if (PyFloat_CheckExact(lhs)) {
2088-
*instr = _Py_MAKECODEUNIT(COMPARE_OP_FLOAT_JUMP, _Py_OPARG(*instr));
2089-
adaptive->index = when_to_jump_mask;
2088+
*instr = _Py_MAKECODEUNIT(COMPARE_OP_FLOAT_JUMP, oparg);
2089+
cache->mask = when_to_jump_mask;
20902090
goto success;
20912091
}
20922092
if (PyLong_CheckExact(lhs)) {
20932093
if (Py_ABS(Py_SIZE(lhs)) <= 1 && Py_ABS(Py_SIZE(rhs)) <= 1) {
2094-
*instr = _Py_MAKECODEUNIT(COMPARE_OP_INT_JUMP, _Py_OPARG(*instr));
2095-
adaptive->index = when_to_jump_mask;
2094+
*instr = _Py_MAKECODEUNIT(COMPARE_OP_INT_JUMP, oparg);
2095+
cache->mask = when_to_jump_mask;
20962096
goto success;
20972097
}
20982098
else {
@@ -2101,24 +2101,24 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs,
21012101
}
21022102
}
21032103
if (PyUnicode_CheckExact(lhs)) {
2104-
if (op != Py_EQ && op != Py_NE) {
2104+
if (oparg != Py_EQ && oparg != Py_NE) {
21052105
SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_STRING);
21062106
goto failure;
21072107
}
21082108
else {
2109-
*instr = _Py_MAKECODEUNIT(COMPARE_OP_STR_JUMP, _Py_OPARG(*instr));
2110-
adaptive->index = (when_to_jump_mask & 2) == 0;
2109+
*instr = _Py_MAKECODEUNIT(COMPARE_OP_STR_JUMP, oparg);
2110+
cache->mask = (when_to_jump_mask & 2) == 0;
21112111
goto success;
21122112
}
21132113
}
21142114
SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs));
21152115
failure:
21162116
STAT_INC(COMPARE_OP, failure);
2117-
cache_backoff(adaptive);
2117+
cache->counter = ADAPTIVE_CACHE_BACKOFF;
21182118
return;
21192119
success:
21202120
STAT_INC(COMPARE_OP, success);
2121-
adaptive->counter = initial_counter_value();
2121+
cache->counter = initial_counter_value();
21222122
}
21232123

21242124
#ifdef Py_STATS

0 commit comments

Comments
 (0)