Update Python from 3.11.8 to 3.12.2

1 files changed, 817 insertions, 655 deletions

diff --git a/contrib/tools/python3/src/Python/specialize.c b/contrib/tools/python3/src/Python/specialize.c
index 3441e844e1..2c0d99b04e 100644
--- a/contrib/tools/python3/src/Python/specialize.c
+++ b/contrib/tools/python3/src/Python/specialize.c
@@ -8,6 +8,7 @@
 #include "pycore_object.h"
 #include "pycore_opcode.h"        // _PyOpcode_Caches
 #include "structmember.h"         // struct PyMemberDef, T_OFFSET_EX
+#include "pycore_descrobject.h"
 
 #include <stdlib.h> // rand()
 
@@ -15,25 +16,9 @@
  * ./adaptive.md
  */
 
-/* Map from opcode to adaptive opcode.
-  Values of zero are ignored. */
-uint8_t _PyOpcode_Adaptive[256] = {
-    [LOAD_ATTR] = LOAD_ATTR_ADAPTIVE,
-    [LOAD_GLOBAL] = LOAD_GLOBAL_ADAPTIVE,
-    [LOAD_METHOD] = LOAD_METHOD_ADAPTIVE,
-    [BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE,
-    [STORE_SUBSCR] = STORE_SUBSCR_ADAPTIVE,
-    [CALL] = CALL_ADAPTIVE,
-    [PRECALL] = PRECALL_ADAPTIVE,
-    [STORE_ATTR] = STORE_ATTR_ADAPTIVE,
-    [BINARY_OP] = BINARY_OP_ADAPTIVE,
-    [COMPARE_OP] = COMPARE_OP_ADAPTIVE,
-    [UNPACK_SEQUENCE] = UNPACK_SEQUENCE_ADAPTIVE,
-};
-
-Py_ssize_t _Py_QuickenedCount = 0;
 #ifdef Py_STATS
-PyStats _py_stats = { 0 };
+PyStats _py_stats_struct = { 0 };
+PyStats *_py_stats = NULL;
 
 #define ADD_STAT_TO_DICT(res, field) \
     do { \
@@ -93,7 +78,7 @@ add_stat_dict(
     int opcode,
     const char *name) {
 
-    SpecializationStats *stats = &_py_stats.opcode_stats[opcode].specialization;
+    SpecializationStats *stats = &_py_stats_struct.opcode_stats[opcode].specialization;
     PyObject *d = stats_to_dict(stats);
     if (d == NULL) {
         return -1;
@@ -111,9 +96,9 @@ _Py_GetSpecializationStats(void) {
         return NULL;
     }
     int err = 0;
+    err += add_stat_dict(stats, LOAD_SUPER_ATTR, "load_super_attr");
     err += add_stat_dict(stats, LOAD_ATTR, "load_attr");
     err += add_stat_dict(stats, LOAD_GLOBAL, "load_global");
-    err += add_stat_dict(stats, LOAD_METHOD, "load_method");
     err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr");
     err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr");
     err += add_stat_dict(stats, STORE_ATTR, "store_attr");
@@ -121,7 +106,7 @@ _Py_GetSpecializationStats(void) {
     err += add_stat_dict(stats, BINARY_OP, "binary_op");
     err += add_stat_dict(stats, COMPARE_OP, "compare_op");
     err += add_stat_dict(stats, UNPACK_SEQUENCE, "unpack_sequence");
-    err += add_stat_dict(stats, PRECALL, "precall");
+    err += add_stat_dict(stats, FOR_ITER, "for_iter");
     if (err < 0) {
         Py_DECREF(stats);
         return NULL;
@@ -141,9 +126,12 @@ print_spec_stats(FILE *out, OpcodeStats *stats)
 {
     /* Mark some opcodes as specializable for stats,
      * even though we don't specialize them yet. */
-    fprintf(out, "opcode[%d].specializable : 1\n", FOR_ITER);
+    fprintf(out, "opcode[%d].specializable : 1\n", BINARY_SLICE);
+    fprintf(out, "opcode[%d].specializable : 1\n", COMPARE_OP);
+    fprintf(out, "opcode[%d].specializable : 1\n", STORE_SLICE);
+    fprintf(out, "opcode[%d].specializable : 1\n", SEND);
     for (int i = 0; i < 256; i++) {
-        if (_PyOpcode_Adaptive[i]) {
+        if (_PyOpcode_Caches[i]) {
             fprintf(out, "opcode[%d].specializable : 1\n", i);
         }
         PRINT_STAT(i, specialization.success);
@@ -160,7 +148,7 @@ print_spec_stats(FILE *out, OpcodeStats *stats)
                     PRIu64 "\n", i, j, val);
             }
         }
-        for(int j = 0; j < 256; j++) {
+        for (int j = 0; j < 256; j++) {
             if (stats[i].pair_count[j]) {
                 fprintf(out, "opcode[%d].pair_count[%d] : %" PRIu64 "\n",
                         i, j, stats[i].pair_count[j]);
@@ -178,6 +166,9 @@ print_call_stats(FILE *out, CallStats *stats)
     fprintf(out, "Calls to Python functions inlined: %" PRIu64 "\n", stats->inlined_py_calls);
     fprintf(out, "Frames pushed: %" PRIu64 "\n", stats->frames_pushed);
     fprintf(out, "Frame objects created: %" PRIu64 "\n", stats->frame_objects_created);
+    for (int i = 0; i < EVAL_CALL_KINDS; i++) {
+        fprintf(out, "Calls via PyEval_EvalFrame[%d] : %" PRIu64 "\n", i, stats->eval_calls[i]);
+    }
 }
 
 static void
@@ -191,10 +182,19 @@ print_object_stats(FILE *out, ObjectStats *stats)
     fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big);
     fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees);
     fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values);
+    fprintf(out, "Object interpreter increfs: %" PRIu64 "\n", stats->interpreter_increfs);
+    fprintf(out, "Object interpreter decrefs: %" PRIu64 "\n", stats->interpreter_decrefs);
+    fprintf(out, "Object increfs: %" PRIu64 "\n", stats->increfs);
+    fprintf(out, "Object decrefs: %" PRIu64 "\n", stats->decrefs);
     fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request);
     fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key);
     fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big);
     fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass);
+    fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits);
+    fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses);
+    fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions);
+    fprintf(out, "Object method cache dunder hits: %" PRIu64 "\n", stats->type_cache_dunder_hits);
+    fprintf(out, "Object method cache dunder misses: %" PRIu64 "\n", stats->type_cache_dunder_misses);
 }
 
 static void
@@ -205,6 +205,12 @@ print_stats(FILE *out, PyStats *stats) {
 }
 
 void
+_Py_StatsClear(void)
+{
+    _py_stats_struct = (PyStats) { 0 };
+}
+
+void
 _Py_PrintSpecializationStats(int to_file)
 {
     FILE *out = stderr;
@@ -237,7 +243,7 @@ _Py_PrintSpecializationStats(int to_file)
     else {
         fprintf(out, "Specialization stats:\n");
     }
-    print_stats(out, &_py_stats);
+    print_stats(out, &_py_stats_struct);
     if (out != stderr) {
         fclose(out);
     }
@@ -245,8 +251,12 @@ _Py_PrintSpecializationStats(int to_file)
 
 #ifdef Py_STATS
 
-#define SPECIALIZATION_FAIL(opcode, kind) _py_stats.opcode_stats[opcode].specialization.failure_kinds[kind]++
-
+#define SPECIALIZATION_FAIL(opcode, kind) \
+do { \
+    if (_py_stats) { \
+        _py_stats->opcode_stats[opcode].specialization.failure_kinds[kind]++; \
+    } \
+} while (0)
 
 #endif
 #endif
@@ -255,81 +265,45 @@ _Py_PrintSpecializationStats(int to_file)
 #define SPECIALIZATION_FAIL(opcode, kind) ((void)0)
 #endif
 
-// Insert adaptive instructions and superinstructions. This cannot fail.
+// Initialize warmup counters and insert superinstructions. This cannot fail.
 void
 _PyCode_Quicken(PyCodeObject *code)
 {
-    _Py_QuickenedCount++;
-    int previous_opcode = -1;
+    #if ENABLE_SPECIALIZATION
+    int opcode = 0;
     _Py_CODEUNIT *instructions = _PyCode_CODE(code);
     for (int i = 0; i < Py_SIZE(code); i++) {
-        int opcode = _Py_OPCODE(instructions[i]);
-        uint8_t adaptive_opcode = _PyOpcode_Adaptive[opcode];
-        if (adaptive_opcode) {
-            _Py_SET_OPCODE(instructions[i], adaptive_opcode);
-            // Make sure the adaptive counter is zero:
-            assert(instructions[i + 1] == 0);
-            previous_opcode = -1;
-            i += _PyOpcode_Caches[opcode];
-        }
-        else {
-            assert(!_PyOpcode_Caches[opcode]);
-            switch (opcode) {
-                case EXTENDED_ARG:
-                    _Py_SET_OPCODE(instructions[i], EXTENDED_ARG_QUICK);
-                    break;
-                case JUMP_BACKWARD:
-                    _Py_SET_OPCODE(instructions[i], JUMP_BACKWARD_QUICK);
-                    break;
-                case RESUME:
-                    _Py_SET_OPCODE(instructions[i], RESUME_QUICK);
-                    break;
-                case LOAD_FAST:
-                    switch(previous_opcode) {
-                        case LOAD_FAST:
-                            _Py_SET_OPCODE(instructions[i - 1],
-                                           LOAD_FAST__LOAD_FAST);
-                            break;
-                        case STORE_FAST:
-                            _Py_SET_OPCODE(instructions[i - 1],
-                                           STORE_FAST__LOAD_FAST);
-                            break;
-                        case LOAD_CONST:
-                            _Py_SET_OPCODE(instructions[i - 1],
-                                           LOAD_CONST__LOAD_FAST);
-                            break;
-                    }
-                    break;
-                case STORE_FAST:
-                    if (previous_opcode == STORE_FAST) {
-                        _Py_SET_OPCODE(instructions[i - 1],
-                                       STORE_FAST__STORE_FAST);
-                    }
-                    break;
-                case LOAD_CONST:
-                    if (previous_opcode == LOAD_FAST) {
-                        _Py_SET_OPCODE(instructions[i - 1],
-                                       LOAD_FAST__LOAD_CONST);
-                    }
-                    break;
-            }
-            previous_opcode = opcode;
+        int previous_opcode = opcode;
+        opcode = _Py_GetBaseOpcode(code, i);
+        assert(opcode < MIN_INSTRUMENTED_OPCODE);
+        int caches = _PyOpcode_Caches[opcode];
+        if (caches) {
+            instructions[i + 1].cache = adaptive_counter_warmup();
+            i += caches;
+            continue;
+        }
+        switch (previous_opcode << 8 | opcode) {
+            case LOAD_CONST << 8 | LOAD_FAST:
+                instructions[i - 1].op.code = LOAD_CONST__LOAD_FAST;
+                break;
+            case LOAD_FAST << 8 | LOAD_CONST:
+                instructions[i - 1].op.code = LOAD_FAST__LOAD_CONST;
+                break;
+            case LOAD_FAST << 8 | LOAD_FAST:
+                instructions[i - 1].op.code = LOAD_FAST__LOAD_FAST;
+                break;
+            case STORE_FAST << 8 | LOAD_FAST:
+                instructions[i - 1].op.code = STORE_FAST__LOAD_FAST;
+                break;
+            case STORE_FAST << 8 | STORE_FAST:
+                instructions[i - 1].op.code = STORE_FAST__STORE_FAST;
+                break;
         }
     }
+    #endif /* ENABLE_SPECIALIZATION */
 }
 
-static inline int
-miss_counter_start(void) {
-    /* Starting value for the counter.
-     * This value needs to be not too low, otherwise
-     * it would cause excessive de-optimization.
-     * Neither should it be too high, or that would delay
-     * de-optimization excessively when it is needed.
-     * A value around 50 seems to work, and we choose a
-     * prime number to avoid artifacts.
-     */
-    return 53;
-}
+#define SIMPLE_FUNCTION 0
 
 /* Common */
 
@@ -340,51 +314,55 @@ miss_counter_start(void) {
 #define SPEC_FAIL_OUT_OF_RANGE 4
 #define SPEC_FAIL_EXPECTED_ERROR 5
 #define SPEC_FAIL_WRONG_NUMBER_ARGUMENTS 6
+#define SPEC_FAIL_CODE_COMPLEX_PARAMETERS 7
+#define SPEC_FAIL_CODE_NOT_OPTIMIZED 8
 
+
+#define SPEC_FAIL_LOAD_GLOBAL_NON_DICT 17
 #define SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT 18
 
+/* Super */
+
+#define SPEC_FAIL_SUPER_BAD_CLASS 9
+#define SPEC_FAIL_SUPER_SHADOWED 10
+
 /* Attributes */
 
-#define SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR 8
-#define SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR 9
-#define SPEC_FAIL_ATTR_NOT_DESCRIPTOR 10
-#define SPEC_FAIL_ATTR_METHOD 11
-#define SPEC_FAIL_ATTR_MUTABLE_CLASS 12
-#define SPEC_FAIL_ATTR_PROPERTY 13
-#define SPEC_FAIL_ATTR_NON_OBJECT_SLOT 14
-#define SPEC_FAIL_ATTR_READ_ONLY 15
-#define SPEC_FAIL_ATTR_AUDITED_SLOT 16
-#define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 17
-#define SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT 18
-#define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 19
-
-/* Methods */
-
-#define SPEC_FAIL_LOAD_METHOD_OVERRIDING_DESCRIPTOR 8
-#define SPEC_FAIL_LOAD_METHOD_NON_OVERRIDING_DESCRIPTOR 9
-#define SPEC_FAIL_LOAD_METHOD_NOT_DESCRIPTOR 10
-#define SPEC_FAIL_LOAD_METHOD_METHOD 11
-#define SPEC_FAIL_LOAD_METHOD_MUTABLE_CLASS 12
-#define SPEC_FAIL_LOAD_METHOD_PROPERTY 13
-#define SPEC_FAIL_LOAD_METHOD_NON_OBJECT_SLOT 14
-#define SPEC_FAIL_LOAD_METHOD_IS_ATTR 15
-#define SPEC_FAIL_LOAD_METHOD_DICT_SUBCLASS 16
-#define SPEC_FAIL_LOAD_METHOD_BUILTIN_CLASS_METHOD 17
-#define SPEC_FAIL_LOAD_METHOD_CLASS_METHOD_OBJ 18
-#define SPEC_FAIL_LOAD_METHOD_OBJECT_SLOT 19
-#define SPEC_FAIL_LOAD_METHOD_HAS_DICT 20
-#define SPEC_FAIL_LOAD_METHOD_HAS_MANAGED_DICT 21
-#define SPEC_FAIL_LOAD_METHOD_INSTANCE_ATTRIBUTE 22
-#define SPEC_FAIL_LOAD_METHOD_METACLASS_ATTRIBUTE 23
+#define SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR 9
+#define SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR 10
+#define SPEC_FAIL_ATTR_NOT_DESCRIPTOR 11
+#define SPEC_FAIL_ATTR_METHOD 12
+#define SPEC_FAIL_ATTR_MUTABLE_CLASS 13
+#define SPEC_FAIL_ATTR_PROPERTY 14
+#define SPEC_FAIL_ATTR_NON_OBJECT_SLOT 15
+#define SPEC_FAIL_ATTR_READ_ONLY 16
+#define SPEC_FAIL_ATTR_AUDITED_SLOT 17
+#define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 18
+#define SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT 19
+#define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 20
+
+#define SPEC_FAIL_ATTR_SHADOWED 21
+#define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22
+#define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23
+#define SPEC_FAIL_ATTR_OBJECT_SLOT 24
+#define SPEC_FAIL_ATTR_HAS_MANAGED_DICT 25
+#define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26
+#define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27
+#define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28
+#define SPEC_FAIL_ATTR_NOT_IN_KEYS 29
+#define SPEC_FAIL_ATTR_NOT_IN_DICT 30
+#define SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE 31
+#define SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR 32
+#define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ 33
 
 /* Binary subscr and store subscr */
 
-#define SPEC_FAIL_SUBSCR_ARRAY_INT 8
-#define SPEC_FAIL_SUBSCR_ARRAY_SLICE 9
-#define SPEC_FAIL_SUBSCR_LIST_SLICE 10
-#define SPEC_FAIL_SUBSCR_TUPLE_SLICE 11
-#define SPEC_FAIL_SUBSCR_STRING_INT 12
-#define SPEC_FAIL_SUBSCR_STRING_SLICE 13
+#define SPEC_FAIL_SUBSCR_ARRAY_INT 9
+#define SPEC_FAIL_SUBSCR_ARRAY_SLICE 10
+#define SPEC_FAIL_SUBSCR_LIST_SLICE 11
+#define SPEC_FAIL_SUBSCR_TUPLE_SLICE 12
+#define SPEC_FAIL_SUBSCR_STRING_INT 13
+#define SPEC_FAIL_SUBSCR_STRING_SLICE 14
 #define SPEC_FAIL_SUBSCR_BUFFER_INT 15
 #define SPEC_FAIL_SUBSCR_BUFFER_SLICE 16
 #define SPEC_FAIL_SUBSCR_SEQUENCE_INT 17
@@ -399,128 +377,135 @@ miss_counter_start(void) {
 
 /* Binary op */
 
-#define SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES          8
-#define SPEC_FAIL_BINARY_OP_ADD_OTHER                    9
-#define SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES         10
-#define SPEC_FAIL_BINARY_OP_AND_INT                     11
-#define SPEC_FAIL_BINARY_OP_AND_OTHER                   12
-#define SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE                13
-#define SPEC_FAIL_BINARY_OP_LSHIFT                      14
-#define SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY             15
-#define SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES    16
-#define SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER              17
-#define SPEC_FAIL_BINARY_OP_OR                          18
-#define SPEC_FAIL_BINARY_OP_POWER                       19
-#define SPEC_FAIL_BINARY_OP_REMAINDER                   20
-#define SPEC_FAIL_BINARY_OP_RSHIFT                      21
-#define SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES    22
-#define SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER              23
-#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES 24
-#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT           25
-#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER           26
-#define SPEC_FAIL_BINARY_OP_XOR                         27
+#define SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES          9
+#define SPEC_FAIL_BINARY_OP_ADD_OTHER                   10
+#define SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES         11
+#define SPEC_FAIL_BINARY_OP_AND_INT                     12
+#define SPEC_FAIL_BINARY_OP_AND_OTHER                   13
+#define SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE                14
+#define SPEC_FAIL_BINARY_OP_LSHIFT                      15
+#define SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY             16
+#define SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES    17
+#define SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER              18
+#define SPEC_FAIL_BINARY_OP_OR                          19
+#define SPEC_FAIL_BINARY_OP_POWER                       20
+#define SPEC_FAIL_BINARY_OP_REMAINDER                   21
+#define SPEC_FAIL_BINARY_OP_RSHIFT                      22
+#define SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES    23
+#define SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER              24
+#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES 25
+#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT           26
+#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER           27
+#define SPEC_FAIL_BINARY_OP_XOR                         28
 
 /* Calls */
-#define SPEC_FAIL_CALL_COMPLEX_PARAMETERS 9
-#define SPEC_FAIL_CALL_CO_NOT_OPTIMIZED 10
-/* SPEC_FAIL_METHOD  defined as 11 above */
 
 #define SPEC_FAIL_CALL_INSTANCE_METHOD 11
 #define SPEC_FAIL_CALL_CMETHOD 12
-#define SPEC_FAIL_CALL_PYCFUNCTION 13
-#define SPEC_FAIL_CALL_PYCFUNCTION_WITH_KEYWORDS 14
-#define SPEC_FAIL_CALL_PYCFUNCTION_FAST_WITH_KEYWORDS 15
-#define SPEC_FAIL_CALL_PYCFUNCTION_NOARGS 16
-#define SPEC_FAIL_CALL_BAD_CALL_FLAGS 17
-#define SPEC_FAIL_CALL_CLASS 18
-#define SPEC_FAIL_CALL_PYTHON_CLASS 19
-#define SPEC_FAIL_CALL_METHOD_DESCRIPTOR 20
-#define SPEC_FAIL_CALL_BOUND_METHOD 21
-#define SPEC_FAIL_CALL_STR 22
-#define SPEC_FAIL_CALL_CLASS_NO_VECTORCALL 23
-#define SPEC_FAIL_CALL_CLASS_MUTABLE 24
-#define SPEC_FAIL_CALL_KWNAMES 25
-#define SPEC_FAIL_CALL_METHOD_WRAPPER 26
-#define SPEC_FAIL_CALL_OPERATOR_WRAPPER 27
-#define SPEC_FAIL_CALL_PYFUNCTION 28
-#define SPEC_FAIL_CALL_PEP_523 29
+#define SPEC_FAIL_CALL_CFUNC_VARARGS 13
+#define SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS 14
+#define SPEC_FAIL_CALL_CFUNC_NOARGS 15
+#define SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS 16
+#define SPEC_FAIL_CALL_METH_DESCR_VARARGS 17
+#define SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS 18
+#define SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS 19
+#define SPEC_FAIL_CALL_BAD_CALL_FLAGS 20
+#define SPEC_FAIL_CALL_PYTHON_CLASS 21
+#define SPEC_FAIL_CALL_PEP_523 22
+#define SPEC_FAIL_CALL_BOUND_METHOD 23
+#define SPEC_FAIL_CALL_STR 24
+#define SPEC_FAIL_CALL_CLASS_NO_VECTORCALL 25
+#define SPEC_FAIL_CALL_CLASS_MUTABLE 26
+#define SPEC_FAIL_CALL_KWNAMES 27
+#define SPEC_FAIL_CALL_METHOD_WRAPPER 28
+#define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29
 
 /* COMPARE_OP */
 #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12
 #define SPEC_FAIL_COMPARE_OP_STRING 13
-#define SPEC_FAIL_COMPARE_OP_NOT_FOLLOWED_BY_COND_JUMP 14
-#define SPEC_FAIL_COMPARE_OP_BIG_INT 15
-#define SPEC_FAIL_COMPARE_OP_BYTES 16
-#define SPEC_FAIL_COMPARE_OP_TUPLE 17
-#define SPEC_FAIL_COMPARE_OP_LIST 18
-#define SPEC_FAIL_COMPARE_OP_SET 19
-#define SPEC_FAIL_COMPARE_OP_BOOL 20
-#define SPEC_FAIL_COMPARE_OP_BASEOBJECT 21
-#define SPEC_FAIL_COMPARE_OP_FLOAT_LONG 22
-#define SPEC_FAIL_COMPARE_OP_LONG_FLOAT 23
-#define SPEC_FAIL_COMPARE_OP_EXTENDED_ARG 24
-
-/* FOR_ITER */
-#define SPEC_FAIL_FOR_ITER_GENERATOR 10
-#define SPEC_FAIL_FOR_ITER_COROUTINE 11
-#define SPEC_FAIL_FOR_ITER_ASYNC_GENERATOR 12
-#define SPEC_FAIL_FOR_ITER_LIST 13
-#define SPEC_FAIL_FOR_ITER_TUPLE 14
-#define SPEC_FAIL_FOR_ITER_SET 15
-#define SPEC_FAIL_FOR_ITER_STRING 16
-#define SPEC_FAIL_FOR_ITER_BYTES 17
-#define SPEC_FAIL_FOR_ITER_RANGE 18
-#define SPEC_FAIL_FOR_ITER_ITERTOOLS 19
-#define SPEC_FAIL_FOR_ITER_DICT_KEYS 20
-#define SPEC_FAIL_FOR_ITER_DICT_ITEMS 21
-#define SPEC_FAIL_FOR_ITER_DICT_VALUES 22
-#define SPEC_FAIL_FOR_ITER_ENUMERATE 23
+#define SPEC_FAIL_COMPARE_OP_BIG_INT 14
+#define SPEC_FAIL_COMPARE_OP_BYTES 15
+#define SPEC_FAIL_COMPARE_OP_TUPLE 16
+#define SPEC_FAIL_COMPARE_OP_LIST 17
+#define SPEC_FAIL_COMPARE_OP_SET 18
+#define SPEC_FAIL_COMPARE_OP_BOOL 19
+#define SPEC_FAIL_COMPARE_OP_BASEOBJECT 20
+#define SPEC_FAIL_COMPARE_OP_FLOAT_LONG 21
+#define SPEC_FAIL_COMPARE_OP_LONG_FLOAT 22
+
+/* FOR_ITER and SEND */
+#define SPEC_FAIL_ITER_GENERATOR 10
+#define SPEC_FAIL_ITER_COROUTINE 11
+#define SPEC_FAIL_ITER_ASYNC_GENERATOR 12
+#define SPEC_FAIL_ITER_LIST 13
+#define SPEC_FAIL_ITER_TUPLE 14
+#define SPEC_FAIL_ITER_SET 15
+#define SPEC_FAIL_ITER_STRING 16
+#define SPEC_FAIL_ITER_BYTES 17
+#define SPEC_FAIL_ITER_RANGE 18
+#define SPEC_FAIL_ITER_ITERTOOLS 19
+#define SPEC_FAIL_ITER_DICT_KEYS 20
+#define SPEC_FAIL_ITER_DICT_ITEMS 21
+#define SPEC_FAIL_ITER_DICT_VALUES 22
+#define SPEC_FAIL_ITER_ENUMERATE 23
+#define SPEC_FAIL_ITER_MAP 24
+#define SPEC_FAIL_ITER_ZIP 25
+#define SPEC_FAIL_ITER_SEQ_ITER 26
+#define SPEC_FAIL_ITER_REVERSED_LIST 27
+#define SPEC_FAIL_ITER_CALLABLE 28
+#define SPEC_FAIL_ITER_ASCII_STRING 29
+#define SPEC_FAIL_ITER_ASYNC_GENERATOR_SEND 30
 
 // UNPACK_SEQUENCE
 
-#define SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR 8
-#define SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE 9
+#define SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR 9
+#define SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE 10
 
+static int function_kind(PyCodeObject *code);
+static bool function_check_args(PyObject *o, int expected_argcount, int opcode);
+static uint32_t function_get_version(PyObject *o, int opcode);
 static uint32_t type_get_version(PyTypeObject *t, int opcode);
 
 static int
-specialize_module_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
-                            PyObject *name, int opcode, int opcode_module)
-{
+specialize_module_load_attr(
+    PyObject *owner, _Py_CODEUNIT *instr, PyObject *name
+) {
     _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
     PyModuleObject *m = (PyModuleObject *)owner;
-    PyObject *value = NULL;
     assert((owner->ob_type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);
     PyDictObject *dict = (PyDictObject *)m->md_dict;
     if (dict == NULL) {
-        SPECIALIZATION_FAIL(opcode, SPEC_FAIL_NO_DICT);
+        SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_NO_DICT);
         return -1;
     }
     if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
-        SPECIALIZATION_FAIL(opcode, SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT);
+        SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT);
         return -1;
     }
-    Py_ssize_t index = _PyDict_GetItemHint(dict, &_Py_ID(__getattr__), -1,
-                                           &value);
+    Py_ssize_t index = _PyDict_LookupIndex(dict, &_Py_ID(__getattr__));
     assert(index != DKIX_ERROR);
     if (index != DKIX_EMPTY) {
-        SPECIALIZATION_FAIL(opcode, SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND);
+        SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND);
         return -1;
     }
-    index = _PyDict_GetItemHint(dict, name, -1, &value);
+    index = _PyDict_LookupIndex(dict, name);
     assert (index != DKIX_ERROR);
     if (index != (uint16_t)index) {
-        SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_RANGE);
+        SPECIALIZATION_FAIL(LOAD_ATTR,
+                            index == DKIX_EMPTY ?
+                            SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND :
+                            SPEC_FAIL_OUT_OF_RANGE);
         return -1;
     }
-    uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(dict->ma_keys);
+    uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(
+            _PyInterpreterState_GET(), dict->ma_keys);
     if (keys_version == 0) {
-        SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS);
+        SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
         return -1;
     }
     write_u32(cache->version, keys_version);
     cache->index = (uint16_t)index;
-    _Py_SET_OPCODE(*instr, opcode_module);
+    instr->op.code = LOAD_ATTR_MODULE;
     return 0;
 }
 
@@ -528,6 +513,34 @@ specialize_module_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
 
 /* Attribute specialization */
 
+void
+_Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, _Py_CODEUNIT *instr, int load_method) {
+    assert(ENABLE_SPECIALIZATION);
+    assert(_PyOpcode_Caches[LOAD_SUPER_ATTR] == INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR);
+    _PySuperAttrCache *cache = (_PySuperAttrCache *)(instr + 1);
+    if (global_super != (PyObject *)&PySuper_Type) {
+        SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_SHADOWED);
+        goto fail;
+    }
+    if (!PyType_Check(cls)) {
+        SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_BAD_CLASS);
+        goto fail;
+    }
+    instr->op.code = load_method ? LOAD_SUPER_ATTR_METHOD : LOAD_SUPER_ATTR_ATTR;
+    goto success;
+
+fail:
+    STAT_INC(LOAD_SUPER_ATTR, failure);
+    assert(!PyErr_Occurred());
+    instr->op.code = LOAD_SUPER_ATTR;
+    cache->counter = adaptive_counter_backoff(cache->counter);
+    return;
+success:
+    STAT_INC(LOAD_SUPER_ATTR, success);
+    assert(!PyErr_Occurred());
+    cache->counter = adaptive_counter_cooldown();
+}
+
 typedef enum {
     OVERRIDING, /* Is an overriding descriptor, and will remain so. */
     METHOD, /* Attribute has Py_TPFLAGS_METHOD_DESCRIPTOR set */
@@ -541,13 +554,15 @@ typedef enum {
     MUTABLE,   /* Instance of a mutable class; might, or might not, be a descriptor */
     ABSENT, /* Attribute is not present on the class */
     DUNDER_CLASS, /* __class__ attribute */
-    GETSET_OVERRIDDEN /* __getattribute__ or __setattr__ has been overridden */
+    GETSET_OVERRIDDEN, /* __getattribute__ or __setattr__ has been overridden */
+    GETATTRIBUTE_IS_PYTHON_FUNCTION  /* Descriptor requires calling a Python __getattribute__ */
 } DescriptorClassification;
 
 
 static DescriptorClassification
 analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int store)
 {
+    bool has_getattr = false;
     if (store) {
         if (type->tp_setattro != PyObject_GenericSetAttr) {
             *descr = NULL;
@@ -555,7 +570,42 @@ analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int sto
         }
     }
     else {
-        if (type->tp_getattro != PyObject_GenericGetAttr) {
+        getattrofunc getattro_slot = type->tp_getattro;
+        if (getattro_slot == PyObject_GenericGetAttr) {
+            /* Normal attribute lookup; */
+            has_getattr = false;
+        }
+        else if (getattro_slot == _Py_slot_tp_getattr_hook ||
+            getattro_slot == _Py_slot_tp_getattro) {
+            /* One or both of __getattribute__ or __getattr__ may have been
+             overridden See typeobject.c for why these functions are special. */
+            PyObject *getattribute = _PyType_Lookup(type,
+                &_Py_ID(__getattribute__));
+            PyInterpreterState *interp = _PyInterpreterState_GET();
+            bool has_custom_getattribute = getattribute != NULL &&
+                getattribute != interp->callable_cache.object__getattribute__;
+            has_getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)) != NULL;
+            if (has_custom_getattribute) {
+                if (getattro_slot == _Py_slot_tp_getattro &&
+                    !has_getattr &&
+                    Py_IS_TYPE(getattribute, &PyFunction_Type)) {
+                    *descr = getattribute;
+                    return GETATTRIBUTE_IS_PYTHON_FUNCTION;
+                }
+                /* Potentially both __getattr__ and __getattribute__ are set.
+                   Too complicated */
+                *descr = NULL;
+                return GETSET_OVERRIDDEN;
+            }
+            /* Potentially has __getattr__ but no custom __getattribute__.
+               Fall through to usual descriptor analysis.
+               Usual attribute lookup should only be allowed at runtime
+               if we can guarantee that there is no way an exception can be
+               raised. This means some specializations, e.g. specializing
+               for property() isn't safe.
+            */
+        }
+        else {
             *descr = NULL;
             return GETSET_OVERRIDDEN;
         }
@@ -579,14 +629,19 @@ analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int sto
             return OTHER_SLOT;
         }
         if (desc_cls == &PyProperty_Type) {
-            return PROPERTY;
+            /* We can't detect at runtime whether an attribute exists
+               with property. So that means we may have to call
+               __getattr__. */
+            return has_getattr ? GETSET_OVERRIDDEN : PROPERTY;
         }
         if (PyUnicode_CompareWithASCIIString(name, "__class__") == 0) {
             if (descriptor == _PyType_Lookup(&PyBaseObject_Type, name)) {
                 return DUNDER_CLASS;
             }
         }
-        return OVERRIDING;
+        if (store) {
+            return OVERRIDING;
+        }
     }
     if (desc_cls->tp_descr_get) {
         if (desc_cls->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR) {
@@ -617,63 +672,81 @@ specialize_dict_access(
         return 0;
     }
     _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
-    PyObject **dictptr = _PyObject_ManagedDictPointer(owner);
-    PyDictObject *dict = (PyDictObject *)*dictptr;
-    if (dict == NULL) {
+    PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
+    if (_PyDictOrValues_IsValues(dorv)) {
         // Virtual dictionary
         PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
         assert(PyUnicode_CheckExact(name));
         Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
         assert (index != DKIX_ERROR);
         if (index != (uint16_t)index) {
-            SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
+            SPECIALIZATION_FAIL(base_op,
+                                index == DKIX_EMPTY ?
+                                SPEC_FAIL_ATTR_NOT_IN_KEYS :
+                                SPEC_FAIL_OUT_OF_RANGE);
             return 0;
         }
         write_u32(cache->version, type->tp_version_tag);
         cache->index = (uint16_t)index;
-        _Py_SET_OPCODE(*instr, values_op);
+        instr->op.code = values_op;
     }
     else {
-        if (!PyDict_CheckExact(dict)) {
+        PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
+        if (dict == NULL || !PyDict_CheckExact(dict)) {
             SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT);
             return 0;
         }
         // We found an instance with a __dict__.
-        PyObject *value = NULL;
-        Py_ssize_t hint =
-            _PyDict_GetItemHint(dict, name, -1, &value);
-        if (hint != (uint16_t)hint) {
-            SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
+        Py_ssize_t index =
+            _PyDict_LookupIndex(dict, name);
+        if (index != (uint16_t)index) {
+            SPECIALIZATION_FAIL(base_op,
+                                index == DKIX_EMPTY ?
+                                SPEC_FAIL_ATTR_NOT_IN_DICT :
+                                SPEC_FAIL_OUT_OF_RANGE);
             return 0;
         }
-        cache->index = (uint16_t)hint;
+        cache->index = (uint16_t)index;
         write_u32(cache->version, type->tp_version_tag);
-        _Py_SET_OPCODE(*instr, hint_op);
+        instr->op.code = hint_op;
     }
     return 1;
 }
 
-int
+static int specialize_attr_loadmethod(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name,
+    PyObject* descr, DescriptorClassification kind);
+static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name);
+
+void
 _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
 {
+    assert(ENABLE_SPECIALIZATION);
     assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR);
     _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
+    PyTypeObject *type = Py_TYPE(owner);
+    if (!_PyType_IsReady(type)) {
+        // We *might* not really need this check, but we inherited it from
+        // PyObject_GenericGetAttr and friends... and this way we still do the
+        // right thing if someone forgets to call PyType_Ready(type):
+        SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);
+        goto fail;
+    }
     if (PyModule_CheckExact(owner)) {
-        int err = specialize_module_load_attr(owner, instr, name, LOAD_ATTR,
-                                              LOAD_ATTR_MODULE);
-        if (err) {
+        if (specialize_module_load_attr(owner, instr, name))
+        {
             goto fail;
         }
         goto success;
     }
-    PyTypeObject *type = Py_TYPE(owner);
-    if (type->tp_dict == NULL) {
-        if (PyType_Ready(type) < 0) {
-            return -1;
+    if (PyType_Check(owner)) {
+        if (specialize_class_load_attr(owner, instr, name)) {
+            goto fail;
         }
+        goto success;
     }
-    PyObject *descr;
+    PyObject *descr = NULL;
     DescriptorClassification kind = analyze_descriptor(type, name, &descr, 0);
+    assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN);
     if (type_get_version(type, LOAD_ATTR) == 0) {
         goto fail;
     }
@@ -682,11 +755,50 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
             SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
             goto fail;
         case METHOD:
-            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
+        {
+            int oparg = instr->op.arg;
+            if (oparg & 1) {
+                if (specialize_attr_loadmethod(owner, instr, name, descr, kind)) {
+                    goto success;
+                }
+            }
+            else {
+                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
+            }
             goto fail;
+        }
         case PROPERTY:
-            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_PROPERTY);
-            goto fail;
+        {
+            _PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1);
+            assert(Py_TYPE(descr) == &PyProperty_Type);
+            PyObject *fget = ((_PyPropertyObject *)descr)->prop_get;
+            if (fget == NULL) {
+                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
+                goto fail;
+            }
+            if (!Py_IS_TYPE(fget, &PyFunction_Type)) {
+                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION);
+                goto fail;
+            }
+            if (!function_check_args(fget, 1, LOAD_ATTR)) {
+                goto fail;
+            }
+            uint32_t version = function_get_version(fget, LOAD_ATTR);
+            if (version == 0) {
+                goto fail;
+            }
+            if (_PyInterpreterState_GET()->eval_frame) {
+                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);
+                goto fail;
+            }
+            write_u32(lm_cache->keys_version, version);
+            assert(type->tp_version_tag != 0);
+            write_u32(lm_cache->type_version, type->tp_version_tag);
+            /* borrowed */
+            write_obj(lm_cache->descr, fget);
+            instr->op.code = LOAD_ATTR_PROPERTY;
+            goto success;
+        }
         case OBJECT_SLOT:
         {
             PyMemberDescrObject *member = (PyMemberDescrObject *)descr;
@@ -708,7 +820,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
             assert(offset > 0);
             cache->index = (uint16_t)offset;
             write_u32(cache->version, type->tp_version_tag);
-            _Py_SET_OPCODE(*instr, LOAD_ATTR_SLOT);
+            instr->op.code = LOAD_ATTR_SLOT;
             goto success;
         }
         case DUNDER_CLASS:
@@ -717,7 +829,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
             assert(offset == (uint16_t)offset);
             cache->index = (uint16_t)offset;
             write_u32(cache->version, type->tp_version_tag);
-            _Py_SET_OPCODE(*instr, LOAD_ATTR_SLOT);
+            instr->op.code = LOAD_ATTR_SLOT;
             goto success;
         }
         case OTHER_SLOT:
@@ -729,41 +841,80 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
         case GETSET_OVERRIDDEN:
             SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OVERRIDDEN);
             goto fail;
+        case GETATTRIBUTE_IS_PYTHON_FUNCTION:
+        {
+            assert(type->tp_getattro == _Py_slot_tp_getattro);
+            assert(Py_IS_TYPE(descr, &PyFunction_Type));
+            _PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1);
+            if (!function_check_args(descr, 2, LOAD_ATTR)) {
+                goto fail;
+            }
+            uint32_t version = function_get_version(descr, LOAD_ATTR);
+            if (version == 0) {
+                goto fail;
+            }
+            if (_PyInterpreterState_GET()->eval_frame) {
+                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);
+                goto fail;
+            }
+            write_u32(lm_cache->keys_version, version);
+            /* borrowed */
+            write_obj(lm_cache->descr, descr);
+            write_u32(lm_cache->type_version, type->tp_version_tag);
+            instr->op.code = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN;
+            goto success;
+        }
         case BUILTIN_CLASSMETHOD:
+            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ);
+            goto fail;
         case PYTHON_CLASSMETHOD:
+            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_METHOD_OBJ);
+            goto fail;
         case NON_OVERRIDING:
+            SPECIALIZATION_FAIL(LOAD_ATTR,
+                                (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) ?
+                                SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR :
+                                SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
+            goto fail;
         case NON_DESCRIPTOR:
+            SPECIALIZATION_FAIL(LOAD_ATTR,
+                                (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) ?
+                                SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE :
+                                SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
+            goto fail;
         case ABSENT:
-            break;
-    }
-    int err = specialize_dict_access(
-        owner, instr, type, kind, name,
-        LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT
-    );
-    if (err < 0) {
-        return -1;
-    }
-    if (err) {
-        goto success;
+            if (specialize_dict_access(owner, instr, type, kind, name, LOAD_ATTR,
+                                    LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT))
+            {
+                goto success;
+            }
     }
 fail:
     STAT_INC(LOAD_ATTR, failure);
     assert(!PyErr_Occurred());
+    instr->op.code = LOAD_ATTR;
     cache->counter = adaptive_counter_backoff(cache->counter);
-    return 0;
+    return;
 success:
     STAT_INC(LOAD_ATTR, success);
     assert(!PyErr_Occurred());
-    cache->counter = miss_counter_start();
-    return 0;
+    cache->counter = adaptive_counter_cooldown();
 }
 
-int
+void
 _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
 {
+    assert(ENABLE_SPECIALIZATION);
     assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR);
     _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
     PyTypeObject *type = Py_TYPE(owner);
+    if (!_PyType_IsReady(type)) {
+        // We *might* not really need this check, but we inherited it from
+        // PyObject_GenericSetAttr and friends... and this way we still do the
+        // right thing if someone forgets to call PyType_Ready(type):
+        SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OTHER);
+        goto fail;
+    }
     if (PyModule_CheckExact(owner)) {
         SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN);
         goto fail;
@@ -804,7 +955,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
             assert(offset > 0);
             cache->index = (uint16_t)offset;
             write_u32(cache->version, type->tp_version_tag);
-            _Py_SET_OPCODE(*instr, STORE_ATTR_SLOT);
+            instr->op.code = STORE_ATTR_SLOT;
             goto success;
         }
         case DUNDER_CLASS:
@@ -814,89 +965,92 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
         case MUTABLE:
             SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS);
             goto fail;
+        case GETATTRIBUTE_IS_PYTHON_FUNCTION:
         case GETSET_OVERRIDDEN:
             SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN);
             goto fail;
         case BUILTIN_CLASSMETHOD:
+            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ);
+            goto fail;
         case PYTHON_CLASSMETHOD:
+            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_METHOD_OBJ);
+            goto fail;
         case NON_OVERRIDING:
+            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR);
+            goto fail;
         case NON_DESCRIPTOR:
+            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE);
+            goto fail;
         case ABSENT:
-            break;
-    }
-
-    int err = specialize_dict_access(
-        owner, instr, type, kind, name,
-        STORE_ATTR, STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT
-    );
-    if (err < 0) {
-        return -1;
-    }
-    if (err) {
-        goto success;
+            if (specialize_dict_access(owner, instr, type, kind, name, STORE_ATTR,
+                                    STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT))
+            {
+                goto success;
+            }
     }
 fail:
     STAT_INC(STORE_ATTR, failure);
     assert(!PyErr_Occurred());
+    instr->op.code = STORE_ATTR;
     cache->counter = adaptive_counter_backoff(cache->counter);
-    return 0;
+    return;
 success:
     STAT_INC(STORE_ATTR, success);
     assert(!PyErr_Occurred());
-    cache->counter = miss_counter_start();
-    return 0;
+    cache->counter = adaptive_counter_cooldown();
 }
 
 
 #ifdef Py_STATS
 static int
-load_method_fail_kind(DescriptorClassification kind)
+load_attr_fail_kind(DescriptorClassification kind)
 {
     switch (kind) {
         case OVERRIDING:
-            return SPEC_FAIL_LOAD_METHOD_OVERRIDING_DESCRIPTOR;
+            return SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR;
         case METHOD:
-            return SPEC_FAIL_LOAD_METHOD_METHOD;
+            return SPEC_FAIL_ATTR_METHOD;
         case PROPERTY:
-            return SPEC_FAIL_LOAD_METHOD_PROPERTY;
+            return SPEC_FAIL_ATTR_PROPERTY;
         case OBJECT_SLOT:
-            return SPEC_FAIL_LOAD_METHOD_OBJECT_SLOT;
+            return SPEC_FAIL_ATTR_OBJECT_SLOT;
         case OTHER_SLOT:
-            return SPEC_FAIL_LOAD_METHOD_NON_OBJECT_SLOT;
+            return SPEC_FAIL_ATTR_NON_OBJECT_SLOT;
         case DUNDER_CLASS:
             return SPEC_FAIL_OTHER;
         case MUTABLE:
-            return SPEC_FAIL_LOAD_METHOD_MUTABLE_CLASS;
+            return SPEC_FAIL_ATTR_MUTABLE_CLASS;
         case GETSET_OVERRIDDEN:
+        case GETATTRIBUTE_IS_PYTHON_FUNCTION:
             return SPEC_FAIL_OVERRIDDEN;
         case BUILTIN_CLASSMETHOD:
-            return SPEC_FAIL_LOAD_METHOD_BUILTIN_CLASS_METHOD;
+            return SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD;
         case PYTHON_CLASSMETHOD:
-            return SPEC_FAIL_LOAD_METHOD_CLASS_METHOD_OBJ;
+            return SPEC_FAIL_ATTR_CLASS_METHOD_OBJ;
         case NON_OVERRIDING:
-            return SPEC_FAIL_LOAD_METHOD_NON_OVERRIDING_DESCRIPTOR;
+            return SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR;
         case NON_DESCRIPTOR:
-            return SPEC_FAIL_LOAD_METHOD_NOT_DESCRIPTOR;
+            return SPEC_FAIL_ATTR_NOT_DESCRIPTOR;
         case ABSENT:
-            return SPEC_FAIL_LOAD_METHOD_INSTANCE_ATTRIBUTE;
+            return SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE;
     }
     Py_UNREACHABLE();
 }
 #endif
 
 static int
-specialize_class_load_method(PyObject *owner, _Py_CODEUNIT *instr,
+specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
                              PyObject *name)
 {
     _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
     if (!PyType_CheckExact(owner) || _PyType_Lookup(Py_TYPE(owner), name)) {
-        SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_LOAD_METHOD_METACLASS_ATTRIBUTE);
+        SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE);
         return -1;
     }
     PyObject *descr = NULL;
     DescriptorClassification kind = 0;
     kind = analyze_descriptor((PyTypeObject *)owner, name, &descr, 0);
-    if (type_get_version((PyTypeObject *)owner, LOAD_METHOD) == 0) {
+    if (type_get_version((PyTypeObject *)owner, LOAD_ATTR) == 0) {
         return -1;
     }
     switch (kind) {
@@ -904,131 +1058,70 @@ specialize_class_load_method(PyObject *owner, _Py_CODEUNIT *instr,
         case NON_DESCRIPTOR:
             write_u32(cache->type_version, ((PyTypeObject *)owner)->tp_version_tag);
             write_obj(cache->descr, descr);
-            _Py_SET_OPCODE(*instr, LOAD_METHOD_CLASS);
+            instr->op.code = LOAD_ATTR_CLASS;
             return 0;
 #ifdef Py_STATS
         case ABSENT:
-            SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_EXPECTED_ERROR);
+            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
             return -1;
 #endif
         default:
-            SPECIALIZATION_FAIL(LOAD_METHOD, load_method_fail_kind(kind));
+            SPECIALIZATION_FAIL(LOAD_ATTR, load_attr_fail_kind(kind));
             return -1;
     }
 }
 
-typedef enum {
-    MANAGED_VALUES = 1,
-    MANAGED_DICT = 2,
-    OFFSET_DICT = 3,
-    NO_DICT = 4
-} ObjectDictKind;
-
 // Please collect stats carefully before and after modifying. A subtle change
 // can cause a significant drop in cache hits. A possible test is
 // python.exe -m test_typing test_re test_dis test_zlib.
-int
-_Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
+static int
+specialize_attr_loadmethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name,
+PyObject *descr, DescriptorClassification kind)
 {
-    assert(_PyOpcode_Caches[LOAD_METHOD] == INLINE_CACHE_ENTRIES_LOAD_METHOD);
     _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
     PyTypeObject *owner_cls = Py_TYPE(owner);
 
-    if (PyModule_CheckExact(owner)) {
-        assert(INLINE_CACHE_ENTRIES_LOAD_ATTR <=
-               INLINE_CACHE_ENTRIES_LOAD_METHOD);
-        int err = specialize_module_load_attr(owner, instr, name, LOAD_METHOD,
-                                              LOAD_METHOD_MODULE);
-        if (err) {
-            goto fail;
-        }
-        goto success;
-    }
-    if (owner_cls->tp_dict == NULL) {
-        if (PyType_Ready(owner_cls) < 0) {
-            return -1;
-        }
-    }
-    if (PyType_Check(owner)) {
-        int err = specialize_class_load_method(owner, instr, name);
-        if (err) {
-            goto fail;
-        }
-        goto success;
-    }
-
-    PyObject *descr = NULL;
-    DescriptorClassification kind = 0;
-    kind = analyze_descriptor(owner_cls, name, &descr, 0);
-    if (type_get_version(owner_cls, LOAD_METHOD) == 0) {
-        goto fail;
-    }
-    assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN);
-    if (kind != METHOD) {
-        SPECIALIZATION_FAIL(LOAD_METHOD, load_method_fail_kind(kind));
-        goto fail;
-    }
-    ObjectDictKind dictkind;
-    PyDictKeysObject *keys;
+    assert(kind == METHOD && descr != NULL);
     if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
-        PyObject *dict = *_PyObject_ManagedDictPointer(owner);
-        keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys;
-        if (dict == NULL) {
-            dictkind = MANAGED_VALUES;
+        PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
+        PyDictKeysObject *keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys;
+        if (!_PyDictOrValues_IsValues(dorv)) {
+            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT);
+            return 0;
         }
-        else {
-            dictkind = MANAGED_DICT;
+        Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
+        if (index != DKIX_EMPTY) {
+            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SHADOWED);
+            return 0;
         }
+        uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(
+                _PyInterpreterState_GET(), keys);
+        if (keys_version == 0) {
+            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
+            return 0;
+        }
+        write_u32(cache->keys_version, keys_version);
+        instr->op.code = LOAD_ATTR_METHOD_WITH_VALUES;
     }
     else {
         Py_ssize_t dictoffset = owner_cls->tp_dictoffset;
         if (dictoffset < 0 || dictoffset > INT16_MAX) {
-            SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OUT_OF_RANGE);
-            goto fail;
+            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);
+            return 0;
         }
         if (dictoffset == 0) {
-            dictkind = NO_DICT;
-            keys = NULL;
+            instr->op.code = LOAD_ATTR_METHOD_NO_DICT;
         }
         else {
             PyObject *dict = *(PyObject **) ((char *)owner + dictoffset);
-            if (dict == NULL) {
-                SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_NO_DICT);
-                goto fail;
+            if (dict) {
+                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
+                return 0;
             }
-            keys = ((PyDictObject *)dict)->ma_keys;
-            dictkind = OFFSET_DICT;
-        }
-    }
-    if (dictkind != NO_DICT) {
-        Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
-        if (index != DKIX_EMPTY) {
-            SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_LOAD_METHOD_IS_ATTR);
-            goto fail;
-        }
-        uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(keys);
-        if (keys_version == 0) {
-            SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OUT_OF_VERSIONS);
-            goto fail;
+            assert(owner_cls->tp_dictoffset > 0);
+            assert(owner_cls->tp_dictoffset <= INT16_MAX);
+            instr->op.code = LOAD_ATTR_METHOD_LAZY_DICT;
         }
-        write_u32(cache->keys_version, keys_version);
-    }
-    switch(dictkind) {
-        case NO_DICT:
-            _Py_SET_OPCODE(*instr, LOAD_METHOD_NO_DICT);
-            break;
-        case MANAGED_VALUES:
-            _Py_SET_OPCODE(*instr, LOAD_METHOD_WITH_VALUES);
-            break;
-        case MANAGED_DICT:
-            *(int16_t *)&cache->dict_offset = (int16_t)MANAGED_DICT_OFFSET;
-            _Py_SET_OPCODE(*instr, LOAD_METHOD_WITH_DICT);
-            break;
-        case OFFSET_DICT:
-            assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX);
-            cache->dict_offset = (uint16_t)owner_cls->tp_dictoffset;
-            _Py_SET_OPCODE(*instr, LOAD_METHOD_WITH_DICT);
-            break;
     }
     /* `descr` is borrowed. This is safe for methods (even inherited ones from
     *  super classes!) as long as tp_version_tag is validated for two main reasons:
@@ -1046,29 +1139,21 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
     */
     write_u32(cache->type_version, owner_cls->tp_version_tag);
     write_obj(cache->descr, descr);
-    // Fall through.
-success:
-    STAT_INC(LOAD_METHOD, success);
-    assert(!PyErr_Occurred());
-    cache->counter = miss_counter_start();
-    return 0;
-fail:
-    STAT_INC(LOAD_METHOD, failure);
-    assert(!PyErr_Occurred());
-    cache->counter = adaptive_counter_backoff(cache->counter);
-    return 0;
+    return 1;
 }
 
-int
+void
 _Py_Specialize_LoadGlobal(
     PyObject *globals, PyObject *builtins,
     _Py_CODEUNIT *instr, PyObject *name)
 {
+    assert(ENABLE_SPECIALIZATION);
     assert(_PyOpcode_Caches[LOAD_GLOBAL] == INLINE_CACHE_ENTRIES_LOAD_GLOBAL);
     /* Use inline cache */
     _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)(instr + 1);
     assert(PyUnicode_CheckExact(name));
     if (!PyDict_CheckExact(globals)) {
+        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_DICT);
         goto fail;
     }
     PyDictKeysObject * globals_keys = ((PyDictObject *)globals)->ma_keys;
@@ -1078,23 +1163,32 @@ _Py_Specialize_LoadGlobal(
     }
     Py_ssize_t index = _PyDictKeys_StringLookup(globals_keys, name);
     if (index == DKIX_ERROR) {
-        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT);
+        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_EXPECTED_ERROR);
         goto fail;
     }
+    PyInterpreterState *interp = _PyInterpreterState_GET();
     if (index != DKIX_EMPTY) {
         if (index != (uint16_t)index) {
+            SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE);
             goto fail;
         }
-        uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(globals_keys);
+        uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(
+                interp, globals_keys);
         if (keys_version == 0) {
+            SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS);
+            goto fail;
+        }
+        if (keys_version != (uint16_t)keys_version) {
+            SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE);
             goto fail;
         }
         cache->index = (uint16_t)index;
-        write_u32(cache->module_keys_version, keys_version);
-        _Py_SET_OPCODE(*instr, LOAD_GLOBAL_MODULE);
+        cache->module_keys_version = (uint16_t)keys_version;
+        instr->op.code = LOAD_GLOBAL_MODULE;
         goto success;
     }
     if (!PyDict_CheckExact(builtins)) {
+        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_DICT);
         goto fail;
     }
     PyDictKeysObject * builtin_keys = ((PyDictObject *)builtins)->ma_keys;
@@ -1104,18 +1198,25 @@ _Py_Specialize_LoadGlobal(
     }
     index = _PyDictKeys_StringLookup(builtin_keys, name);
     if (index == DKIX_ERROR) {
-        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT);
+        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_EXPECTED_ERROR);
         goto fail;
     }
     if (index != (uint16_t)index) {
+        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE);
         goto fail;
     }
-    uint32_t globals_version = _PyDictKeys_GetVersionForCurrentState(globals_keys);
+    uint32_t globals_version = _PyDictKeys_GetVersionForCurrentState(
+            interp, globals_keys);
     if (globals_version == 0) {
         SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS);
         goto fail;
     }
-    uint32_t builtins_version = _PyDictKeys_GetVersionForCurrentState(builtin_keys);
+    if (globals_version != (uint16_t)globals_version) {
+        SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE);
+        goto fail;
+    }
+    uint32_t builtins_version = _PyDictKeys_GetVersionForCurrentState(
+            interp, builtin_keys);
     if (builtins_version == 0) {
         SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS);
         goto fail;
@@ -1125,20 +1226,20 @@ _Py_Specialize_LoadGlobal(
         goto fail;
     }
     cache->index = (uint16_t)index;
-    write_u32(cache->module_keys_version, globals_version);
+    cache->module_keys_version = (uint16_t)globals_version;
     cache->builtin_keys_version = (uint16_t)builtins_version;
-    _Py_SET_OPCODE(*instr, LOAD_GLOBAL_BUILTIN);
+    instr->op.code = LOAD_GLOBAL_BUILTIN;
     goto success;
 fail:
     STAT_INC(LOAD_GLOBAL, failure);
     assert(!PyErr_Occurred());
+    instr->op.code = LOAD_GLOBAL;
     cache->counter = adaptive_counter_backoff(cache->counter);
-    return 0;
+    return;
 success:
     STAT_INC(LOAD_GLOBAL, success);
     assert(!PyErr_Occurred());
-    cache->counter = miss_counter_start();
-    return 0;
+    cache->counter = adaptive_counter_cooldown();
 }
 
 #ifdef Py_STATS
@@ -1181,21 +1282,51 @@ binary_subscr_fail_kind(PyTypeObject *container_type, PyObject *sub)
 }
 #endif
 
-
-#define SIMPLE_FUNCTION 0
-
 static int
 function_kind(PyCodeObject *code) {
     int flags = code->co_flags;
     if ((flags & (CO_VARKEYWORDS | CO_VARARGS)) || code->co_kwonlyargcount) {
-        return SPEC_FAIL_CALL_COMPLEX_PARAMETERS;
+        return SPEC_FAIL_CODE_COMPLEX_PARAMETERS;
     }
     if ((flags & CO_OPTIMIZED) == 0) {
-        return SPEC_FAIL_CALL_CO_NOT_OPTIMIZED;
+        return SPEC_FAIL_CODE_NOT_OPTIMIZED;
     }
     return SIMPLE_FUNCTION;
 }
 
+/* Returning false indicates a failure. */
+static bool
+function_check_args(PyObject *o, int expected_argcount, int opcode)
+{
+    assert(Py_IS_TYPE(o, &PyFunction_Type));
+    PyFunctionObject *func = (PyFunctionObject *)o;
+    PyCodeObject *fcode = (PyCodeObject *)func->func_code;
+    int kind = function_kind(fcode);
+    if (kind != SIMPLE_FUNCTION) {
+        SPECIALIZATION_FAIL(opcode, kind);
+        return false;
+    }
+    if (fcode->co_argcount != expected_argcount) {
+        SPECIALIZATION_FAIL(opcode, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
+        return false;
+    }
+    return true;
+}
+
+/* Returning 0 indicates a failure. */
+static uint32_t
+function_get_version(PyObject *o, int opcode)
+{
+    assert(Py_IS_TYPE(o, &PyFunction_Type));
+    PyFunctionObject *func = (PyFunctionObject *)o;
+    uint32_t version = _PyFunction_GetVersionForCurrentState(func);
+    if (version == 0) {
+        SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS);
+        return 0;
+    }
+    return version;
+}
+
 /* Returning 0 indicates a failure. */
 static uint32_t
 type_get_version(PyTypeObject *t, int opcode)
@@ -1208,18 +1339,23 @@ type_get_version(PyTypeObject *t, int opcode)
     return version;
 }
 
-int
+void
 _Py_Specialize_BinarySubscr(
      PyObject *container, PyObject *sub, _Py_CODEUNIT *instr)
 {
+    assert(ENABLE_SPECIALIZATION);
     assert(_PyOpcode_Caches[BINARY_SUBSCR] ==
            INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
     _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)(instr + 1);
     PyTypeObject *container_type = Py_TYPE(container);
     if (container_type == &PyList_Type) {
         if (PyLong_CheckExact(sub)) {
-            _Py_SET_OPCODE(*instr, BINARY_SUBSCR_LIST_INT);
-            goto success;
+            if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
+                instr->op.code = BINARY_SUBSCR_LIST_INT;
+                goto success;
+            }
+            SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE);
+            goto fail;
         }
         SPECIALIZATION_FAIL(BINARY_SUBSCR,
             PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_LIST_SLICE : SPEC_FAIL_OTHER);
@@ -1227,15 +1363,19 @@ _Py_Specialize_BinarySubscr(
     }
     if (container_type == &PyTuple_Type) {
         if (PyLong_CheckExact(sub)) {
-            _Py_SET_OPCODE(*instr, BINARY_SUBSCR_TUPLE_INT);
-            goto success;
+            if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
+                instr->op.code = BINARY_SUBSCR_TUPLE_INT;
+                goto success;
+            }
+            SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE);
+            goto fail;
         }
         SPECIALIZATION_FAIL(BINARY_SUBSCR,
             PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_TUPLE_SLICE : SPEC_FAIL_OTHER);
         goto fail;
     }
     if (container_type == &PyDict_Type) {
-        _Py_SET_OPCODE(*instr, BINARY_SUBSCR_DICT);
+        instr->op.code = BINARY_SUBSCR_DICT;
         goto success;
     }
     PyTypeObject *cls = Py_TYPE(container);
@@ -1256,13 +1396,8 @@ _Py_Specialize_BinarySubscr(
             SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
             goto fail;
         }
-        if (type_get_version(cls, BINARY_SUBSCR) == 0) {
-            goto fail;
-        }
-        assert(cls->tp_version_tag != 0);
-        write_u32(cache->type_version, cls->tp_version_tag);
-        int version = _PyFunction_GetVersionForCurrentState(func);
-        if (version == 0 || version != (uint16_t)version) {
+        uint32_t version = _PyFunction_GetVersionForCurrentState(func);
+        if (version == 0) {
             SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_VERSIONS);
             goto fail;
         }
@@ -1270,9 +1405,12 @@ _Py_Specialize_BinarySubscr(
             SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OTHER);
             goto fail;
         }
-        cache->func_version = version;
-        ((PyHeapTypeObject *)container_type)->_spec_cache.getitem = descriptor;
-        _Py_SET_OPCODE(*instr, BINARY_SUBSCR_GETITEM);
+        PyHeapTypeObject *ht = (PyHeapTypeObject *)container_type;
+        // This pointer is invalidated by PyType_Modified (see the comment on
+        // struct _specialization_cache):
+        ht->_spec_cache.getitem = descriptor;
+        ht->_spec_cache.getitem_version = version;
+        instr->op.code = BINARY_SUBSCR_GETITEM;
         goto success;
     }
     SPECIALIZATION_FAIL(BINARY_SUBSCR,
@@ -1280,26 +1418,27 @@ _Py_Specialize_BinarySubscr(
 fail:
     STAT_INC(BINARY_SUBSCR, failure);
     assert(!PyErr_Occurred());
+    instr->op.code = BINARY_SUBSCR;
     cache->counter = adaptive_counter_backoff(cache->counter);
-    return 0;
+    return;
 success:
     STAT_INC(BINARY_SUBSCR, success);
     assert(!PyErr_Occurred());
-    cache->counter = miss_counter_start();
-    return 0;
+    cache->counter = adaptive_counter_cooldown();
 }
 
-int
+void
 _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr)
 {
+    assert(ENABLE_SPECIALIZATION);
     _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)(instr + 1);
     PyTypeObject *container_type = Py_TYPE(container);
     if (container_type == &PyList_Type) {
         if (PyLong_CheckExact(sub)) {
-            if ((Py_SIZE(sub) == 0 || Py_SIZE(sub) == 1)
-                && ((PyLongObject *)sub)->ob_digit[0] < (size_t)PyList_GET_SIZE(container))
+            if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)
+                && ((PyLongObject *)sub)->long_value.ob_digit[0] < (size_t)PyList_GET_SIZE(container))
             {
-                _Py_SET_OPCODE(*instr, STORE_SUBSCR_LIST_INT);
+                instr->op.code = STORE_SUBSCR_LIST_INT;
                 goto success;
             }
             else {
@@ -1317,8 +1456,8 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins
         }
     }
     if (container_type == &PyDict_Type) {
-        _Py_SET_OPCODE(*instr, STORE_SUBSCR_DICT);
-         goto success;
+        instr->op.code = STORE_SUBSCR_DICT;
+        goto success;
     }
 #ifdef Py_STATS
     PyMappingMethods *as_mapping = container_type->tp_as_mapping;
@@ -1328,7 +1467,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins
         goto fail;
     }
     if (PyObject_CheckBuffer(container)) {
-        if (PyLong_CheckExact(sub) && (((size_t)Py_SIZE(sub)) > 1)) {
+        if (PyLong_CheckExact(sub) && (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub))) {
             SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE);
         }
         else if (strcmp(container_type->tp_name, "array.array") == 0) {
@@ -1384,49 +1523,49 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins
 fail:
     STAT_INC(STORE_SUBSCR, failure);
     assert(!PyErr_Occurred());
+    instr->op.code = STORE_SUBSCR;
     cache->counter = adaptive_counter_backoff(cache->counter);
-    return 0;
+    return;
 success:
     STAT_INC(STORE_SUBSCR, success);
     assert(!PyErr_Occurred());
-    cache->counter = miss_counter_start();
-    return 0;
+    cache->counter = adaptive_counter_cooldown();
 }
 
 static int
 specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
-                      PyObject *kwnames, int oparg)
+                      PyObject *kwnames)
 {
-    assert(_Py_OPCODE(*instr) == PRECALL_ADAPTIVE);
     PyTypeObject *tp = _PyType_CAST(callable);
     if (tp->tp_new == PyBaseObject_Type.tp_new) {
-        SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_PYTHON_CLASS);
+        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PYTHON_CLASS);
         return -1;
     }
     if (tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) {
+        int oparg = instr->op.arg;
         if (nargs == 1 && kwnames == NULL && oparg == 1) {
             if (tp == &PyUnicode_Type) {
-                _Py_SET_OPCODE(*instr, PRECALL_NO_KW_STR_1);
+                instr->op.code = CALL_NO_KW_STR_1;
                 return 0;
             }
             else if (tp == &PyType_Type) {
-                _Py_SET_OPCODE(*instr, PRECALL_NO_KW_TYPE_1);
+                instr->op.code = CALL_NO_KW_TYPE_1;
                 return 0;
             }
             else if (tp == &PyTuple_Type) {
-                _Py_SET_OPCODE(*instr, PRECALL_NO_KW_TUPLE_1);
+                instr->op.code = CALL_NO_KW_TUPLE_1;
                 return 0;
             }
         }
         if (tp->tp_vectorcall != NULL) {
-            _Py_SET_OPCODE(*instr, PRECALL_BUILTIN_CLASS);
+            instr->op.code = CALL_BUILTIN_CLASS;
             return 0;
         }
-        SPECIALIZATION_FAIL(PRECALL, tp == &PyUnicode_Type ?
+        SPECIALIZATION_FAIL(CALL, tp == &PyUnicode_Type ?
             SPEC_FAIL_CALL_STR : SPEC_FAIL_CALL_CLASS_NO_VECTORCALL);
         return -1;
     }
-    SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_CLASS_MUTABLE);
+    SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_CLASS_MUTABLE);
     return -1;
 }
 
@@ -1437,17 +1576,40 @@ builtin_call_fail_kind(int ml_flags)
     switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
         METH_KEYWORDS | METH_METHOD)) {
         case METH_VARARGS:
-            return SPEC_FAIL_CALL_PYCFUNCTION;
+            return SPEC_FAIL_CALL_CFUNC_VARARGS;
         case METH_VARARGS | METH_KEYWORDS:
-            return SPEC_FAIL_CALL_PYCFUNCTION_WITH_KEYWORDS;
-        case METH_FASTCALL | METH_KEYWORDS:
-            return SPEC_FAIL_CALL_PYCFUNCTION_FAST_WITH_KEYWORDS;
+            return SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS;
         case METH_NOARGS:
-            return SPEC_FAIL_CALL_PYCFUNCTION_NOARGS;
-        /* This case should never happen with PyCFunctionObject -- only
-            PyMethodObject. See zlib.compressobj()'s methods for an example.
-        */
+            return SPEC_FAIL_CALL_CFUNC_NOARGS;
+        case METH_METHOD | METH_FASTCALL | METH_KEYWORDS:
+            return SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS;
+        /* These cases should be optimized, but return "other" just in case */
+        case METH_O:
+        case METH_FASTCALL:
+        case METH_FASTCALL | METH_KEYWORDS:
+            return SPEC_FAIL_OTHER;
+        default:
+            return SPEC_FAIL_CALL_BAD_CALL_FLAGS;
+    }
+}
+
+static int
+meth_descr_call_fail_kind(int ml_flags)
+{
+    switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
+                        METH_KEYWORDS | METH_METHOD)) {
+        case METH_VARARGS:
+            return SPEC_FAIL_CALL_METH_DESCR_VARARGS;
+        case METH_VARARGS | METH_KEYWORDS:
+            return SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS;
         case METH_METHOD | METH_FASTCALL | METH_KEYWORDS:
+            return SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS;
+            /* These cases should be optimized, but return "other" just in case */
+        case METH_NOARGS:
+        case METH_O:
+        case METH_FASTCALL:
+        case METH_FASTCALL | METH_KEYWORDS:
+            return SPEC_FAIL_OTHER;
         default:
             return SPEC_FAIL_CALL_BAD_CALL_FLAGS;
     }
@@ -1456,11 +1618,10 @@ builtin_call_fail_kind(int ml_flags)
 
 static int
 specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr,
-                             int nargs, PyObject *kwnames, int oparg)
+                             int nargs, PyObject *kwnames)
 {
-    assert(_Py_OPCODE(*instr) == PRECALL_ADAPTIVE);
     if (kwnames) {
-        SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_KWNAMES);
+        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES);
         return -1;
     }
 
@@ -1469,48 +1630,47 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr,
         METH_KEYWORDS | METH_METHOD)) {
         case METH_NOARGS: {
             if (nargs != 1) {
-                SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
+                SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
                 return -1;
             }
-            _Py_SET_OPCODE(*instr, PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS);
+            instr->op.code = CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS;
             return 0;
         }
         case METH_O: {
             if (nargs != 2) {
-                SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
+                SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
                 return -1;
             }
             PyInterpreterState *interp = _PyInterpreterState_GET();
             PyObject *list_append = interp->callable_cache.list_append;
-            _Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_PRECALL + 1
-                                      + INLINE_CACHE_ENTRIES_CALL + 1];
-            bool pop = (_Py_OPCODE(next) == POP_TOP);
+            _Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_CALL + 1];
+            bool pop = (next.op.code == POP_TOP);
+            int oparg = instr->op.arg;
             if ((PyObject *)descr == list_append && oparg == 1 && pop) {
-                _Py_SET_OPCODE(*instr, PRECALL_NO_KW_LIST_APPEND);
+                instr->op.code = CALL_NO_KW_LIST_APPEND;
                 return 0;
             }
-            _Py_SET_OPCODE(*instr, PRECALL_NO_KW_METHOD_DESCRIPTOR_O);
+            instr->op.code = CALL_NO_KW_METHOD_DESCRIPTOR_O;
             return 0;
         }
         case METH_FASTCALL: {
-            _Py_SET_OPCODE(*instr, PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST);
+            instr->op.code = CALL_NO_KW_METHOD_DESCRIPTOR_FAST;
             return 0;
         }
-        case METH_FASTCALL|METH_KEYWORDS: {
-            _Py_SET_OPCODE(*instr, PRECALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS);
+        case METH_FASTCALL | METH_KEYWORDS: {
+            instr->op.code = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS;
             return 0;
         }
     }
-    SPECIALIZATION_FAIL(PRECALL, builtin_call_fail_kind(descr->d_method->ml_flags));
+    SPECIALIZATION_FAIL(CALL, meth_descr_call_fail_kind(descr->d_method->ml_flags));
     return -1;
 }
 
 static int
 specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs,
-                   PyObject *kwnames)
+                   PyObject *kwnames, bool bound_method)
 {
     _PyCallCache *cache = (_PyCallCache *)(instr + 1);
-    assert(_Py_OPCODE(*instr) == CALL_ADAPTIVE);
     PyCodeObject *code = (PyCodeObject *)func->func_code;
     int kind = function_kind(code);
     /* Don't specialize if PEP 523 is active */
@@ -1537,22 +1697,21 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs,
     assert(nargs <= argcount && nargs >= min_args);
     assert(min_args >= 0 && defcount >= 0);
     assert(defcount == 0 || func->func_defaults != NULL);
-    if (min_args > 0xffff) {
-        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_RANGE);
-        return -1;
-    }
     int version = _PyFunction_GetVersionForCurrentState(func);
     if (version == 0) {
         SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS);
         return -1;
     }
     write_u32(cache->func_version, version);
-    cache->min_args = min_args;
     if (argcount == nargs) {
-        _Py_SET_OPCODE(*instr, CALL_PY_EXACT_ARGS);
+        instr->op.code = bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS;
+    }
+    else if (bound_method) {
+        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD);
+        return -1;
     }
     else {
-        _Py_SET_OPCODE(*instr, CALL_PY_WITH_DEFAULTS);
+        instr->op.code = CALL_PY_WITH_DEFAULTS;
     }
     return 0;
 }
@@ -1561,7 +1720,6 @@ static int
 specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
                   PyObject *kwnames)
 {
-    assert(_Py_OPCODE(*instr) == PRECALL_ADAPTIVE);
     if (PyCFunction_GET_FUNCTION(callable) == NULL) {
         return 1;
     }
@@ -1570,44 +1728,44 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
         METH_KEYWORDS | METH_METHOD)) {
         case METH_O: {
             if (kwnames) {
-                SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_KWNAMES);
+                SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES);
                 return -1;
             }
             if (nargs != 1) {
-                SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
+                SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
                 return 1;
             }
             /* len(o) */
             PyInterpreterState *interp = _PyInterpreterState_GET();
             if (callable == interp->callable_cache.len) {
-                _Py_SET_OPCODE(*instr, PRECALL_NO_KW_LEN);
+                instr->op.code = CALL_NO_KW_LEN;
                 return 0;
             }
-            _Py_SET_OPCODE(*instr, PRECALL_NO_KW_BUILTIN_O);
+            instr->op.code = CALL_NO_KW_BUILTIN_O;
             return 0;
         }
         case METH_FASTCALL: {
             if (kwnames) {
-                SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_KWNAMES);
+                SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES);
                 return -1;
             }
             if (nargs == 2) {
                 /* isinstance(o1, o2) */
                 PyInterpreterState *interp = _PyInterpreterState_GET();
                 if (callable == interp->callable_cache.isinstance) {
-                    _Py_SET_OPCODE(*instr, PRECALL_NO_KW_ISINSTANCE);
+                    instr->op.code = CALL_NO_KW_ISINSTANCE;
                     return 0;
                 }
             }
-            _Py_SET_OPCODE(*instr, PRECALL_NO_KW_BUILTIN_FAST);
+            instr->op.code = CALL_NO_KW_BUILTIN_FAST;
             return 0;
         }
         case METH_FASTCALL | METH_KEYWORDS: {
-            _Py_SET_OPCODE(*instr, PRECALL_BUILTIN_FAST_WITH_KEYWORDS);
+            instr->op.code = CALL_BUILTIN_FAST_WITH_KEYWORDS;
             return 0;
         }
         default:
-            SPECIALIZATION_FAIL(PRECALL,
+            SPECIALIZATION_FAIL(CALL,
                 builtin_call_fail_kind(PyCFunction_GET_FLAGS(callable)));
             return 1;
     }
@@ -1617,33 +1775,18 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
 static int
 call_fail_kind(PyObject *callable)
 {
-    if (PyCFunction_CheckExact(callable)) {
-        return SPEC_FAIL_CALL_PYCFUNCTION;
-    }
-    else if (PyFunction_Check(callable)) {
-        return SPEC_FAIL_CALL_PYFUNCTION;
-    }
-    else if (PyInstanceMethod_Check(callable)) {
+    assert(!PyCFunction_CheckExact(callable));
+    assert(!PyFunction_Check(callable));
+    assert(!PyType_Check(callable));
+    assert(!Py_IS_TYPE(callable, &PyMethodDescr_Type));
+    assert(!PyMethod_Check(callable));
+    if (PyInstanceMethod_Check(callable)) {
         return SPEC_FAIL_CALL_INSTANCE_METHOD;
     }
-    else if (PyMethod_Check(callable)) {
-        return SPEC_FAIL_CALL_BOUND_METHOD;
-    }
     // builtin method
     else if (PyCMethod_Check(callable)) {
         return SPEC_FAIL_CALL_CMETHOD;
     }
-    else if (PyType_Check(callable)) {
-        if (((PyTypeObject *)callable)->tp_new == PyBaseObject_Type.tp_new) {
-            return SPEC_FAIL_CALL_PYTHON_CLASS;
-        }
-        else {
-            return SPEC_FAIL_CALL_CLASS;
-        }
-    }
-    else if (Py_IS_TYPE(callable, &PyMethodDescr_Type)) {
-        return SPEC_FAIL_CALL_METHOD_DESCRIPTOR;
-    }
     else if (Py_TYPE(callable) == &PyWrapperDescr_Type) {
         return SPEC_FAIL_CALL_OPERATOR_WRAPPER;
     }
@@ -1655,62 +1798,41 @@ call_fail_kind(PyObject *callable)
 #endif
 
 
-int
-_Py_Specialize_Precall(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
-                       PyObject *kwnames, int oparg)
+/* TODO:
+    - Specialize calling classes.
+*/
+void
+_Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
+                    PyObject *kwnames)
 {
-    assert(_PyOpcode_Caches[PRECALL] == INLINE_CACHE_ENTRIES_PRECALL);
-    _PyPrecallCache *cache = (_PyPrecallCache *)(instr + 1);
+    assert(ENABLE_SPECIALIZATION);
+    assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL);
+    assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL);
+    _PyCallCache *cache = (_PyCallCache *)(instr + 1);
     int fail;
     if (PyCFunction_CheckExact(callable)) {
         fail = specialize_c_call(callable, instr, nargs, kwnames);
     }
     else if (PyFunction_Check(callable)) {
-        _Py_SET_OPCODE(*instr, PRECALL_PYFUNC);
-        fail = 0;
+        fail = specialize_py_call((PyFunctionObject *)callable, instr, nargs,
+                                  kwnames, false);
     }
     else if (PyType_Check(callable)) {
-        fail = specialize_class_call(callable, instr, nargs, kwnames, oparg);
+        fail = specialize_class_call(callable, instr, nargs, kwnames);
     }
     else if (Py_IS_TYPE(callable, &PyMethodDescr_Type)) {
         fail = specialize_method_descriptor((PyMethodDescrObject *)callable,
-                                            instr, nargs, kwnames, oparg);
-    }
-    else if (Py_TYPE(callable) == &PyMethod_Type) {
-        _Py_SET_OPCODE(*instr, PRECALL_BOUND_METHOD);
-        fail = 0;
+                                            instr, nargs, kwnames);
     }
-    else {
-        SPECIALIZATION_FAIL(PRECALL, call_fail_kind(callable));
-        fail = -1;
-    }
-    if (fail) {
-        STAT_INC(PRECALL, failure);
-        assert(!PyErr_Occurred());
-        cache->counter = adaptive_counter_backoff(cache->counter);
-    }
-    else {
-        STAT_INC(PRECALL, success);
-        assert(!PyErr_Occurred());
-        cache->counter = miss_counter_start();
-    }
-    return 0;
-}
-
-
-/* TODO:
-    - Specialize calling classes.
-*/
-int
-_Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
-                    PyObject *kwnames)
-{
-    assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL);
-    _PyCallCache *cache = (_PyCallCache *)(instr + 1);
-    int fail;
-    if (PyFunction_Check(callable)) {
-        fail = specialize_py_call((PyFunctionObject *)callable, instr, nargs,
-                                  kwnames);
+    else if (PyMethod_Check(callable)) {
+        PyObject *func = ((PyMethodObject *)callable)->im_func;
+        if (PyFunction_Check(func)) {
+            fail = specialize_py_call((PyFunctionObject *)func,
+                                      instr, nargs+1, kwnames, true);
+        } else {
+            SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD);
+            fail = -1;
+        }
     }
     else {
         SPECIALIZATION_FAIL(CALL, call_fail_kind(callable));
@@ -1719,14 +1841,14 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
     if (fail) {
         STAT_INC(CALL, failure);
         assert(!PyErr_Occurred());
+        instr->op.code = CALL;
         cache->counter = adaptive_counter_backoff(cache->counter);
     }
     else {
         STAT_INC(CALL, success);
         assert(!PyErr_Occurred());
-        cache->counter = miss_counter_start();
+        cache->counter = adaptive_counter_cooldown();
     }
-    return 0;
 }
 
 #ifdef Py_STATS
@@ -1803,6 +1925,7 @@ void
 _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
                         int oparg, PyObject **locals)
 {
+    assert(ENABLE_SPECIALIZATION);
     assert(_PyOpcode_Caches[BINARY_OP] == INLINE_CACHE_ENTRIES_BINARY_OP);
     _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1);
     switch (oparg) {
@@ -1813,21 +1936,21 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
             }
             if (PyUnicode_CheckExact(lhs)) {
                 _Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_BINARY_OP + 1];
-                bool to_store = (_Py_OPCODE(next) == STORE_FAST ||
-                                 _Py_OPCODE(next) == STORE_FAST__LOAD_FAST);
-                if (to_store && locals[_Py_OPARG(next)] == lhs) {
-                    _Py_SET_OPCODE(*instr, BINARY_OP_INPLACE_ADD_UNICODE);
+                bool to_store = (next.op.code == STORE_FAST ||
+                                 next.op.code == STORE_FAST__LOAD_FAST);
+                if (to_store && locals[next.op.arg] == lhs) {
+                    instr->op.code = BINARY_OP_INPLACE_ADD_UNICODE;
                     goto success;
                 }
-                _Py_SET_OPCODE(*instr, BINARY_OP_ADD_UNICODE);
+                instr->op.code = BINARY_OP_ADD_UNICODE;
                 goto success;
             }
             if (PyLong_CheckExact(lhs)) {
-                _Py_SET_OPCODE(*instr, BINARY_OP_ADD_INT);
+                instr->op.code = BINARY_OP_ADD_INT;
                 goto success;
             }
             if (PyFloat_CheckExact(lhs)) {
-                _Py_SET_OPCODE(*instr, BINARY_OP_ADD_FLOAT);
+                instr->op.code = BINARY_OP_ADD_FLOAT;
                 goto success;
             }
             break;
@@ -1837,11 +1960,11 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
                 break;
             }
             if (PyLong_CheckExact(lhs)) {
-                _Py_SET_OPCODE(*instr, BINARY_OP_MULTIPLY_INT);
+                instr->op.code = BINARY_OP_MULTIPLY_INT;
                 goto success;
             }
             if (PyFloat_CheckExact(lhs)) {
-                _Py_SET_OPCODE(*instr, BINARY_OP_MULTIPLY_FLOAT);
+                instr->op.code = BINARY_OP_MULTIPLY_FLOAT;
                 goto success;
             }
             break;
@@ -1851,32 +1974,23 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
                 break;
             }
             if (PyLong_CheckExact(lhs)) {
-                _Py_SET_OPCODE(*instr, BINARY_OP_SUBTRACT_INT);
+                instr->op.code = BINARY_OP_SUBTRACT_INT;
                 goto success;
             }
             if (PyFloat_CheckExact(lhs)) {
-                _Py_SET_OPCODE(*instr, BINARY_OP_SUBTRACT_FLOAT);
+                instr->op.code = BINARY_OP_SUBTRACT_FLOAT;
                 goto success;
             }
             break;
-#ifndef Py_STATS
-        default:
-            // These operators don't have any available specializations. Rather
-            // than repeatedly attempting to specialize them, just convert them
-            // back to BINARY_OP (unless we're collecting stats, where it's more
-            // important to get accurate hit counts for the unadaptive version
-            // and each of the different failure types):
-            _Py_SET_OPCODE(*instr, BINARY_OP);
-            return;
-#endif
     }
     SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs));
     STAT_INC(BINARY_OP, failure);
+    instr->op.code = BINARY_OP;
     cache->counter = adaptive_counter_backoff(cache->counter);
     return;
 success:
     STAT_INC(BINARY_OP, success);
-    cache->counter = miss_counter_start();
+    cache->counter = adaptive_counter_cooldown();
 }
 
 
@@ -1915,69 +2029,24 @@ compare_op_fail_kind(PyObject *lhs, PyObject *rhs)
 }
 #endif
 
-
-static int compare_masks[] = {
-    // 1-bit: jump if less than
-    // 2-bit: jump if equal
-    // 4-bit: jump if greater
-    [Py_LT] = 1 | 0 | 0,
-    [Py_LE] = 1 | 2 | 0,
-    [Py_EQ] = 0 | 2 | 0,
-    [Py_NE] = 1 | 0 | 4,
-    [Py_GT] = 0 | 0 | 4,
-    [Py_GE] = 0 | 2 | 4,
-};
-
 void
 _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
                          int oparg)
 {
+    assert(ENABLE_SPECIALIZATION);
     assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP);
     _PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1);
-    int next_opcode = _Py_OPCODE(instr[INLINE_CACHE_ENTRIES_COMPARE_OP + 1]);
-    if (next_opcode != POP_JUMP_FORWARD_IF_FALSE &&
-        next_opcode != POP_JUMP_BACKWARD_IF_FALSE &&
-        next_opcode != POP_JUMP_FORWARD_IF_TRUE &&
-        next_opcode != POP_JUMP_BACKWARD_IF_TRUE) {
-        // Can't ever combine, so don't don't bother being adaptive (unless
-        // we're collecting stats, where it's more important to get accurate hit
-        // counts for the unadaptive version and each of the different failure
-        // types):
-#ifndef Py_STATS
-        _Py_SET_OPCODE(*instr, COMPARE_OP);
-        return;
-#else
-        if (next_opcode == EXTENDED_ARG) {
-            SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_EXTENDED_ARG);
-            goto failure;
-        }
-        SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_NOT_FOLLOWED_BY_COND_JUMP);
-        goto failure;
-#endif
-    }
-    assert(oparg <= Py_GE);
-    int when_to_jump_mask = compare_masks[oparg];
-    if (next_opcode == POP_JUMP_FORWARD_IF_FALSE ||
-        next_opcode == POP_JUMP_BACKWARD_IF_FALSE) {
-        when_to_jump_mask = (1 | 2 | 4) & ~when_to_jump_mask;
-    }
-    if (next_opcode == POP_JUMP_BACKWARD_IF_TRUE ||
-        next_opcode == POP_JUMP_BACKWARD_IF_FALSE) {
-        when_to_jump_mask <<= 3;
-    }
     if (Py_TYPE(lhs) != Py_TYPE(rhs)) {
         SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs));
         goto failure;
     }
     if (PyFloat_CheckExact(lhs)) {
-        _Py_SET_OPCODE(*instr, COMPARE_OP_FLOAT_JUMP);
-        cache->mask = when_to_jump_mask;
+        instr->op.code = COMPARE_OP_FLOAT;
         goto success;
     }
     if (PyLong_CheckExact(lhs)) {
-        if (Py_ABS(Py_SIZE(lhs)) <= 1 && Py_ABS(Py_SIZE(rhs)) <= 1) {
-            _Py_SET_OPCODE(*instr, COMPARE_OP_INT_JUMP);
-            cache->mask = when_to_jump_mask;
+        if (_PyLong_IsCompact((PyLongObject *)lhs) && _PyLong_IsCompact((PyLongObject *)rhs)) {
+            instr->op.code = COMPARE_OP_INT;
             goto success;
         }
         else {
@@ -1986,24 +2055,25 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
         }
     }
     if (PyUnicode_CheckExact(lhs)) {
-        if (oparg != Py_EQ && oparg != Py_NE) {
+        int cmp = oparg >> 4;
+        if (cmp != Py_EQ && cmp != Py_NE) {
             SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_STRING);
             goto failure;
         }
         else {
-            _Py_SET_OPCODE(*instr, COMPARE_OP_STR_JUMP);
-            cache->mask = when_to_jump_mask;
+            instr->op.code = COMPARE_OP_STR;
             goto success;
         }
     }
     SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs));
 failure:
     STAT_INC(COMPARE_OP, failure);
+    instr->op.code = COMPARE_OP;
     cache->counter = adaptive_counter_backoff(cache->counter);
     return;
 success:
     STAT_INC(COMPARE_OP, success);
-    cache->counter = miss_counter_start();
+    cache->counter = adaptive_counter_cooldown();
 }
 
 #ifdef Py_STATS
@@ -2023,6 +2093,7 @@ unpack_sequence_fail_kind(PyObject *seq)
 void
 _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg)
 {
+    assert(ENABLE_SPECIALIZATION);
     assert(_PyOpcode_Caches[UNPACK_SEQUENCE] ==
            INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE);
     _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)(instr + 1);
@@ -2032,10 +2103,10 @@ _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg)
             goto failure;
         }
         if (PyTuple_GET_SIZE(seq) == 2) {
-            _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE_TWO_TUPLE);
+            instr->op.code = UNPACK_SEQUENCE_TWO_TUPLE;
             goto success;
         }
-        _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE_TUPLE);
+        instr->op.code = UNPACK_SEQUENCE_TUPLE;
         goto success;
     }
     if (PyList_CheckExact(seq)) {
@@ -2043,17 +2114,18 @@ _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg)
             SPECIALIZATION_FAIL(UNPACK_SEQUENCE, SPEC_FAIL_EXPECTED_ERROR);
             goto failure;
         }
-        _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE_LIST);
+        instr->op.code = UNPACK_SEQUENCE_LIST;
         goto success;
     }
     SPECIALIZATION_FAIL(UNPACK_SEQUENCE, unpack_sequence_fail_kind(seq));
 failure:
     STAT_INC(UNPACK_SEQUENCE, failure);
+    instr->op.code = UNPACK_SEQUENCE;
     cache->counter = adaptive_counter_backoff(cache->counter);
     return;
 success:
     STAT_INC(UNPACK_SEQUENCE, success);
-    cache->counter = miss_counter_start();
+    cache->counter = adaptive_counter_cooldown();
 }
 
 #ifdef Py_STATS
@@ -2062,50 +2134,140 @@ int
  _PySpecialization_ClassifyIterator(PyObject *iter)
 {
     if (PyGen_CheckExact(iter)) {
-        return SPEC_FAIL_FOR_ITER_GENERATOR;
+        return SPEC_FAIL_ITER_GENERATOR;
     }
     if (PyCoro_CheckExact(iter)) {
-        return SPEC_FAIL_FOR_ITER_COROUTINE;
+        return SPEC_FAIL_ITER_COROUTINE;
     }
     if (PyAsyncGen_CheckExact(iter)) {
-        return SPEC_FAIL_FOR_ITER_ASYNC_GENERATOR;
+        return SPEC_FAIL_ITER_ASYNC_GENERATOR;
+    }
+    if (PyAsyncGenASend_CheckExact(iter)) {
+        return SPEC_FAIL_ITER_ASYNC_GENERATOR_SEND;
     }
     PyTypeObject *t = Py_TYPE(iter);
     if (t == &PyListIter_Type) {
-        return SPEC_FAIL_FOR_ITER_LIST;
+        return SPEC_FAIL_ITER_LIST;
     }
     if (t == &PyTupleIter_Type) {
-        return SPEC_FAIL_FOR_ITER_TUPLE;
+        return SPEC_FAIL_ITER_TUPLE;
     }
     if (t == &PyDictIterKey_Type) {
-        return SPEC_FAIL_FOR_ITER_DICT_KEYS;
+        return SPEC_FAIL_ITER_DICT_KEYS;
     }
     if (t == &PyDictIterValue_Type) {
-        return SPEC_FAIL_FOR_ITER_DICT_VALUES;
+        return SPEC_FAIL_ITER_DICT_VALUES;
     }
     if (t == &PyDictIterItem_Type) {
-        return SPEC_FAIL_FOR_ITER_DICT_ITEMS;
+        return SPEC_FAIL_ITER_DICT_ITEMS;
     }
     if (t == &PySetIter_Type) {
-        return SPEC_FAIL_FOR_ITER_SET;
+        return SPEC_FAIL_ITER_SET;
     }
     if (t == &PyUnicodeIter_Type) {
-        return SPEC_FAIL_FOR_ITER_STRING;
+        return SPEC_FAIL_ITER_STRING;
     }
     if (t == &PyBytesIter_Type) {
-        return SPEC_FAIL_FOR_ITER_BYTES;
+        return SPEC_FAIL_ITER_BYTES;
     }
     if (t == &PyRangeIter_Type) {
-        return SPEC_FAIL_FOR_ITER_RANGE;
+        return SPEC_FAIL_ITER_RANGE;
     }
     if (t == &PyEnum_Type) {
-        return SPEC_FAIL_FOR_ITER_ENUMERATE;
+        return SPEC_FAIL_ITER_ENUMERATE;
     }
-
-    if (strncmp(t->tp_name, "itertools", 8) == 0) {
-        return SPEC_FAIL_FOR_ITER_ITERTOOLS;
+    if (t == &PyMap_Type) {
+        return SPEC_FAIL_ITER_MAP;
+    }
+    if (t == &PyZip_Type) {
+        return SPEC_FAIL_ITER_ZIP;
+    }
+    if (t == &PySeqIter_Type) {
+        return SPEC_FAIL_ITER_SEQ_ITER;
+    }
+    if (t == &PyListRevIter_Type) {
+        return SPEC_FAIL_ITER_REVERSED_LIST;
+    }
+    if (t == &_PyUnicodeASCIIIter_Type) {
+        return SPEC_FAIL_ITER_ASCII_STRING;
+    }
+    const char *name = t->tp_name;
+    if (strncmp(name, "itertools", 9) == 0) {
+        return SPEC_FAIL_ITER_ITERTOOLS;
+    }
+    if (strncmp(name, "callable_iterator", 17) == 0) {
+        return SPEC_FAIL_ITER_CALLABLE;
     }
     return SPEC_FAIL_OTHER;
 }
 
 #endif
+
+void
+_Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg)
+{
+    assert(ENABLE_SPECIALIZATION);
+    assert(_PyOpcode_Caches[FOR_ITER] == INLINE_CACHE_ENTRIES_FOR_ITER);
+    _PyForIterCache *cache = (_PyForIterCache *)(instr + 1);
+    PyTypeObject *tp = Py_TYPE(iter);
+    if (tp == &PyListIter_Type) {
+        instr->op.code = FOR_ITER_LIST;
+        goto success;
+    }
+    else if (tp == &PyTupleIter_Type) {
+        instr->op.code = FOR_ITER_TUPLE;
+        goto success;
+    }
+    else if (tp == &PyRangeIter_Type) {
+        instr->op.code = FOR_ITER_RANGE;
+        goto success;
+    }
+    else if (tp == &PyGen_Type && oparg <= SHRT_MAX) {
+        assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == END_FOR  ||
+            instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == INSTRUMENTED_END_FOR
+        );
+        if (_PyInterpreterState_GET()->eval_frame) {
+            SPECIALIZATION_FAIL(FOR_ITER, SPEC_FAIL_OTHER);
+            goto failure;
+        }
+        instr->op.code = FOR_ITER_GEN;
+        goto success;
+    }
+    SPECIALIZATION_FAIL(FOR_ITER,
+                        _PySpecialization_ClassifyIterator(iter));
+failure:
+    STAT_INC(FOR_ITER, failure);
+    instr->op.code = FOR_ITER;
+    cache->counter = adaptive_counter_backoff(cache->counter);
+    return;
+success:
+    STAT_INC(FOR_ITER, success);
+    cache->counter = adaptive_counter_cooldown();
+}
+
+void
+_Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr)
+{
+    assert(ENABLE_SPECIALIZATION);
+    assert(_PyOpcode_Caches[SEND] == INLINE_CACHE_ENTRIES_SEND);
+    _PySendCache *cache = (_PySendCache *)(instr + 1);
+    PyTypeObject *tp = Py_TYPE(receiver);
+    if (tp == &PyGen_Type || tp == &PyCoro_Type) {
+        if (_PyInterpreterState_GET()->eval_frame) {
+            SPECIALIZATION_FAIL(SEND, SPEC_FAIL_OTHER);
+            goto failure;
+        }
+        instr->op.code = SEND_GEN;
+        goto success;
+    }
+    SPECIALIZATION_FAIL(SEND,
+                        _PySpecialization_ClassifyIterator(receiver));
+failure:
+    STAT_INC(SEND, failure);
+    instr->op.code = SEND;
+    cache->counter = adaptive_counter_backoff(cache->counter);
+    return;
+success:
+    STAT_INC(SEND, success);
+    cache->counter = adaptive_counter_cooldown();
+}