#include "Python.h"

#include "opcode.h"

#include "pycore_code.h"

#include "pycore_critical_section.h"

#include "pycore_descrobject.h" // _PyMethodWrapper_Type

#include "pycore_dict.h" // DICT_KEYS_UNICODE

#include "pycore_function.h" // _PyFunction_GetVersionForCurrentState()

#include "pycore_long.h" // _PyLong_IsNonNegativeCompact()

#include "pycore_moduleobject.h"

#include "pycore_object.h"

#include "pycore_opcode_metadata.h" // _PyOpcode_Caches

#include "pycore_uop_metadata.h" // _PyOpcode_uop_name

#include "pycore_uop_ids.h" // MAX_UOP_ID

#include "pycore_opcode_utils.h" // RESUME_AT_FUNC_START

#include "pycore_pylifecycle.h" // _PyOS_URandomNonblock()

#include "pycore_runtime.h" // _Py_ID()

#include <stdlib.h> // rand()

extern const char *_PyUOpName(int index);

#ifdef Py_STATS

GCStats _py_gc_stats[NUM_GENERATIONS] = { 0 };

static PyStats _Py_stats_struct = { .gc_stats = _py_gc_stats };

PyStats *_Py_stats = NULL;

#if PYSTATS_MAX_UOP_ID < MAX_UOP_ID

#error "Not enough space allocated for pystats. Increase PYSTATS_MAX_UOP_ID to at least MAX_UOP_ID"

#define ADD_STAT_TO_DICT(res, field) \

do { \

PyObject *val = PyLong_FromUnsignedLongLong(stats->field); \

if (val == NULL) { \

Py_DECREF(res); \

return NULL; \

} \

if (PyDict_SetItemString(res, #field, val) == -1) { \

Py_DECREF(res); \

Py_DECREF(val); \

return NULL; \

} \

Py_DECREF(val); \

} while(0);

static PyObject*

stats_to_dict(SpecializationStats *stats)

{

PyObject *res = PyDict_New();

if (res == NULL) {

return NULL;

}

ADD_STAT_TO_DICT(res, success);

ADD_STAT_TO_DICT(res, failure);

ADD_STAT_TO_DICT(res, hit);

ADD_STAT_TO_DICT(res, deferred);

ADD_STAT_TO_DICT(res, miss);

ADD_STAT_TO_DICT(res, deopt);

PyObject *failure_kinds = PyTuple_New(SPECIALIZATION_FAILURE_KINDS);

if (failure_kinds == NULL) {

Py_DECREF(res);

return NULL;

}

for (int i = 0; i < SPECIALIZATION_FAILURE_KINDS; i++) {

PyObject *stat = PyLong_FromUnsignedLongLong(stats->failure_kinds[i]);

if (stat == NULL) {

Py_DECREF(res);

Py_DECREF(failure_kinds);

return NULL;

}

PyTuple_SET_ITEM(failure_kinds, i, stat);

}

if (PyDict_SetItemString(res, "failure_kinds", failure_kinds)) {

Py_DECREF(res);

Py_DECREF(failure_kinds);

return NULL;

}

Py_DECREF(failure_kinds);

return res;

}

#undef ADD_STAT_TO_DICT

static int

add_stat_dict(

PyObject *res,

int opcode,

const char *name) {

SpecializationStats *stats = &_Py_stats_struct.opcode_stats[opcode].specialization;

PyObject *d = stats_to_dict(stats);

if (d == NULL) {

return -1;

}

int err = PyDict_SetItemString(res, name, d);

Py_DECREF(d);

return err;

}

_Py_GetSpecializationStats(void) {

PyObject *stats = PyDict_New();

if (stats == NULL) {

return NULL;

}

int err = 0;

err += add_stat_dict(stats, CONTAINS_OP, "contains_op");

err += add_stat_dict(stats, LOAD_SUPER_ATTR, "load_super_attr");

err += add_stat_dict(stats, LOAD_ATTR, "load_attr");

err += add_stat_dict(stats, LOAD_GLOBAL, "load_global");

err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr");

err += add_stat_dict(stats, STORE_ATTR, "store_attr");

err += add_stat_dict(stats, CALL, "call");

err += add_stat_dict(stats, CALL_KW, "call_kw");

err += add_stat_dict(stats, BINARY_OP, "binary_op");

err += add_stat_dict(stats, COMPARE_OP, "compare_op");

err += add_stat_dict(stats, UNPACK_SEQUENCE, "unpack_sequence");

err += add_stat_dict(stats, FOR_ITER, "for_iter");

err += add_stat_dict(stats, TO_BOOL, "to_bool");

err += add_stat_dict(stats, SEND, "send");

if (err < 0) {

Py_DECREF(stats);

return NULL;

}

return stats;

}

#define PRINT_STAT(i, field) \

if (stats[i].field) { \

fprintf(out, " opcode[%s]." #field " : %" PRIu64 "\n", _PyOpcode_OpName[i], stats[i].field); \

}

static void

print_spec_stats(FILE *out, OpcodeStats *stats)

{

/* Mark some opcodes as specializable for stats,

fprintf(out, "opcode[BINARY_SLICE].specializable : 1\n");

fprintf(out, "opcode[STORE_SLICE].specializable : 1\n");

for (int i = 0; i < 256; i++) {

if (_PyOpcode_Caches[i]) {

/* Ignore jumps as they cannot be specialized */

switch (i) {

case POP_JUMP_IF_FALSE:

case POP_JUMP_IF_TRUE:

case POP_JUMP_IF_NONE:

case POP_JUMP_IF_NOT_NONE:

case JUMP_BACKWARD:

break;

default:

fprintf(out, "opcode[%s].specializable : 1\n", _PyOpcode_OpName[i]);

}

}

PRINT_STAT(i, specialization.success);

PRINT_STAT(i, specialization.failure);

PRINT_STAT(i, specialization.hit);

PRINT_STAT(i, specialization.deferred);

PRINT_STAT(i, specialization.miss);

PRINT_STAT(i, specialization.deopt);

PRINT_STAT(i, execution_count);

for (int j = 0; j < SPECIALIZATION_FAILURE_KINDS; j++) {

uint64_t val = stats[i].specialization.failure_kinds[j];

if (val) {

fprintf(out, " opcode[%s].specialization.failure_kinds[%d] : %"

PRIu64 "\n", _PyOpcode_OpName[i], j, val);

}

}

for (int j = 0; j < 256; j++) {

if (stats[i].pair_count[j]) {

fprintf(out, "opcode[%s].pair_count[%s] : %" PRIu64 "\n",

_PyOpcode_OpName[i], _PyOpcode_OpName[j], stats[i].pair_count[j]);

}

}

}

}

#undef PRINT_STAT

static void

print_call_stats(FILE *out, CallStats *stats)

{

fprintf(out, "Calls to PyEval_EvalDefault: %" PRIu64 "\n", stats->pyeval_calls);

fprintf(out, "Calls to Python functions inlined: %" PRIu64 "\n", stats->inlined_py_calls);

fprintf(out, "Frames pushed: %" PRIu64 "\n", stats->frames_pushed);

fprintf(out, "Frame objects created: %" PRIu64 "\n", stats->frame_objects_created);

for (int i = 0; i < EVAL_CALL_KINDS; i++) {

fprintf(out, "Calls via PyEval_EvalFrame[%d] : %" PRIu64 "\n", i, stats->eval_calls[i]);

}

}

static void

print_object_stats(FILE *out, ObjectStats *stats)

{

fprintf(out, "Object allocations from freelist: %" PRIu64 "\n", stats->from_freelist);

fprintf(out, "Object frees to freelist: %" PRIu64 "\n", stats->to_freelist);

fprintf(out, "Object allocations: %" PRIu64 "\n", stats->allocations);

fprintf(out, "Object allocations to 512 bytes: %" PRIu64 "\n", stats->allocations512);

fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n", stats->allocations4k);

fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big);

fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees);

fprintf(out, "Object inline values: %" PRIu64 "\n", stats->inline_values);

fprintf(out, "Object interpreter mortal increfs: %" PRIu64 "\n", stats->interpreter_increfs);

fprintf(out, "Object interpreter mortal decrefs: %" PRIu64 "\n", stats->interpreter_decrefs);

fprintf(out, "Object mortal increfs: %" PRIu64 "\n", stats->increfs);

fprintf(out, "Object mortal decrefs: %" PRIu64 "\n", stats->decrefs);

fprintf(out, "Object interpreter immortal increfs: %" PRIu64 "\n", stats->interpreter_immortal_increfs);

fprintf(out, "Object interpreter immortal decrefs: %" PRIu64 "\n", stats->interpreter_immortal_decrefs);

fprintf(out, "Object immortal increfs: %" PRIu64 "\n", stats->immortal_increfs);

fprintf(out, "Object immortal decrefs: %" PRIu64 "\n", stats->immortal_decrefs);

fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request);

fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key);

fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big);

fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass);

fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits);

fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses);

fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions);

fprintf(out, "Object method cache dunder hits: %" PRIu64 "\n", stats->type_cache_dunder_hits);

fprintf(out, "Object method cache dunder misses: %" PRIu64 "\n", stats->type_cache_dunder_misses);

}

static void

print_gc_stats(FILE *out, GCStats *stats)

{

for (int i = 0; i < NUM_GENERATIONS; i++) {

fprintf(out, "GC[%d] collections: %" PRIu64 "\n", i, stats[i].collections);

fprintf(out, "GC[%d] object visits: %" PRIu64 "\n", i, stats[i].object_visits);

fprintf(out, "GC[%d] objects collected: %" PRIu64 "\n", i, stats[i].objects_collected);

fprintf(out, "GC[%d] objects reachable from roots: %" PRIu64 "\n", i, stats[i].objects_transitively_reachable);

fprintf(out, "GC[%d] objects not reachable from roots: %" PRIu64 "\n", i, stats[i].objects_not_transitively_reachable);

}

}

#ifdef _Py_TIER2

static void

print_histogram(FILE *out, const char *name, uint64_t hist[_Py_UOP_HIST_SIZE])

{

for (int i = 0; i < _Py_UOP_HIST_SIZE; i++) {

fprintf(out, "%s[%" PRIu64"]: %" PRIu64 "\n", name, (uint64_t)1 << i, hist[i]);

}

}

static void

print_optimization_stats(FILE *out, OptimizationStats *stats)

{

fprintf(out, "Optimization attempts: %" PRIu64 "\n", stats->attempts);

fprintf(out, "Optimization traces created: %" PRIu64 "\n", stats->traces_created);

fprintf(out, "Optimization traces executed: %" PRIu64 "\n", stats->traces_executed);

fprintf(out, "Optimization uops executed: %" PRIu64 "\n", stats->uops_executed);

fprintf(out, "Optimization trace stack overflow: %" PRIu64 "\n", stats->trace_stack_overflow);

fprintf(out, "Optimization trace stack underflow: %" PRIu64 "\n", stats->trace_stack_underflow);

fprintf(out, "Optimization trace too long: %" PRIu64 "\n", stats->trace_too_long);

fprintf(out, "Optimization trace too short: %" PRIu64 "\n", stats->trace_too_short);

fprintf(out, "Optimization inner loop: %" PRIu64 "\n", stats->inner_loop);

fprintf(out, "Optimization recursive call: %" PRIu64 "\n", stats->recursive_call);

fprintf(out, "Optimization low confidence: %" PRIu64 "\n", stats->low_confidence);

fprintf(out, "Optimization unknown callee: %" PRIu64 "\n", stats->unknown_callee);

fprintf(out, "Executors invalidated: %" PRIu64 "\n", stats->executors_invalidated);

print_histogram(out, "Trace length", stats->trace_length_hist);

print_histogram(out, "Trace run length", stats->trace_run_length_hist);

print_histogram(out, "Optimized trace length", stats->optimized_trace_length_hist);

fprintf(out, "Optimization optimizer attempts: %" PRIu64 "\n", stats->optimizer_attempts);

fprintf(out, "Optimization optimizer successes: %" PRIu64 "\n", stats->optimizer_successes);

fprintf(out, "Optimization optimizer failure no memory: %" PRIu64 "\n",

stats->optimizer_failure_reason_no_memory);

fprintf(out, "Optimizer remove globals builtins changed: %" PRIu64 "\n", stats->remove_globals_builtins_changed);

fprintf(out, "Optimizer remove globals incorrect keys: %" PRIu64 "\n", stats->remove_globals_incorrect_keys);

for (int i = 0; i <= MAX_UOP_ID; i++) {

if (stats->opcode[i].execution_count) {

fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].execution_count);

}

if (stats->opcode[i].miss) {

fprintf(out, "uops[%s].specialization.miss : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].miss);

}

}

for (int i = 0; i < 256; i++) {

if (stats->unsupported_opcode[i]) {

fprintf(

out,

"unsupported_opcode[%s].count : %" PRIu64 "\n",

_PyOpcode_OpName[i],

stats->unsupported_opcode[i]

);

}

}

for (int i = 1; i <= MAX_UOP_ID; i++){

for (int j = 1; j <= MAX_UOP_ID; j++) {

if (stats->opcode[i].pair_count[j]) {

fprintf(out, "uop[%s].pair_count[%s] : %" PRIu64 "\n",

_PyOpcode_uop_name[i], _PyOpcode_uop_name[j], stats->opcode[i].pair_count[j]);

}

}

}

for (int i = 0; i < MAX_UOP_ID; i++) {

if (stats->error_in_opcode[i]) {

fprintf(

out,

"error_in_opcode[%s].count : %" PRIu64 "\n",

_PyUOpName(i),

stats->error_in_opcode[i]

);

}

}

fprintf(out, "JIT total memory size: %" PRIu64 "\n", stats->jit_total_memory_size);

fprintf(out, "JIT code size: %" PRIu64 "\n", stats->jit_code_size);

fprintf(out, "JIT trampoline size: %" PRIu64 "\n", stats->jit_trampoline_size);

fprintf(out, "JIT data size: %" PRIu64 "\n", stats->jit_data_size);

fprintf(out, "JIT padding size: %" PRIu64 "\n", stats->jit_padding_size);

fprintf(out, "JIT freed memory size: %" PRIu64 "\n", stats->jit_freed_memory_size);

print_histogram(out, "Trace total memory size", stats->trace_total_memory_hist);

}

static void

print_rare_event_stats(FILE *out, RareEventStats *stats)

{

fprintf(out, "Rare event (set_class): %" PRIu64 "\n", stats->set_class);

fprintf(out, "Rare event (set_bases): %" PRIu64 "\n", stats->set_bases);

fprintf(out, "Rare event (set_eval_frame_func): %" PRIu64 "\n", stats->set_eval_frame_func);

fprintf(out, "Rare event (builtin_dict): %" PRIu64 "\n", stats->builtin_dict);

fprintf(out, "Rare event (func_modification): %" PRIu64 "\n", stats->func_modification);

fprintf(out, "Rare event (watched_dict_modification): %" PRIu64 "\n", stats->watched_dict_modification);

fprintf(out, "Rare event (watched_globals_modification): %" PRIu64 "\n", stats->watched_globals_modification);

}

static void

print_stats(FILE *out, PyStats *stats)

{

print_spec_stats(out, stats->opcode_stats);

print_call_stats(out, &stats->call_stats);

print_object_stats(out, &stats->object_stats);

print_gc_stats(out, stats->gc_stats);

#ifdef _Py_TIER2

print_optimization_stats(out, &stats->optimization_stats);

print_rare_event_stats(out, &stats->rare_event_stats);

}

_Py_StatsOn(void)

{

_Py_stats = &_Py_stats_struct;

}

_Py_StatsOff(void)

{

_Py_stats = NULL;

}

_Py_StatsClear(void)

{

memset(&_py_gc_stats, 0, sizeof(_py_gc_stats));

memset(&_Py_stats_struct, 0, sizeof(_Py_stats_struct));

_Py_stats_struct.gc_stats = _py_gc_stats;

}

static int

mem_is_zero(unsigned char *ptr, size_t size)

{

for (size_t i=0; i < size; i++) {

if (*ptr != 0) {

return 0;

}

ptr++;

}

return 1;

}

_Py_PrintSpecializationStats(int to_file)

{

PyStats *stats = &_Py_stats_struct;

#define MEM_IS_ZERO(DATA) mem_is_zero((unsigned char*)DATA, sizeof(*(DATA)))

int is_zero = (

MEM_IS_ZERO(stats->gc_stats) // is a pointer

&& MEM_IS_ZERO(&stats->opcode_stats)

&& MEM_IS_ZERO(&stats->call_stats)

&& MEM_IS_ZERO(&stats->object_stats)

);

#undef MEM_IS_ZERO

if (is_zero) {

// gh-108753: -X pystats command line was used, but then _stats_off()

// and _stats_clear() have been called: in this case, avoid printing

// useless "all zeros" statistics.

return 0;

}

FILE *out = stderr;

if (to_file) {

/* Write to a file instead of stderr. */

# ifdef MS_WINDOWS

const char *dirname = "c:\\temp\\py_stats\\";

const char *dirname = "/tmp/py_stats/";

/* Use random 160 bit number as file name,

unsigned char rand[20];

char hex_name[41];

_PyOS_URandomNonblock(rand, 20);

for (int i = 0; i < 20; i++) {

hex_name[2*i] = Py_hexdigits[rand[i]&15];

hex_name[2*i+1] = Py_hexdigits[(rand[i]>>4)&15];

}

hex_name[40] = '\0';

char buf[64];

assert(strlen(dirname) + 40 + strlen(".txt") < 64);

sprintf(buf, "%s%s.txt", dirname, hex_name);

FILE *fout = fopen(buf, "w");

if (fout) {

out = fout;

}

}

else {

fprintf(out, "Specialization stats:\n");

}

print_stats(out, stats);

if (out != stderr) {

fclose(out);

}

return 1;

}

#define SPECIALIZATION_FAIL(opcode, kind) \

do { \

if (_Py_stats) { \

_Py_stats->opcode_stats[opcode].specialization.failure_kinds[kind]++; \

} \

} while (0)

#endif // Py_STATS

#ifndef SPECIALIZATION_FAIL

# define SPECIALIZATION_FAIL(opcode, kind) ((void)0)

_PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, int enable_counters)

{

#if ENABLE_SPECIALIZATION_FT

_Py_BackoffCounter jump_counter, adaptive_counter;

if (enable_counters) {

jump_counter = initial_jump_backoff_counter();

adaptive_counter = adaptive_counter_warmup();

}

else {

jump_counter = initial_unreachable_backoff_counter();

adaptive_counter = initial_unreachable_backoff_counter();

}

int opcode = 0;

int oparg = 0;

/* The last code unit cannot have a cache, so we don't need to check it */

for (Py_ssize_t i = 0; i < size-1; i++) {

opcode = instructions[i].op.code;

int caches = _PyOpcode_Caches[opcode];

oparg = (oparg << 8) | instructions[i].op.arg;

if (caches) {

// The initial value depends on the opcode

switch (opcode) {

case JUMP_BACKWARD:

instructions[i + 1].counter = jump_counter;

break;

case POP_JUMP_IF_FALSE:

case POP_JUMP_IF_TRUE:

case POP_JUMP_IF_NONE:

case POP_JUMP_IF_NOT_NONE:

instructions[i + 1].cache = 0x5555; // Alternating 0, 1 bits

break;

default:

instructions[i + 1].counter = adaptive_counter;

break;

}

i += caches;

}

if (opcode != EXTENDED_ARG) {

oparg = 0;

}

}

#endif /* ENABLE_SPECIALIZATION_FT */

}

#define SIMPLE_FUNCTION 0

#define SPEC_FAIL_OTHER 0

#define SPEC_FAIL_NO_DICT 1

#define SPEC_FAIL_OVERRIDDEN 2

#define SPEC_FAIL_OUT_OF_VERSIONS 3

#define SPEC_FAIL_OUT_OF_RANGE 4

#define SPEC_FAIL_EXPECTED_ERROR 5

#define SPEC_FAIL_WRONG_NUMBER_ARGUMENTS 6

#define SPEC_FAIL_CODE_COMPLEX_PARAMETERS 7

#define SPEC_FAIL_CODE_NOT_OPTIMIZED 8

#define SPEC_FAIL_LOAD_GLOBAL_NON_DICT 17

#define SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT 18

#define SPEC_FAIL_SUPER_BAD_CLASS 9

#define SPEC_FAIL_SUPER_SHADOWED 10

#define SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR 9

#define SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR 10

#define SPEC_FAIL_ATTR_NOT_DESCRIPTOR 11

#define SPEC_FAIL_ATTR_METHOD 12

#define SPEC_FAIL_ATTR_MUTABLE_CLASS 13

#define SPEC_FAIL_ATTR_PROPERTY 14

#define SPEC_FAIL_ATTR_NON_OBJECT_SLOT 15

#define SPEC_FAIL_ATTR_READ_ONLY 16

#define SPEC_FAIL_ATTR_AUDITED_SLOT 17

#define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 18

#define SPEC_FAIL_ATTR_NON_STRING 19

#define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 20

#define SPEC_FAIL_ATTR_SHADOWED 21

#define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22

#define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23

#define SPEC_FAIL_ATTR_OBJECT_SLOT 24

#define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26

#define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27

#define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28

#define SPEC_FAIL_ATTR_NOT_IN_KEYS 29

#define SPEC_FAIL_ATTR_NOT_IN_DICT 30

#define SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE 31

#define SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR 32

#define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ 33

#define SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN 34

#define SPEC_FAIL_ATTR_SPLIT_DICT 35

#define SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED 36

#define SPEC_FAIL_SUBSCR_ARRAY_INT 9

#define SPEC_FAIL_SUBSCR_ARRAY_SLICE 10

#define SPEC_FAIL_SUBSCR_LIST_SLICE 11

#define SPEC_FAIL_SUBSCR_BUFFER_INT 12

#define SPEC_FAIL_SUBSCR_BUFFER_SLICE 13

#define SPEC_FAIL_SUBSCR_BYTEARRAY_INT 18

#define SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE 19

#define SPEC_FAIL_SUBSCR_PY_SIMPLE 20

#define SPEC_FAIL_SUBSCR_PY_OTHER 21

#define SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE 22

#define SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE 23

#define SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES 9

#define SPEC_FAIL_BINARY_OP_ADD_OTHER 10

#define SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES 11

#define SPEC_FAIL_BINARY_OP_AND_INT 12

#define SPEC_FAIL_BINARY_OP_AND_OTHER 13

#define SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE 14

#define SPEC_FAIL_BINARY_OP_LSHIFT 15

#define SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY 16

#define SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES 17

#define SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER 18

#define SPEC_FAIL_BINARY_OP_OR 19

#define SPEC_FAIL_BINARY_OP_POWER 20

#define SPEC_FAIL_BINARY_OP_REMAINDER 21

#define SPEC_FAIL_BINARY_OP_RSHIFT 22

#define SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES 23

#define SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER 24

#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES 25

#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT 26

#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER 27

#define SPEC_FAIL_BINARY_OP_XOR 28

#define SPEC_FAIL_BINARY_OP_OR_INT 29

#define SPEC_FAIL_BINARY_OP_OR_DIFFERENT_TYPES 30

#define SPEC_FAIL_BINARY_OP_XOR_INT 31

#define SPEC_FAIL_BINARY_OP_XOR_DIFFERENT_TYPES 32

#define SPEC_FAIL_BINARY_OP_SUBSCR 33

#define SPEC_FAIL_BINARY_OP_SUBSCR_LIST_SLICE 34

#define SPEC_FAIL_BINARY_OP_SUBSCR_TUPLE_SLICE 35

#define SPEC_FAIL_BINARY_OP_SUBSCR_STRING_SLICE 36

#define SPEC_FAIL_BINARY_OP_SUBSCR_NOT_HEAP_TYPE 37

#define SPEC_FAIL_CALL_INSTANCE_METHOD 11

#define SPEC_FAIL_CALL_CMETHOD 12

#define SPEC_FAIL_CALL_CFUNC_VARARGS 13

#define SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS 14

#define SPEC_FAIL_CALL_CFUNC_NOARGS 15

#define SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS 16

#define SPEC_FAIL_CALL_METH_DESCR_VARARGS 17

#define SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS 18

#define SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS 19

#define SPEC_FAIL_CALL_BAD_CALL_FLAGS 20

#define SPEC_FAIL_CALL_INIT_NOT_PYTHON 21

#define SPEC_FAIL_CALL_PEP_523 22

#define SPEC_FAIL_CALL_BOUND_METHOD 23

#define SPEC_FAIL_CALL_CLASS_MUTABLE 26

#define SPEC_FAIL_CALL_METHOD_WRAPPER 28

#define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29

#define SPEC_FAIL_CALL_INIT_NOT_SIMPLE 30

#define SPEC_FAIL_CALL_METACLASS 31

#define SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES 32

#define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12

#define SPEC_FAIL_COMPARE_OP_STRING 13

#define SPEC_FAIL_COMPARE_OP_BIG_INT 14

#define SPEC_FAIL_COMPARE_OP_BYTES 15

#define SPEC_FAIL_COMPARE_OP_TUPLE 16

#define SPEC_FAIL_COMPARE_OP_LIST 17

#define SPEC_FAIL_COMPARE_OP_SET 18

#define SPEC_FAIL_COMPARE_OP_BOOL 19

#define SPEC_FAIL_COMPARE_OP_BASEOBJECT 20

#define SPEC_FAIL_COMPARE_OP_FLOAT_LONG 21

#define SPEC_FAIL_COMPARE_OP_LONG_FLOAT 22

#define SPEC_FAIL_ITER_GENERATOR 10

#define SPEC_FAIL_ITER_COROUTINE 11

#define SPEC_FAIL_ITER_ASYNC_GENERATOR 12

#define SPEC_FAIL_ITER_LIST 13

#define SPEC_FAIL_ITER_TUPLE 14

#define SPEC_FAIL_ITER_SET 15

#define SPEC_FAIL_ITER_STRING 16

#define SPEC_FAIL_ITER_BYTES 17

#define SPEC_FAIL_ITER_RANGE 18

#define SPEC_FAIL_ITER_ITERTOOLS 19

#define SPEC_FAIL_ITER_DICT_KEYS 20

#define SPEC_FAIL_ITER_DICT_ITEMS 21

#define SPEC_FAIL_ITER_DICT_VALUES 22

#define SPEC_FAIL_ITER_ENUMERATE 23

#define SPEC_FAIL_ITER_MAP 24

#define SPEC_FAIL_ITER_ZIP 25

#define SPEC_FAIL_ITER_SEQ_ITER 26

#define SPEC_FAIL_ITER_REVERSED_LIST 27

#define SPEC_FAIL_ITER_CALLABLE 28

#define SPEC_FAIL_ITER_ASCII_STRING 29

#define SPEC_FAIL_ITER_ASYNC_GENERATOR_SEND 30

#define SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR 9

#define SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE 10

#define SPEC_FAIL_TO_BOOL_BYTEARRAY 9

#define SPEC_FAIL_TO_BOOL_BYTES 10

#define SPEC_FAIL_TO_BOOL_DICT 11

#define SPEC_FAIL_TO_BOOL_FLOAT 12

#define SPEC_FAIL_TO_BOOL_MAPPING 13

#define SPEC_FAIL_TO_BOOL_MEMORY_VIEW 14

#define SPEC_FAIL_TO_BOOL_NUMBER 15

#define SPEC_FAIL_TO_BOOL_SEQUENCE 16

#define SPEC_FAIL_TO_BOOL_SET 17

#define SPEC_FAIL_TO_BOOL_TUPLE 18

#define SPEC_FAIL_CONTAINS_OP_STR 9

#define SPEC_FAIL_CONTAINS_OP_TUPLE 10

#define SPEC_FAIL_CONTAINS_OP_LIST 11

#define SPEC_FAIL_CONTAINS_OP_USER_CLASS 12

static inline int

set_opcode(_Py_CODEUNIT *instr, uint8_t opcode)

{

#ifdef Py_GIL_DISABLED

uint8_t old_op = _Py_atomic_load_uint8_relaxed(&instr->op.code);

if (old_op >= MIN_INSTRUMENTED_OPCODE) {

/* Lost race with instrumentation */

return 0;

}

if (!_Py_atomic_compare_exchange_uint8(&instr->op.code, &old_op, opcode)) {

/* Lost race with instrumentation */

assert(old_op >= MIN_INSTRUMENTED_OPCODE);

return 0;

}

return 1;

instr->op.code = opcode;

return 1;

}

static inline void

set_counter(_Py_BackoffCounter *counter, _Py_BackoffCounter value)

{

FT_ATOMIC_STORE_UINT16_RELAXED(counter->value_and_backoff,

value.value_and_backoff);

}

static inline _Py_BackoffCounter

load_counter(_Py_BackoffCounter *counter)

{

_Py_BackoffCounter result = {

.value_and_backoff =

FT_ATOMIC_LOAD_UINT16_RELAXED(counter->value_and_backoff)};

return result;

}

static inline void

specialize(_Py_CODEUNIT *instr, uint8_t specialized_opcode)

{

assert(!PyErr_Occurred());

if (!set_opcode(instr, specialized_opcode)) {

STAT_INC(_PyOpcode_Deopt[specialized_opcode], failure);

SPECIALIZATION_FAIL(_PyOpcode_Deopt[specialized_opcode],

SPEC_FAIL_OTHER);

return;

}

STAT_INC(_PyOpcode_Deopt[specialized_opcode], success);

set_counter((_Py_BackoffCounter *)instr + 1, adaptive_counter_cooldown());

}

static inline void

unspecialize(_Py_CODEUNIT *instr)

{

assert(!PyErr_Occurred());

uint8_t opcode = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.code);

uint8_t generic_opcode = _PyOpcode_Deopt[opcode];

STAT_INC(generic_opcode, failure);

if (!set_opcode(instr, generic_opcode)) {

SPECIALIZATION_FAIL(generic_opcode, SPEC_FAIL_OTHER);

return;

}

_Py_BackoffCounter *counter = (_Py_BackoffCounter *)instr + 1;

_Py_BackoffCounter cur = load_counter(counter);

set_counter(counter, adaptive_counter_backoff(cur));

}

static int function_kind(PyCodeObject *code);

static bool function_check_args(PyObject *o, int expected_argcount, int opcode);

static uint32_t function_get_version(PyObject *o, int opcode);

static int

specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr, PyObject *name)

{

_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);

if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_STRING);

return -1;

}

Py_ssize_t index = _PyDict_LookupIndex(dict, &_Py_ID(__getattr__));

assert(index != DKIX_ERROR);

if (index != DKIX_EMPTY) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND);

return -1;

}

index = _PyDict_LookupIndex(dict, name);

assert (index != DKIX_ERROR);

if (index != (uint16_t)index) {

SPECIALIZATION_FAIL(LOAD_ATTR,

index == DKIX_EMPTY ?

SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND :

SPEC_FAIL_OUT_OF_RANGE);

return -1;

}

uint32_t keys_version = _PyDict_GetKeysVersionForCurrentState(

_PyInterpreterState_GET(), dict);

if (keys_version == 0) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);

return -1;

}

write_u32(cache->version, keys_version);

cache->index = (uint16_t)index;

specialize(instr, LOAD_ATTR_MODULE);

return 0;

}

static int

specialize_module_load_attr(

PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)

{

PyModuleObject *m = (PyModuleObject *)owner;

assert((Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);

PyDictObject *dict = (PyDictObject *)m->md_dict;

if (dict == NULL) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_NO_DICT);

return -1;

}

int result;

Py_BEGIN_CRITICAL_SECTION(dict);

result = specialize_module_load_attr_lock_held(dict, instr, name);

Py_END_CRITICAL_SECTION();

return result;

}

_Py_Specialize_LoadSuperAttr(_PyStackRef global_super_st, _PyStackRef cls_st, _Py_CODEUNIT *instr, int load_method) {

PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);

PyObject *cls = PyStackRef_AsPyObjectBorrow(cls_st);

assert(ENABLE_SPECIALIZATION_FT);

assert(_PyOpcode_Caches[LOAD_SUPER_ATTR] == INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR);

if (global_super != (PyObject *)&PySuper_Type) {

SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_SHADOWED);

goto fail;

}

if (!PyType_Check(cls)) {

SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_BAD_CLASS);

goto fail;

}

uint8_t load_code = load_method ? LOAD_SUPER_ATTR_METHOD : LOAD_SUPER_ATTR_ATTR;

specialize(instr, load_code);

return;

fail:

unspecialize(instr);

}

typedef enum {

OVERRIDING, /* Is an overriding descriptor, and will remain so. */

METHOD, /* Attribute has Py_TPFLAGS_METHOD_DESCRIPTOR set */

PROPERTY, /* Is a property */

OBJECT_SLOT, /* Is an object slot descriptor */

OTHER_SLOT, /* Is a slot descriptor of another type */

NON_OVERRIDING, /* Is another non-overriding descriptor, and is an instance of an immutable class*/

BUILTIN_CLASSMETHOD, /* Builtin methods with METH_CLASS */

PYTHON_CLASSMETHOD, /* Python classmethod(func) object */

NON_DESCRIPTOR, /* Is not a descriptor, and is an instance of an immutable class */

MUTABLE, /* Instance of a mutable class; might, or might not, be a descriptor */

ABSENT, /* Attribute is not present on the class */

DUNDER_CLASS, /* __class__ attribute */

GETSET_OVERRIDDEN, /* __getattribute__ or __setattr__ has been overridden */

GETATTRIBUTE_IS_PYTHON_FUNCTION /* Descriptor requires calling a Python __getattribute__ */

} DescriptorClassification;

static DescriptorClassification

classify_descriptor(PyObject *descriptor, bool has_getattr)

{

if (descriptor == NULL) {

return ABSENT;

}

PyTypeObject *desc_cls = Py_TYPE(descriptor);

if (!(desc_cls->tp_flags & Py_TPFLAGS_IMMUTABLETYPE)) {

return MUTABLE;

}

if (desc_cls->tp_descr_set) {

if (desc_cls == &PyMemberDescr_Type) {

PyMemberDescrObject *member = (PyMemberDescrObject *)descriptor;

struct PyMemberDef *dmem = member->d_member;

if (dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT) {

return OBJECT_SLOT;

}

return OTHER_SLOT;

}

if (desc_cls == &PyProperty_Type) {

/* We can't detect at runtime whether an attribute exists

return has_getattr ? GETSET_OVERRIDDEN : PROPERTY;

}

return OVERRIDING;

}

if (desc_cls->tp_descr_get) {

if (desc_cls->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR) {

return METHOD;

}

if (Py_IS_TYPE(descriptor, &PyClassMethodDescr_Type)) {

return BUILTIN_CLASSMETHOD;

}

if (Py_IS_TYPE(descriptor, &PyClassMethod_Type)) {

return PYTHON_CLASSMETHOD;

}

return NON_OVERRIDING;

}

return NON_DESCRIPTOR;

}

static bool

descriptor_is_class(PyObject *descriptor, PyObject *name)

{

return ((PyUnicode_CompareWithASCIIString(name, "__class__") == 0) &&

(descriptor == _PyType_Lookup(&PyBaseObject_Type, name)));

}

static DescriptorClassification

analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version) {

bool has_getattr = false;

bool have_ga_version = false;

unsigned int ga_version;

getattrofunc getattro_slot = type->tp_getattro;

if (getattro_slot == PyObject_GenericGetAttr) {

/* Normal attribute lookup; */

has_getattr = false;

}

else if (getattro_slot == _Py_slot_tp_getattr_hook ||

getattro_slot == _Py_slot_tp_getattro) {

/* One or both of __getattribute__ or __getattr__ may have been

PyObject *getattribute = _PyType_LookupRefAndVersion(type,

&_Py_ID(__getattribute__), &ga_version);

have_ga_version = true;

PyInterpreterState *interp = _PyInterpreterState_GET();

bool has_custom_getattribute = getattribute != NULL &&

getattribute != interp->callable_cache.object__getattribute__;

PyObject *getattr = _PyType_Lookup(type, &_Py_ID(__getattr__));

has_getattr = getattr != NULL;

if (has_custom_getattribute) {

if (getattro_slot == _Py_slot_tp_getattro &&

!has_getattr &&

Py_IS_TYPE(getattribute, &PyFunction_Type)) {

*descr = getattribute;

*tp_version = ga_version;

return GETATTRIBUTE_IS_PYTHON_FUNCTION;

}

/* Potentially both __getattr__ and __getattribute__ are set.

Py_DECREF(getattribute);

*descr = NULL;

*tp_version = ga_version;

return GETSET_OVERRIDDEN;

}

/* Potentially has __getattr__ but no custom __getattribute__.

Py_XDECREF(getattribute);

}

else {

*descr = NULL;

*tp_version = FT_ATOMIC_LOAD_UINT_RELAXED(type->tp_version_tag);

return GETSET_OVERRIDDEN;

}

unsigned int descr_version;

PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, &descr_version);

*descr = descriptor;

*tp_version = have_ga_version ? ga_version : descr_version;

if (descriptor_is_class(descriptor, name)) {

return DUNDER_CLASS;

}

return classify_descriptor(descriptor, has_getattr);

}

static DescriptorClassification

analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version)

{

if (type->tp_setattro != PyObject_GenericSetAttr) {

*descr = NULL;

return GETSET_OVERRIDDEN;

}

PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, tp_version);

*descr = descriptor;

if (descriptor_is_class(descriptor, name)) {

return DUNDER_CLASS;

}

return classify_descriptor(descriptor, false);

}

static int

specialize_dict_access_inline(

PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,

PyObject *name, unsigned int tp_version,

int base_op, int values_op)

{

_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);

PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;

assert(PyUnicode_CheckExact(name));

_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(owner);

Py_ssize_t index = _PyDictKeys_StringLookupSplit(keys, name);

assert (index != DKIX_ERROR);

if (index == DKIX_EMPTY) {

SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_IN_KEYS);

return 0;

}

assert(index >= 0);

assert(_PyObject_InlineValues(owner)->valid);

char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index];

Py_ssize_t offset = value_addr - (char *)owner;

if (offset != (uint16_t)offset) {

SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);

return 0;

}

cache->index = (uint16_t)offset;

write_u32(cache->version, tp_version);

specialize(instr, values_op);

return 1;

}

static int