#include "Python.h"

#include "pycore_code.h"

#include "pycore_dict.h"

#include "pycore_function.h" // _PyFunction_GetVersionForCurrentState()

#include "pycore_global_strings.h" // _Py_ID()

#include "pycore_long.h"

#include "pycore_moduleobject.h"

#include "pycore_object.h"

#include "pycore_opcode.h" // _PyOpcode_Caches

#include "structmember.h" // struct PyMemberDef, T_OFFSET_EX

#include "pycore_descrobject.h"

#include "pycore_pylifecycle.h" // _PyOS_URandomNonblock()

#include <stdlib.h> // rand()

#ifdef Py_STATS

PyStats _py_stats_struct = { 0 };

PyStats *_py_stats = NULL;

#define ADD_STAT_TO_DICT(res, field) \

do { \

PyObject *val = PyLong_FromUnsignedLongLong(stats->field); \

if (val == NULL) { \

Py_DECREF(res); \

return NULL; \

} \

if (PyDict_SetItemString(res, #field, val) == -1) { \

Py_DECREF(res); \

Py_DECREF(val); \

return NULL; \

} \

Py_DECREF(val); \

} while(0);

static PyObject*

stats_to_dict(SpecializationStats *stats)

{

PyObject *res = PyDict_New();

if (res == NULL) {

return NULL;

}

ADD_STAT_TO_DICT(res, success);

ADD_STAT_TO_DICT(res, failure);

ADD_STAT_TO_DICT(res, hit);

ADD_STAT_TO_DICT(res, deferred);

ADD_STAT_TO_DICT(res, miss);

ADD_STAT_TO_DICT(res, deopt);

PyObject *failure_kinds = PyTuple_New(SPECIALIZATION_FAILURE_KINDS);

if (failure_kinds == NULL) {

Py_DECREF(res);

return NULL;

}

for (int i = 0; i < SPECIALIZATION_FAILURE_KINDS; i++) {

PyObject *stat = PyLong_FromUnsignedLongLong(stats->failure_kinds[i]);

if (stat == NULL) {

Py_DECREF(res);

Py_DECREF(failure_kinds);

return NULL;

}

PyTuple_SET_ITEM(failure_kinds, i, stat);

}

if (PyDict_SetItemString(res, "failure_kinds", failure_kinds)) {

Py_DECREF(res);

Py_DECREF(failure_kinds);

return NULL;

}

Py_DECREF(failure_kinds);

return res;

}

#undef ADD_STAT_TO_DICT

static int

add_stat_dict(

PyObject *res,

int opcode,

const char *name) {

SpecializationStats *stats = &_py_stats_struct.opcode_stats[opcode].specialization;

PyObject *d = stats_to_dict(stats);

if (d == NULL) {

return -1;

}

int err = PyDict_SetItemString(res, name, d);

Py_DECREF(d);

return err;

}

#ifdef Py_STATS

_Py_GetSpecializationStats(void) {

PyObject *stats = PyDict_New();

if (stats == NULL) {

return NULL;

}

int err = 0;

err += add_stat_dict(stats, LOAD_SUPER_ATTR, "load_super_attr");

err += add_stat_dict(stats, LOAD_ATTR, "load_attr");

err += add_stat_dict(stats, LOAD_GLOBAL, "load_global");

err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr");

err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr");

err += add_stat_dict(stats, STORE_ATTR, "store_attr");

err += add_stat_dict(stats, CALL, "call");

err += add_stat_dict(stats, BINARY_OP, "binary_op");

err += add_stat_dict(stats, COMPARE_OP, "compare_op");

err += add_stat_dict(stats, UNPACK_SEQUENCE, "unpack_sequence");

err += add_stat_dict(stats, FOR_ITER, "for_iter");

err += add_stat_dict(stats, TO_BOOL, "to_bool");

err += add_stat_dict(stats, SEND, "send");

if (err < 0) {

Py_DECREF(stats);

return NULL;

}

return stats;

}

#define PRINT_STAT(i, field) \

if (stats[i].field) { \

fprintf(out, " opcode[%d]." #field " : %" PRIu64 "\n", i, stats[i].field); \

}

static void

print_spec_stats(FILE *out, OpcodeStats *stats)

{

/* Mark some opcodes as specializable for stats,

fprintf(out, "opcode[%d].specializable : 1\n", BINARY_SLICE);

fprintf(out, "opcode[%d].specializable : 1\n", STORE_SLICE);

for (int i = 0; i < 256; i++) {

if (_PyOpcode_Caches[i]) {

fprintf(out, "opcode[%d].specializable : 1\n", i);

}

PRINT_STAT(i, specialization.success);

PRINT_STAT(i, specialization.failure);

PRINT_STAT(i, specialization.hit);

PRINT_STAT(i, specialization.deferred);

PRINT_STAT(i, specialization.miss);

PRINT_STAT(i, specialization.deopt);

PRINT_STAT(i, execution_count);

for (int j = 0; j < SPECIALIZATION_FAILURE_KINDS; j++) {

uint64_t val = stats[i].specialization.failure_kinds[j];

if (val) {

fprintf(out, " opcode[%d].specialization.failure_kinds[%d] : %"

PRIu64 "\n", i, j, val);

}

}

for (int j = 0; j < 256; j++) {

if (stats[i].pair_count[j]) {

fprintf(out, "opcode[%d].pair_count[%d] : %" PRIu64 "\n",

i, j, stats[i].pair_count[j]);

}

}

}

}

#undef PRINT_STAT

static void

print_call_stats(FILE *out, CallStats *stats)

{

fprintf(out, "Calls to PyEval_EvalDefault: %" PRIu64 "\n", stats->pyeval_calls);

fprintf(out, "Calls to Python functions inlined: %" PRIu64 "\n", stats->inlined_py_calls);

fprintf(out, "Frames pushed: %" PRIu64 "\n", stats->frames_pushed);

fprintf(out, "Frame objects created: %" PRIu64 "\n", stats->frame_objects_created);

for (int i = 0; i < EVAL_CALL_KINDS; i++) {

fprintf(out, "Calls via PyEval_EvalFrame[%d] : %" PRIu64 "\n", i, stats->eval_calls[i]);

}

}

static void

print_object_stats(FILE *out, ObjectStats *stats)

{

fprintf(out, "Object allocations from freelist: %" PRIu64 "\n", stats->from_freelist);

fprintf(out, "Object frees to freelist: %" PRIu64 "\n", stats->to_freelist);

fprintf(out, "Object allocations: %" PRIu64 "\n", stats->allocations);

fprintf(out, "Object allocations to 512 bytes: %" PRIu64 "\n", stats->allocations512);

fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n", stats->allocations4k);

fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big);

fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees);

fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values);

fprintf(out, "Object interpreter increfs: %" PRIu64 "\n", stats->interpreter_increfs);

fprintf(out, "Object interpreter decrefs: %" PRIu64 "\n", stats->interpreter_decrefs);

fprintf(out, "Object increfs: %" PRIu64 "\n", stats->increfs);

fprintf(out, "Object decrefs: %" PRIu64 "\n", stats->decrefs);

fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request);

fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key);

fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big);

fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass);

fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits);

fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses);

fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions);

fprintf(out, "Object method cache dunder hits: %" PRIu64 "\n", stats->type_cache_dunder_hits);

fprintf(out, "Object method cache dunder misses: %" PRIu64 "\n", stats->type_cache_dunder_misses);

fprintf(out, "Optimization attempts: %" PRIu64 "\n", stats->optimization_attempts);

fprintf(out, "Optimization traces created: %" PRIu64 "\n", stats->optimization_traces_created);

fprintf(out, "Optimization traces executed: %" PRIu64 "\n", stats->optimization_traces_executed);

fprintf(out, "Optimization uops executed: %" PRIu64 "\n", stats->optimization_uops_executed);

}

static void

print_stats(FILE *out, PyStats *stats) {

print_spec_stats(out, stats->opcode_stats);

print_call_stats(out, &stats->call_stats);

print_object_stats(out, &stats->object_stats);

}

_Py_StatsClear(void)

{

_py_stats_struct = (PyStats) { 0 };

}

_Py_PrintSpecializationStats(int to_file)

{

FILE *out = stderr;

if (to_file) {

/* Write to a file instead of stderr. */

# ifdef MS_WINDOWS

const char *dirname = "c:\\temp\\py_stats\\";

const char *dirname = "/tmp/py_stats/";

/* Use random 160 bit number as file name,

unsigned char rand[20];

char hex_name[41];

_PyOS_URandomNonblock(rand, 20);

for (int i = 0; i < 20; i++) {

hex_name[2*i] = "0123456789abcdef"[rand[i]&15];

hex_name[2*i+1] = "0123456789abcdef"[(rand[i]>>4)&15];

}

hex_name[40] = '\0';

char buf[64];

assert(strlen(dirname) + 40 + strlen(".txt") < 64);

sprintf(buf, "%s%s.txt", dirname, hex_name);

FILE *fout = fopen(buf, "w");

if (fout) {

out = fout;

}

}

else {

fprintf(out, "Specialization stats:\n");

}

print_stats(out, &_py_stats_struct);

if (out != stderr) {

fclose(out);

}

}

#ifdef Py_STATS

#define SPECIALIZATION_FAIL(opcode, kind) \

do { \

if (_py_stats) { \

_py_stats->opcode_stats[opcode].specialization.failure_kinds[kind]++; \

} \

} while (0)

#ifndef SPECIALIZATION_FAIL

#define SPECIALIZATION_FAIL(opcode, kind) ((void)0)

_PyCode_Quicken(PyCodeObject *code)

{

#if ENABLE_SPECIALIZATION

int opcode = 0;

_Py_CODEUNIT *instructions = _PyCode_CODE(code);

for (int i = 0; i < Py_SIZE(code); i++) {

opcode = _Py_GetBaseOpcode(code, i);

assert(opcode < MIN_INSTRUMENTED_OPCODE);

int caches = _PyOpcode_Caches[opcode];

if (caches) {

instructions[i + 1].cache = adaptive_counter_warmup();

i += caches;

}

}

#endif /* ENABLE_SPECIALIZATION */

}

#define SIMPLE_FUNCTION 0

#define SPEC_FAIL_OTHER 0

#define SPEC_FAIL_NO_DICT 1

#define SPEC_FAIL_OVERRIDDEN 2

#define SPEC_FAIL_OUT_OF_VERSIONS 3

#define SPEC_FAIL_OUT_OF_RANGE 4

#define SPEC_FAIL_EXPECTED_ERROR 5

#define SPEC_FAIL_WRONG_NUMBER_ARGUMENTS 6

#define SPEC_FAIL_CODE_COMPLEX_PARAMETERS 7

#define SPEC_FAIL_CODE_NOT_OPTIMIZED 8

#define SPEC_FAIL_LOAD_GLOBAL_NON_DICT 17

#define SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT 18

#define SPEC_FAIL_SUPER_BAD_CLASS 9

#define SPEC_FAIL_SUPER_SHADOWED 10

#define SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR 9

#define SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR 10

#define SPEC_FAIL_ATTR_NOT_DESCRIPTOR 11

#define SPEC_FAIL_ATTR_METHOD 12

#define SPEC_FAIL_ATTR_MUTABLE_CLASS 13

#define SPEC_FAIL_ATTR_PROPERTY 14

#define SPEC_FAIL_ATTR_NON_OBJECT_SLOT 15

#define SPEC_FAIL_ATTR_READ_ONLY 16

#define SPEC_FAIL_ATTR_AUDITED_SLOT 17

#define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 18

#define SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT 19

#define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 20

#define SPEC_FAIL_ATTR_SHADOWED 21

#define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22

#define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23

#define SPEC_FAIL_ATTR_OBJECT_SLOT 24

#define SPEC_FAIL_ATTR_HAS_MANAGED_DICT 25

#define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26

#define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27

#define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28

#define SPEC_FAIL_ATTR_NOT_IN_KEYS 29

#define SPEC_FAIL_ATTR_NOT_IN_DICT 30

#define SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE 31

#define SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR 32

#define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ 33

#define SPEC_FAIL_SUBSCR_ARRAY_INT 9

#define SPEC_FAIL_SUBSCR_ARRAY_SLICE 10

#define SPEC_FAIL_SUBSCR_LIST_SLICE 11

#define SPEC_FAIL_SUBSCR_TUPLE_SLICE 12

#define SPEC_FAIL_SUBSCR_STRING_INT 13

#define SPEC_FAIL_SUBSCR_STRING_SLICE 14

#define SPEC_FAIL_SUBSCR_BUFFER_INT 15

#define SPEC_FAIL_SUBSCR_BUFFER_SLICE 16

#define SPEC_FAIL_SUBSCR_SEQUENCE_INT 17

#define SPEC_FAIL_SUBSCR_BYTEARRAY_INT 18

#define SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE 19

#define SPEC_FAIL_SUBSCR_PY_SIMPLE 20

#define SPEC_FAIL_SUBSCR_PY_OTHER 21

#define SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE 22

#define SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE 23

#define SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES 9

#define SPEC_FAIL_BINARY_OP_ADD_OTHER 10

#define SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES 11

#define SPEC_FAIL_BINARY_OP_AND_INT 12

#define SPEC_FAIL_BINARY_OP_AND_OTHER 13

#define SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE 14

#define SPEC_FAIL_BINARY_OP_LSHIFT 15

#define SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY 16

#define SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES 17

#define SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER 18

#define SPEC_FAIL_BINARY_OP_OR 19

#define SPEC_FAIL_BINARY_OP_POWER 20

#define SPEC_FAIL_BINARY_OP_REMAINDER 21

#define SPEC_FAIL_BINARY_OP_RSHIFT 22

#define SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES 23

#define SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER 24

#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES 25

#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT 26

#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER 27

#define SPEC_FAIL_BINARY_OP_XOR 28

#define SPEC_FAIL_CALL_INSTANCE_METHOD 11

#define SPEC_FAIL_CALL_CMETHOD 12

#define SPEC_FAIL_CALL_CFUNC_VARARGS 13

#define SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS 14

#define SPEC_FAIL_CALL_CFUNC_NOARGS 15

#define SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS 16

#define SPEC_FAIL_CALL_METH_DESCR_VARARGS 17

#define SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS 18

#define SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS 19

#define SPEC_FAIL_CALL_BAD_CALL_FLAGS 20

#define SPEC_FAIL_CALL_INIT_NOT_PYTHON 21

#define SPEC_FAIL_CALL_PEP_523 22

#define SPEC_FAIL_CALL_BOUND_METHOD 23

#define SPEC_FAIL_CALL_STR 24

#define SPEC_FAIL_CALL_CLASS_NO_VECTORCALL 25

#define SPEC_FAIL_CALL_CLASS_MUTABLE 26

#define SPEC_FAIL_CALL_KWNAMES 27

#define SPEC_FAIL_CALL_METHOD_WRAPPER 28

#define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29

#define SPEC_FAIL_CALL_INIT_NOT_SIMPLE 30

#define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12

#define SPEC_FAIL_COMPARE_OP_STRING 13

#define SPEC_FAIL_COMPARE_OP_BIG_INT 14

#define SPEC_FAIL_COMPARE_OP_BYTES 15

#define SPEC_FAIL_COMPARE_OP_TUPLE 16

#define SPEC_FAIL_COMPARE_OP_LIST 17

#define SPEC_FAIL_COMPARE_OP_SET 18

#define SPEC_FAIL_COMPARE_OP_BOOL 19

#define SPEC_FAIL_COMPARE_OP_BASEOBJECT 20

#define SPEC_FAIL_COMPARE_OP_FLOAT_LONG 21

#define SPEC_FAIL_COMPARE_OP_LONG_FLOAT 22

#define SPEC_FAIL_ITER_GENERATOR 10

#define SPEC_FAIL_ITER_COROUTINE 11

#define SPEC_FAIL_ITER_ASYNC_GENERATOR 12

#define SPEC_FAIL_ITER_LIST 13

#define SPEC_FAIL_ITER_TUPLE 14

#define SPEC_FAIL_ITER_SET 15

#define SPEC_FAIL_ITER_STRING 16

#define SPEC_FAIL_ITER_BYTES 17

#define SPEC_FAIL_ITER_RANGE 18

#define SPEC_FAIL_ITER_ITERTOOLS 19

#define SPEC_FAIL_ITER_DICT_KEYS 20

#define SPEC_FAIL_ITER_DICT_ITEMS 21

#define SPEC_FAIL_ITER_DICT_VALUES 22

#define SPEC_FAIL_ITER_ENUMERATE 23

#define SPEC_FAIL_ITER_MAP 24

#define SPEC_FAIL_ITER_ZIP 25

#define SPEC_FAIL_ITER_SEQ_ITER 26

#define SPEC_FAIL_ITER_REVERSED_LIST 27

#define SPEC_FAIL_ITER_CALLABLE 28

#define SPEC_FAIL_ITER_ASCII_STRING 29

#define SPEC_FAIL_ITER_ASYNC_GENERATOR_SEND 30

#define SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR 9

#define SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE 10

#define SPEC_FAIL_TO_BOOL_BYTEARRAY 9

#define SPEC_FAIL_TO_BOOL_BYTES 10

#define SPEC_FAIL_TO_BOOL_DICT 11

#define SPEC_FAIL_TO_BOOL_FLOAT 12

#define SPEC_FAIL_TO_BOOL_MAPPING 13

#define SPEC_FAIL_TO_BOOL_MEMORY_VIEW 14

#define SPEC_FAIL_TO_BOOL_NUMBER 15

#define SPEC_FAIL_TO_BOOL_SEQUENCE 16

#define SPEC_FAIL_TO_BOOL_SET 17

#define SPEC_FAIL_TO_BOOL_TUPLE 18

static int function_kind(PyCodeObject *code);

static bool function_check_args(PyObject *o, int expected_argcount, int opcode);

static uint32_t function_get_version(PyObject *o, int opcode);

static int

specialize_module_load_attr(

PyObject *owner, _Py_CODEUNIT *instr, PyObject *name

) {

_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);

PyModuleObject *m = (PyModuleObject *)owner;

assert((owner->ob_type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);

PyDictObject *dict = (PyDictObject *)m->md_dict;

if (dict == NULL) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_NO_DICT);

return -1;

}

if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT);

return -1;

}

Py_ssize_t index = _PyDict_LookupIndex(dict, &_Py_ID(__getattr__));

assert(index != DKIX_ERROR);

if (index != DKIX_EMPTY) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND);

return -1;

}

index = _PyDict_LookupIndex(dict, name);

assert (index != DKIX_ERROR);

if (index != (uint16_t)index) {

SPECIALIZATION_FAIL(LOAD_ATTR,

index == DKIX_EMPTY ?

SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND :

SPEC_FAIL_OUT_OF_RANGE);

return -1;

}

uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(

_PyInterpreterState_GET(), dict->ma_keys);

if (keys_version == 0) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);

return -1;

}

write_u32(cache->version, keys_version);

cache->index = (uint16_t)index;

instr->op.code = LOAD_ATTR_MODULE;

return 0;

}

_Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, _Py_CODEUNIT *instr, int load_method) {

assert(ENABLE_SPECIALIZATION);

assert(_PyOpcode_Caches[LOAD_SUPER_ATTR] == INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR);

_PySuperAttrCache *cache = (_PySuperAttrCache *)(instr + 1);

if (global_super != (PyObject *)&PySuper_Type) {

SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_SHADOWED);

goto fail;

}

if (!PyType_Check(cls)) {

SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_BAD_CLASS);

goto fail;

}

instr->op.code = load_method ? LOAD_SUPER_ATTR_METHOD : LOAD_SUPER_ATTR_ATTR;

goto success;

fail:

STAT_INC(LOAD_SUPER_ATTR, failure);

assert(!PyErr_Occurred());

instr->op.code = LOAD_SUPER_ATTR;

cache->counter = adaptive_counter_backoff(cache->counter);

return;

success:

STAT_INC(LOAD_SUPER_ATTR, success);

assert(!PyErr_Occurred());

cache->counter = adaptive_counter_cooldown();

}

typedef enum {

OVERRIDING, /* Is an overriding descriptor, and will remain so. */

METHOD, /* Attribute has Py_TPFLAGS_METHOD_DESCRIPTOR set */

PROPERTY, /* Is a property */

OBJECT_SLOT, /* Is an object slot descriptor */

OTHER_SLOT, /* Is a slot descriptor of another type */

NON_OVERRIDING, /* Is another non-overriding descriptor, and is an instance of an immutable class*/

BUILTIN_CLASSMETHOD, /* Builtin methods with METH_CLASS */

PYTHON_CLASSMETHOD, /* Python classmethod(func) object */

NON_DESCRIPTOR, /* Is not a descriptor, and is an instance of an immutable class */

MUTABLE, /* Instance of a mutable class; might, or might not, be a descriptor */

ABSENT, /* Attribute is not present on the class */

DUNDER_CLASS, /* __class__ attribute */

GETSET_OVERRIDDEN, /* __getattribute__ or __setattr__ has been overridden */

GETATTRIBUTE_IS_PYTHON_FUNCTION /* Descriptor requires calling a Python __getattribute__ */

} DescriptorClassification;

static DescriptorClassification

analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int store)

{

bool has_getattr = false;

if (store) {

if (type->tp_setattro != PyObject_GenericSetAttr) {

*descr = NULL;

return GETSET_OVERRIDDEN;

}

}

else {

getattrofunc getattro_slot = type->tp_getattro;

if (getattro_slot == PyObject_GenericGetAttr) {

/* Normal attribute lookup; */

has_getattr = false;

}

else if (getattro_slot == _Py_slot_tp_getattr_hook ||

getattro_slot == _Py_slot_tp_getattro) {

/* One or both of __getattribute__ or __getattr__ may have been

PyObject *getattribute = _PyType_Lookup(type,

&_Py_ID(__getattribute__));

PyInterpreterState *interp = _PyInterpreterState_GET();

bool has_custom_getattribute = getattribute != NULL &&

getattribute != interp->callable_cache.object__getattribute__;

has_getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)) != NULL;

if (has_custom_getattribute) {

if (getattro_slot == _Py_slot_tp_getattro &&

!has_getattr &&

Py_IS_TYPE(getattribute, &PyFunction_Type)) {

*descr = getattribute;

return GETATTRIBUTE_IS_PYTHON_FUNCTION;

}

/* Potentially both __getattr__ and __getattribute__ are set.

*descr = NULL;

return GETSET_OVERRIDDEN;

}

/* Potentially has __getattr__ but no custom __getattribute__.

}

else {

*descr = NULL;

return GETSET_OVERRIDDEN;

}

}

PyObject *descriptor = _PyType_Lookup(type, name);

*descr = descriptor;

if (descriptor == NULL) {

return ABSENT;

}

PyTypeObject *desc_cls = Py_TYPE(descriptor);

if (!(desc_cls->tp_flags & Py_TPFLAGS_IMMUTABLETYPE)) {

return MUTABLE;

}

if (desc_cls->tp_descr_set) {

if (desc_cls == &PyMemberDescr_Type) {

PyMemberDescrObject *member = (PyMemberDescrObject *)descriptor;

struct PyMemberDef *dmem = member->d_member;

if (dmem->type == T_OBJECT_EX) {

return OBJECT_SLOT;

}

return OTHER_SLOT;

}

if (desc_cls == &PyProperty_Type) {

/* We can't detect at runtime whether an attribute exists

return has_getattr ? GETSET_OVERRIDDEN : PROPERTY;

}

if (PyUnicode_CompareWithASCIIString(name, "__class__") == 0) {

if (descriptor == _PyType_Lookup(&PyBaseObject_Type, name)) {

return DUNDER_CLASS;

}

}

if (store) {

return OVERRIDING;

}

}

if (desc_cls->tp_descr_get) {

if (desc_cls->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR) {

return METHOD;

}

if (Py_IS_TYPE(descriptor, &PyClassMethodDescr_Type)) {

return BUILTIN_CLASSMETHOD;

}

if (Py_IS_TYPE(descriptor, &PyClassMethod_Type)) {

return PYTHON_CLASSMETHOD;

}

return NON_OVERRIDING;

}

return NON_DESCRIPTOR;

}

static int

specialize_dict_access(

PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,

DescriptorClassification kind, PyObject *name,

int base_op, int values_op, int hint_op)

{

assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT ||

kind == BUILTIN_CLASSMETHOD || kind == PYTHON_CLASSMETHOD);

// No descriptor, or non overriding.

if ((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {

SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_MANAGED_DICT);

return 0;

}

_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);

PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);

if (_PyDictOrValues_IsValues(dorv)) {

// Virtual dictionary

PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;

assert(PyUnicode_CheckExact(name));

Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);

assert (index != DKIX_ERROR);

if (index != (uint16_t)index) {

SPECIALIZATION_FAIL(base_op,

index == DKIX_EMPTY ?

SPEC_FAIL_ATTR_NOT_IN_KEYS :

SPEC_FAIL_OUT_OF_RANGE);

return 0;

}

write_u32(cache->version, type->tp_version_tag);

cache->index = (uint16_t)index;

instr->op.code = values_op;

}

else {

PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);

if (dict == NULL || !PyDict_CheckExact(dict)) {

SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT);

return 0;

}

// We found an instance with a __dict__.

Py_ssize_t index =

_PyDict_LookupIndex(dict, name);

if (index != (uint16_t)index) {

SPECIALIZATION_FAIL(base_op,

index == DKIX_EMPTY ?

SPEC_FAIL_ATTR_NOT_IN_DICT :

SPEC_FAIL_OUT_OF_RANGE);

return 0;

}

cache->index = (uint16_t)index;

write_u32(cache->version, type->tp_version_tag);

instr->op.code = hint_op;

}

return 1;

}

static int specialize_attr_loadclassattr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name,

PyObject* descr, DescriptorClassification kind, bool is_method);

static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name);

_Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)

{

assert(ENABLE_SPECIALIZATION);

assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR);

_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);

PyTypeObject *type = Py_TYPE(owner);

if (!_PyType_IsReady(type)) {

// We *might* not really need this check, but we inherited it from

// PyObject_GenericGetAttr and friends... and this way we still do the

// right thing if someone forgets to call PyType_Ready(type):

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);

goto fail;

}

if (PyModule_CheckExact(owner)) {

if (specialize_module_load_attr(owner, instr, name))

{

goto fail;

}

goto success;

}

if (PyType_Check(owner)) {

if (specialize_class_load_attr(owner, instr, name)) {

goto fail;

}

goto success;

}

PyObject *descr = NULL;

DescriptorClassification kind = analyze_descriptor(type, name, &descr, 0);

assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN);

switch(kind) {

case OVERRIDING:

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);

goto fail;

case METHOD:

{

int oparg = instr->op.arg;

if (oparg & 1) {

if (specialize_attr_loadclassattr(owner, instr, name, descr, kind, true)) {

goto success;

}

}

else {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);

}

goto fail;

}

case PROPERTY:

{

_PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1);

assert(Py_TYPE(descr) == &PyProperty_Type);

PyObject *fget = ((_PyPropertyObject *)descr)->prop_get;

if (fget == NULL) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);

goto fail;

}

if (!Py_IS_TYPE(fget, &PyFunction_Type)) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION);

goto fail;

}

if (!function_check_args(fget, 1, LOAD_ATTR)) {

goto fail;

}

uint32_t version = function_get_version(fget, LOAD_ATTR);

if (version == 0) {

goto fail;

}

if (_PyInterpreterState_GET()->eval_frame) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);

goto fail;

}

write_u32(lm_cache->keys_version, version);

assert(type->tp_version_tag != 0);

write_u32(lm_cache->type_version, type->tp_version_tag);

/* borrowed */

write_obj(lm_cache->descr, fget);

instr->op.code = LOAD_ATTR_PROPERTY;

goto success;

}

case OBJECT_SLOT:

{

PyMemberDescrObject *member = (PyMemberDescrObject *)descr;

struct PyMemberDef *dmem = member->d_member;

Py_ssize_t offset = dmem->offset;

if (!PyObject_TypeCheck(owner, member->d_common.d_type)) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);

goto fail;

}

if (dmem->flags & PY_AUDIT_READ) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_AUDITED_SLOT);

goto fail;

}

if (offset != (uint16_t)offset) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);

goto fail;

}

assert(dmem->type == T_OBJECT_EX);

assert(offset > 0);

cache->index = (uint16_t)offset;

write_u32(cache->version, type->tp_version_tag);

instr->op.code = LOAD_ATTR_SLOT;

goto success;

}

case DUNDER_CLASS:

{

Py_ssize_t offset = offsetof(PyObject, ob_type);

assert(offset == (uint16_t)offset);

cache->index = (uint16_t)offset;

write_u32(cache->version, type->tp_version_tag);

instr->op.code = LOAD_ATTR_SLOT;

goto success;

}

case OTHER_SLOT:

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT);

goto fail;

case MUTABLE:

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS);

goto fail;

case GETSET_OVERRIDDEN:

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OVERRIDDEN);

goto fail;

case GETATTRIBUTE_IS_PYTHON_FUNCTION:

{

assert(type->tp_getattro == _Py_slot_tp_getattro);

assert(Py_IS_TYPE(descr, &PyFunction_Type));

_PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1);

if (!function_check_args(descr, 2, LOAD_ATTR)) {

goto fail;

}

uint32_t version = function_get_version(descr, LOAD_ATTR);

if (version == 0) {

goto fail;

}

if (_PyInterpreterState_GET()->eval_frame) {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);

goto fail;

}

write_u32(lm_cache->keys_version, version);

/* borrowed */

write_obj(lm_cache->descr, descr);

write_u32(lm_cache->type_version, type->tp_version_tag);

instr->op.code = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN;

goto success;

}

case BUILTIN_CLASSMETHOD:

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ);

goto fail;

case PYTHON_CLASSMETHOD:

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_METHOD_OBJ);

goto fail;

case NON_OVERRIDING:

SPECIALIZATION_FAIL(LOAD_ATTR,

(type->tp_flags & Py_TPFLAGS_MANAGED_DICT) ?

SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR :

SPEC_FAIL_ATTR_NOT_MANAGED_DICT);

goto fail;

case NON_DESCRIPTOR:

if ((instr->op.arg & 1) == 0) {

if (specialize_attr_loadclassattr(owner, instr, name, descr, kind, false)) {

goto success;

}

}

else {

SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE);

}

goto fail;

case ABSENT:

if (specialize_dict_access(owner, instr, type, kind, name, LOAD_ATTR,

LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT))

{

goto success;

}

}

fail:

STAT_INC(LOAD_ATTR, failure);

assert(!PyErr_Occurred());

instr->op.code = LOAD_ATTR;

cache->counter = adaptive_counter_backoff(cache->counter);

return;

success:

STAT_INC(LOAD_ATTR, success);

assert(!PyErr_Occurred());

cache->counter = adaptive_counter_cooldown();

}

_Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)

{

assert(ENABLE_SPECIALIZATION);

assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR);

_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);

PyTypeObject *type = Py_TYPE(owner);

if (!_PyType_IsReady(type)) {

// We *might* not really need this check, but we inherited it from

// PyObject_GenericSetAttr and friends... and this way we still do the

// right thing if someone forgets to call PyType_Ready(type):

SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OTHER);

goto fail;

}

if (PyModule_CheckExact(owner)) {

SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN);

goto fail;

}

PyObject *descr;

DescriptorClassification kind = analyze_descriptor(type, name, &descr, 1);

switch(kind) {

case OVERRIDING:

SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);

goto fail;

case METHOD:

SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_METHOD);

goto fail;

case PROPERTY:

SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_PROPERTY);

goto fail;

case OBJECT_SLOT:

{

PyMemberDescrObject *member = (PyMemberDescrObject *)descr;

struct PyMemberDef *dmem = member->d_member;

Py_ssize_t offset = dmem->offset;

if (!PyObject_TypeCheck(owner, member->d_common.d_type)) {

SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_EXPECTED_ERROR);

goto fail;

}

if (dmem->flags & READONLY) {

SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_READ_ONLY);

goto fail;

}

if (offset != (uint16_t)offset) {

SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OUT_OF_RANGE);

goto fail;

}

assert(dmem->type == T_OBJECT_EX);

assert(offset > 0);

cache->index = (uint16_t)offset;

write_u32(cache->version, type->tp_version_tag);

instr->op.code = STORE_ATTR_SLOT;

goto success;

}

case DUNDER_CLASS:

case OTHER_SLOT:

SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT);

goto fail;

case MUTABLE:

SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS);

goto fail;

case GETATTRIBUTE_IS_PYTHON_FUNCTION:

case GETSET_OVERRIDDEN:

SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN);

goto fail;

case BUILTIN_CLASSMETHOD:

SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ);

goto fail;

case PYTHON_CLASSMETHOD:

SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_METHOD_OBJ);

goto fail;

case NON_OVERRIDING:

SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR);

goto fail;

case NON_DESCRIPTOR:

SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE);

goto fail;

case ABSENT:

if (specialize_dict_access(owner, instr, type, kind, name, STORE_ATTR,

STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT))

{

goto success;

}

}

fail:

STAT_INC(STORE_ATTR, failure);

assert(!PyErr_Occurred());

instr->op.code = STORE_ATTR;

cache->counter = adaptive_counter_backoff(cache->counter);

return;

success:

STAT_INC(STORE_ATTR, success);

assert(!PyErr_Occurred());

cache->counter = adaptive_counter_cooldown();

}

#ifdef Py_STATS