#include "Python.h"

#include "pycore_call.h"

#include "pycore_frame.h"

#include "pycore_interp.h"

#include "pycore_long.h"

#include "pycore_namespace.h"

#include "pycore_object.h"

#include "pycore_opcode.h"

#include "pycore_pyerrors.h"

#include "pycore_pystate.h"

PyObject _PyInstrumentation_DISABLE =

{

.ob_refcnt = _Py_IMMORTAL_REFCNT,

.ob_type = &PyBaseObject_Type

};

PyObject _PyInstrumentation_MISSING =

{

.ob_refcnt = _Py_IMMORTAL_REFCNT,

.ob_type = &PyBaseObject_Type

};

static const int8_t EVENT_FOR_OPCODE[256] = {

[RETURN_CONST] = PY_MONITORING_EVENT_PY_RETURN,

[INSTRUMENTED_RETURN_CONST] = PY_MONITORING_EVENT_PY_RETURN,

[RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN,

[INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN,

[CALL] = PY_MONITORING_EVENT_CALL,

[INSTRUMENTED_CALL] = PY_MONITORING_EVENT_CALL,

[CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL,

[INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL,

[LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL,

[INSTRUMENTED_LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL,

[RESUME] = -1,

[YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD,

[INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD,

[JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP,

[JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP,

[POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH,

[POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH,

[POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH,

[POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH,

[INSTRUMENTED_JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP,

[INSTRUMENTED_JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP,

[INSTRUMENTED_POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH,

[INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH,

[INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH,

[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH,

[FOR_ITER] = PY_MONITORING_EVENT_BRANCH,

[INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH,

[END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION,

[INSTRUMENTED_END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION,

[END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION,

[INSTRUMENTED_END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION,

};

static const uint8_t DE_INSTRUMENT[256] = {

[INSTRUMENTED_RESUME] = RESUME,

[INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE,

[INSTRUMENTED_RETURN_CONST] = RETURN_CONST,

[INSTRUMENTED_CALL] = CALL,

[INSTRUMENTED_CALL_FUNCTION_EX] = CALL_FUNCTION_EX,

[INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE,

[INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD,

[INSTRUMENTED_JUMP_BACKWARD] = JUMP_BACKWARD,

[INSTRUMENTED_POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE,

[INSTRUMENTED_POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE,

[INSTRUMENTED_POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE,

[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE,

[INSTRUMENTED_FOR_ITER] = FOR_ITER,

[INSTRUMENTED_END_FOR] = END_FOR,

[INSTRUMENTED_END_SEND] = END_SEND,

[INSTRUMENTED_LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR,

};

static const uint8_t INSTRUMENTED_OPCODES[256] = {

[RETURN_CONST] = INSTRUMENTED_RETURN_CONST,

[INSTRUMENTED_RETURN_CONST] = INSTRUMENTED_RETURN_CONST,

[RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE,

[INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE,

[CALL] = INSTRUMENTED_CALL,

[INSTRUMENTED_CALL] = INSTRUMENTED_CALL,

[CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX,

[INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX,

[YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE,

[INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE,

[RESUME] = INSTRUMENTED_RESUME,

[INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME,

[JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD,

[INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD,

[JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD,

[INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD,

[POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE,

[INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE,

[POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE,

[INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE,

[POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE,

[INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE,

[POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE,

[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE,

[END_FOR] = INSTRUMENTED_END_FOR,

[INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR,

[END_SEND] = INSTRUMENTED_END_SEND,

[INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND,

[FOR_ITER] = INSTRUMENTED_FOR_ITER,

[INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER,

[LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR,

[INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR,

[INSTRUMENTED_LINE] = INSTRUMENTED_LINE,

[INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION,

};

static inline bool

opcode_has_event(int opcode)

{

return (

opcode < INSTRUMENTED_LINE &&

INSTRUMENTED_OPCODES[opcode] > 0

);

}

static inline bool

is_instrumented(int opcode)

{

assert(opcode != 0);

assert(opcode != RESERVED);

return opcode >= MIN_INSTRUMENTED_OPCODE;

}

#ifndef NDEBUG

static inline bool

monitors_equals(_Py_LocalMonitors a, _Py_LocalMonitors b)

{

for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {

if (a.tools[i] != b.tools[i]) {

return false;

}

}

return true;

}

static inline _Py_LocalMonitors

monitors_sub(_Py_LocalMonitors a, _Py_LocalMonitors b)

{

_Py_LocalMonitors res;

for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {

res.tools[i] = a.tools[i] & ~b.tools[i];

}

return res;

}

#ifndef NDEBUG

static inline _Py_LocalMonitors

monitors_and(_Py_LocalMonitors a, _Py_LocalMonitors b)

{

_Py_LocalMonitors res;

for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {

res.tools[i] = a.tools[i] & b.tools[i];

}

return res;

}

static inline _Py_LocalMonitors

local_union(_Py_GlobalMonitors a, _Py_LocalMonitors b)

{

_Py_LocalMonitors res;

for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {

res.tools[i] = a.tools[i] | b.tools[i];

}

return res;

}

static inline bool

monitors_are_empty(_Py_LocalMonitors m)

{

for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {

if (m.tools[i]) {

return false;

}

}

return true;

}

static inline bool

multiple_tools(_Py_LocalMonitors *m)

{

for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {

if (_Py_popcount32(m->tools[i]) > 1) {

return true;

}

}

return false;

}

static inline _PyMonitoringEventSet

get_local_events(_Py_LocalMonitors *m, int tool_id)

{

_PyMonitoringEventSet result = 0;

for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) {

if ((m->tools[e] >> tool_id) & 1) {

result |= (1 << e);

}

}

return result;

}

static inline _PyMonitoringEventSet

get_events(_Py_GlobalMonitors *m, int tool_id)

{

_PyMonitoringEventSet result = 0;

for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) {

if ((m->tools[e] >> tool_id) & 1) {

result |= (1 << e);

}

}

return result;

}

#define NO_LINE -128

#define COMPUTED_LINE -127

#define OFFSET_SHIFT 4

static int8_t

compute_line_delta(PyCodeObject *code, int offset, int line)

{

if (line < 0) {

return NO_LINE;

}

int delta = line - code->co_firstlineno - (offset >> OFFSET_SHIFT);

if (delta <= INT8_MAX && delta > COMPUTED_LINE) {

return delta;

}

return COMPUTED_LINE;

}

static int

compute_line(PyCodeObject *code, int offset, int8_t line_delta)

{

if (line_delta > COMPUTED_LINE) {

return code->co_firstlineno + (offset >> OFFSET_SHIFT) + line_delta;

}

if (line_delta == NO_LINE) {

return -1;

}

assert(line_delta == COMPUTED_LINE);

/* Look it up */

return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT));

}

static int

instruction_length(PyCodeObject *code, int offset)

{

int opcode = _PyCode_CODE(code)[offset].op.code;

assert(opcode != 0);

assert(opcode != RESERVED);

if (opcode == INSTRUMENTED_LINE) {

opcode = code->_co_monitoring->lines[offset].original_opcode;

}

if (opcode == INSTRUMENTED_INSTRUCTION) {

opcode = code->_co_monitoring->per_instruction_opcodes[offset];

}

int deinstrumented = DE_INSTRUMENT[opcode];

if (deinstrumented) {

opcode = deinstrumented;

}

else {

opcode = _PyOpcode_Deopt[opcode];

}

assert(opcode != 0);

assert(!is_instrumented(opcode));

assert(opcode == _PyOpcode_Deopt[opcode]);

return 1 + _PyOpcode_Caches[opcode];

}

#ifdef INSTRUMENT_DEBUG

static void

dump_instrumentation_data_tools(PyCodeObject *code, uint8_t *tools, int i, FILE*out)

{

if (tools == NULL) {

fprintf(out, "tools = NULL");

}

else {

fprintf(out, "tools = %d", tools[i]);

}

}

static void

dump_instrumentation_data_lines(PyCodeObject *code, _PyCoLineInstrumentationData *lines, int i, FILE*out)

{

if (lines == NULL) {

fprintf(out, ", lines = NULL");

}

else if (lines[i].original_opcode == 0) {

fprintf(out, ", lines = {original_opcode = No LINE (0), line_delta = %d)", lines[i].line_delta);

}

else {

fprintf(out, ", lines = {original_opcode = %s, line_delta = %d)", _PyOpcode_OpName[lines[i].original_opcode], lines[i].line_delta);

}

}

static void

dump_instrumentation_data_line_tools(PyCodeObject *code, uint8_t *line_tools, int i, FILE*out)

{

if (line_tools == NULL) {

fprintf(out, ", line_tools = NULL");

}

else {

fprintf(out, ", line_tools = %d", line_tools[i]);

}

}

static void

dump_instrumentation_data_per_instruction(PyCodeObject *code, _PyCoMonitoringData *data, int i, FILE*out)

{

if (data->per_instruction_opcodes == NULL) {

fprintf(out, ", per-inst opcode = NULL");

}

else {

fprintf(out, ", per-inst opcode = %s", _PyOpcode_OpName[data->per_instruction_opcodes[i]]);

}

if (data->per_instruction_tools == NULL) {

fprintf(out, ", per-inst tools = NULL");

}

else {

fprintf(out, ", per-inst tools = %d", data->per_instruction_tools[i]);

}

}

static void

dump_global_monitors(const char *prefix, _Py_GlobalMonitors monitors, FILE*out)

{

fprintf(out, "%s monitors:\n", prefix);

for (int event = 0; event < _PY_MONITORING_UNGROUPED_EVENTS; event++) {

fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]);

}

}

static void

dump_local_monitors(const char *prefix, _Py_LocalMonitors monitors, FILE*out)

{

fprintf(out, "%s monitors:\n", prefix);

for (int event = 0; event < _PY_MONITORING_LOCAL_EVENTS; event++) {

fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]);

}

}

static void

dump_instrumentation_data(PyCodeObject *code, int star, FILE*out)

{

_PyCoMonitoringData *data = code->_co_monitoring;

fprintf(out, "\n");

PyObject_Print(code->co_name, out, Py_PRINT_RAW);

fprintf(out, "\n");

if (data == NULL) {

fprintf(out, "NULL\n");

return;

}

dump_global_monitors("Global", _PyInterpreterState_GET()->monitors, out);

dump_local_monitors("Code", data->local_monitors, out);

dump_local_monitors("Active", data->active_monitors, out);

int code_len = (int)Py_SIZE(code);

bool starred = false;

for (int i = 0; i < code_len; i += instruction_length(code, i)) {

_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];

int opcode = instr->op.code;

if (i == star) {

fprintf(out, "** ");

starred = true;

}

fprintf(out, "Offset: %d, line: %d %s: ", i, PyCode_Addr2Line(code, i*2), _PyOpcode_OpName[opcode]);

dump_instrumentation_data_tools(code, data->tools, i, out);

dump_instrumentation_data_lines(code, data->lines, i, out);

dump_instrumentation_data_line_tools(code, data->line_tools, i, out);

dump_instrumentation_data_per_instruction(code, data, i, out);

fprintf(out, "\n");

;

}

if (!starred && star >= 0) {

fprintf(out, "Error offset not at valid instruction offset: %d\n", star);

fprintf(out, " ");

dump_instrumentation_data_tools(code, data->tools, star, out);

dump_instrumentation_data_lines(code, data->lines, star, out);

dump_instrumentation_data_line_tools(code, data->line_tools, star, out);

dump_instrumentation_data_per_instruction(code, data, star, out);

fprintf(out, "\n");

}

}

#define CHECK(test) do { \

if (!(test)) { \

dump_instrumentation_data(code, i, stderr); \

} \

assert(test); \

} while (0)

static bool

valid_opcode(int opcode)

{

if (opcode > 0 &&

opcode != RESERVED &&

opcode < 255 &&

_PyOpcode_OpName[opcode] &&

_PyOpcode_OpName[opcode][0] != '<')

{

return true;

}

return false;

}

static void

sanity_check_instrumentation(PyCodeObject *code)

{

_PyCoMonitoringData *data = code->_co_monitoring;

if (data == NULL) {

return;

}

_Py_GlobalMonitors global_monitors = _PyInterpreterState_GET()->monitors;

_Py_LocalMonitors active_monitors;

if (code->_co_monitoring) {

_Py_LocalMonitors local_monitors = code->_co_monitoring->local_monitors;

active_monitors = local_union(global_monitors, local_monitors);

}

else {

_Py_LocalMonitors empty = (_Py_LocalMonitors) { 0 };

active_monitors = local_union(global_monitors, empty);

}

assert(monitors_equals(

code->_co_monitoring->active_monitors,

active_monitors));

int code_len = (int)Py_SIZE(code);

for (int i = 0; i < code_len;) {

_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];

int opcode = instr->op.code;

int base_opcode = _Py_GetBaseOpcode(code, i);

CHECK(valid_opcode(opcode));

CHECK(valid_opcode(base_opcode));

if (opcode == INSTRUMENTED_INSTRUCTION) {

opcode = data->per_instruction_opcodes[i];

if (!is_instrumented(opcode)) {

CHECK(_PyOpcode_Deopt[opcode] == opcode);

}

if (data->per_instruction_tools) {

uint8_t tools = active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION];

CHECK((tools & data->per_instruction_tools[i]) == data->per_instruction_tools[i]);

}

}

if (opcode == INSTRUMENTED_LINE) {

CHECK(data->lines);

CHECK(valid_opcode(data->lines[i].original_opcode));

opcode = data->lines[i].original_opcode;

CHECK(opcode != END_FOR);

CHECK(opcode != RESUME);

CHECK(opcode != RESUME_CHECK);

CHECK(opcode != INSTRUMENTED_RESUME);

if (!is_instrumented(opcode)) {

CHECK(_PyOpcode_Deopt[opcode] == opcode);

}

CHECK(opcode != INSTRUMENTED_LINE);

}

else if (data->lines) {

/* If original_opcode is INSTRUMENTED_INSTRUCTION

CHECK(data->lines[i].original_opcode != INSTRUMENTED_INSTRUCTION);

}

if (opcode == INSTRUMENTED_INSTRUCTION) {

CHECK(data->per_instruction_opcodes[i] != 0);

opcode = data->per_instruction_opcodes[i];

}

if (is_instrumented(opcode)) {

CHECK(DE_INSTRUMENT[opcode] == base_opcode);

int event = EVENT_FOR_OPCODE[DE_INSTRUMENT[opcode]];

if (event < 0) {

/* RESUME fixup */

event = instr->op.arg ? 1: 0;

}

CHECK(active_monitors.tools[event] != 0);

}

if (data->lines && base_opcode != END_FOR) {

int line1 = compute_line(code, i, data->lines[i].line_delta);

int line2 = PyCode_Addr2Line(code, i*sizeof(_Py_CODEUNIT));

CHECK(line1 == line2);

}

CHECK(valid_opcode(opcode));

if (data->tools) {

uint8_t local_tools = data->tools[i];

if (opcode_has_event(base_opcode)) {

int event = EVENT_FOR_OPCODE[base_opcode];

if (event == -1) {

/* RESUME fixup */

event = _PyCode_CODE(code)[i].op.arg;

}

CHECK((active_monitors.tools[event] & local_tools) == local_tools);

}

else {

CHECK(local_tools == 0xff);

}

}

i += instruction_length(code, i);

assert(i <= code_len);

}

}

#define CHECK(test) assert(test)

int _Py_GetBaseOpcode(PyCodeObject *code, int i)

{

int opcode = _PyCode_CODE(code)[i].op.code;

if (opcode == INSTRUMENTED_LINE) {

opcode = code->_co_monitoring->lines[i].original_opcode;

}

if (opcode == INSTRUMENTED_INSTRUCTION) {

opcode = code->_co_monitoring->per_instruction_opcodes[i];

}

CHECK(opcode != INSTRUMENTED_INSTRUCTION);

CHECK(opcode != INSTRUMENTED_LINE);

int deinstrumented = DE_INSTRUMENT[opcode];

if (deinstrumented) {

return deinstrumented;

}

return _PyOpcode_Deopt[opcode];

}

static void

de_instrument(PyCodeObject *code, int i, int event)

{

assert(event != PY_MONITORING_EVENT_INSTRUCTION);

assert(event != PY_MONITORING_EVENT_LINE);

_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];

uint8_t *opcode_ptr = &instr->op.code;

int opcode = *opcode_ptr;

if (opcode == INSTRUMENTED_LINE) {

opcode_ptr = &code->_co_monitoring->lines[i].original_opcode;

opcode = *opcode_ptr;

}

if (opcode == INSTRUMENTED_INSTRUCTION) {

opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i];

opcode = *opcode_ptr;

}

int deinstrumented = DE_INSTRUMENT[opcode];

if (deinstrumented == 0) {

return;

}

CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented);

*opcode_ptr = deinstrumented;

if (_PyOpcode_Caches[deinstrumented]) {

instr[1].cache = adaptive_counter_warmup();

}

}

static void

de_instrument_line(PyCodeObject *code, int i)

{

_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];

int opcode = instr->op.code;

if (opcode != INSTRUMENTED_LINE) {

return;

}

_PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i];

int original_opcode = lines->original_opcode;

if (original_opcode == INSTRUMENTED_INSTRUCTION) {

lines->original_opcode = code->_co_monitoring->per_instruction_opcodes[i];

}

CHECK(original_opcode != 0);

CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]);

instr->op.code = original_opcode;

if (_PyOpcode_Caches[original_opcode]) {

instr[1].cache = adaptive_counter_warmup();

}

assert(instr->op.code != INSTRUMENTED_LINE);

}

static void

de_instrument_per_instruction(PyCodeObject *code, int i)

{

_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];

uint8_t *opcode_ptr = &instr->op.code;

int opcode = *opcode_ptr;

if (opcode == INSTRUMENTED_LINE) {

opcode_ptr = &code->_co_monitoring->lines[i].original_opcode;

opcode = *opcode_ptr;

}

if (opcode != INSTRUMENTED_INSTRUCTION) {

return;

}

int original_opcode = code->_co_monitoring->per_instruction_opcodes[i];

CHECK(original_opcode != 0);

CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]);

*opcode_ptr = original_opcode;

if (_PyOpcode_Caches[original_opcode]) {

instr[1].cache = adaptive_counter_warmup();

}

assert(*opcode_ptr != INSTRUMENTED_INSTRUCTION);

assert(instr->op.code != INSTRUMENTED_INSTRUCTION);

/* Keep things clean for sanity check */

code->_co_monitoring->per_instruction_opcodes[i] = 0;

}

static void

instrument(PyCodeObject *code, int i)

{

_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];

uint8_t *opcode_ptr = &instr->op.code;

int opcode =*opcode_ptr;

if (opcode == INSTRUMENTED_LINE) {

_PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i];

opcode_ptr = &lines->original_opcode;

opcode = *opcode_ptr;

}

if (opcode == INSTRUMENTED_INSTRUCTION) {

opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i];

opcode = *opcode_ptr;

CHECK(opcode != INSTRUMENTED_INSTRUCTION && opcode != INSTRUMENTED_LINE);

CHECK(opcode == _PyOpcode_Deopt[opcode]);

}

CHECK(opcode != 0);

if (!is_instrumented(opcode)) {

int deopt = _PyOpcode_Deopt[opcode];

int instrumented = INSTRUMENTED_OPCODES[deopt];

assert(instrumented);

*opcode_ptr = instrumented;

if (_PyOpcode_Caches[deopt]) {

instr[1].cache = adaptive_counter_warmup();

}

}

}

static void

instrument_line(PyCodeObject *code, int i)

{

uint8_t *opcode_ptr = &_PyCode_CODE(code)[i].op.code;

int opcode = *opcode_ptr;

if (opcode == INSTRUMENTED_LINE) {

return;

}

_PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i];

lines->original_opcode = _PyOpcode_Deopt[opcode];

CHECK(lines->original_opcode > 0);

*opcode_ptr = INSTRUMENTED_LINE;

}

static void

instrument_per_instruction(PyCodeObject *code, int i)

{

_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];

uint8_t *opcode_ptr = &instr->op.code;

int opcode = *opcode_ptr;

if (opcode == INSTRUMENTED_LINE) {

_PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i];

opcode_ptr = &lines->original_opcode;

opcode = *opcode_ptr;

}

if (opcode == INSTRUMENTED_INSTRUCTION) {

assert(code->_co_monitoring->per_instruction_opcodes[i] > 0);

return;

}

CHECK(opcode != 0);

if (is_instrumented(opcode)) {

code->_co_monitoring->per_instruction_opcodes[i] = opcode;

}

else {

assert(opcode != 0);

assert(_PyOpcode_Deopt[opcode] != 0);

assert(_PyOpcode_Deopt[opcode] != RESUME);

code->_co_monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode];

}

assert(code->_co_monitoring->per_instruction_opcodes[i] > 0);

*opcode_ptr = INSTRUMENTED_INSTRUCTION;

}

static void

remove_tools(PyCodeObject * code, int offset, int event, int tools)

{

assert(event != PY_MONITORING_EVENT_LINE);

assert(event != PY_MONITORING_EVENT_INSTRUCTION);

assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event));

assert(opcode_has_event(_Py_GetBaseOpcode(code, offset)));

_PyCoMonitoringData *monitoring = code->_co_monitoring;

if (monitoring && monitoring->tools) {

monitoring->tools[offset] &= ~tools;

if (monitoring->tools[offset] == 0) {

de_instrument(code, offset, event);

}

}

else {

/* Single tool */

uint8_t single_tool = code->_co_monitoring->active_monitors.tools[event];

assert(_Py_popcount32(single_tool) <= 1);

if (((single_tool & tools) == single_tool)) {

de_instrument(code, offset, event);

}

}

}

#ifndef NDEBUG

static bool

tools_is_subset_for_event(PyCodeObject * code, int event, int tools)

{

int global_tools = PyInterpreterState_Get()->monitors.tools[event];

int local_tools = code->_co_monitoring->local_monitors.tools[event];

return tools == ((global_tools | local_tools) & tools);

}

static void

remove_line_tools(PyCodeObject * code, int offset, int tools)

{

assert(code->_co_monitoring);

if (code->_co_monitoring->line_tools)

{

uint8_t *toolsptr = &code->_co_monitoring->line_tools[offset];

*toolsptr &= ~tools;

if (*toolsptr == 0 ) {

de_instrument_line(code, offset);

}

}

else {

/* Single tool */

uint8_t single_tool = code->_co_monitoring->active_monitors.tools[PY_MONITORING_EVENT_LINE];

assert(_Py_popcount32(single_tool) <= 1);

if (((single_tool & tools) == single_tool)) {

de_instrument_line(code, offset);

}

}

}

static void

add_tools(PyCodeObject * code, int offset, int event, int tools)

{

assert(event != PY_MONITORING_EVENT_LINE);

assert(event != PY_MONITORING_EVENT_INSTRUCTION);

assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event));

assert(code->_co_monitoring);

if (code->_co_monitoring &&

code->_co_monitoring->tools

) {

code->_co_monitoring->tools[offset] |= tools;

}

else {

/* Single tool */

assert(_Py_popcount32(tools) == 1);

assert(tools_is_subset_for_event(code, event, tools));

}

instrument(code, offset);

}

static void

add_line_tools(PyCodeObject * code, int offset, int tools)

{

assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_LINE, tools));

assert(code->_co_monitoring);

if (code->_co_monitoring->line_tools) {

code->_co_monitoring->line_tools[offset] |= tools;

}

else {

/* Single tool */

assert(_Py_popcount32(tools) == 1);

}

instrument_line(code, offset);

}

static void

add_per_instruction_tools(PyCodeObject * code, int offset, int tools)

{

assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_INSTRUCTION, tools));

assert(code->_co_monitoring);

if (code->_co_monitoring->per_instruction_tools) {

code->_co_monitoring->per_instruction_tools[offset] |= tools;

}

else {

/* Single tool */

assert(_Py_popcount32(tools) == 1);

}

instrument_per_instruction(code, offset);

}

static void

remove_per_instruction_tools(PyCodeObject * code, int offset, int tools)

{

assert(code->_co_monitoring);

if (code->_co_monitoring->per_instruction_tools) {

uint8_t *toolsptr = &code->_co_monitoring->per_instruction_tools[offset];

*toolsptr &= ~tools;

if (*toolsptr == 0) {

de_instrument_per_instruction(code, offset);

}

}

else {

/* Single tool */

uint8_t single_tool = code->_co_monitoring->active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION];

assert(_Py_popcount32(single_tool) <= 1);

if (((single_tool & tools) == single_tool)) {

de_instrument_per_instruction(code, offset);

}

}

}

static int

call_one_instrument(

PyInterpreterState *interp, PyThreadState *tstate, PyObject **args,

Py_ssize_t nargsf, int8_t tool, int event)

{

assert(0 <= tool && tool < 8);

assert(tstate->tracing == 0);

PyObject *instrument = interp->monitoring_callables[tool][event];

if (instrument == NULL) {

return 0;

}

int old_what = tstate->what_event;

tstate->what_event = event;

tstate->tracing++;

PyObject *res = _PyObject_VectorcallTstate(tstate, instrument, args, nargsf, NULL);

tstate->tracing--;

tstate->what_event = old_what;

if (res == NULL) {

return -1;

}

Py_DECREF(res);

return (res == &_PyInstrumentation_DISABLE);

}

static const int8_t MOST_SIGNIFICANT_BITS[16] = {

-1, 0, 1, 1,

2, 2, 2, 2,

3, 3, 3, 3,

3, 3, 3, 3,

};

static inline int most_significant_bit(uint8_t bits) {

assert(bits != 0);

if (bits > 15) {

return MOST_SIGNIFICANT_BITS[bits>>4]+4;

}

return MOST_SIGNIFICANT_BITS[bits];

}

static bool

is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp)

{

return interp->monitoring_version == code->_co_instrumentation_version;

}

#ifndef NDEBUG

static bool

instrumentation_cross_checks(PyInterpreterState *interp, PyCodeObject *code)

{

_Py_LocalMonitors expected = local_union(

interp->monitors,

code->_co_monitoring->local_monitors);

return monitors_equals(code->_co_monitoring->active_monitors, expected);

}

static inline uint8_t

get_tools_for_instruction(PyCodeObject *code, PyInterpreterState *interp, int i, int event)

{

uint8_t tools;

assert(event != PY_MONITORING_EVENT_LINE);

assert(event != PY_MONITORING_EVENT_INSTRUCTION);

if (event >= _PY_MONITORING_UNGROUPED_EVENTS) {

assert(event == PY_MONITORING_EVENT_C_RAISE ||

event == PY_MONITORING_EVENT_C_RETURN);

event = PY_MONITORING_EVENT_CALL;

}

if (PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) {

CHECK(is_version_up_to_date(code, interp));

CHECK(instrumentation_cross_checks(interp, code));

if (code->_co_monitoring->tools) {

tools = code->_co_monitoring->tools[i];

}

else {

tools = code->_co_monitoring->active_monitors.tools[event];

}

}

else {

tools = interp->monitors.tools[event];

}

return tools;

}

static const char *const event_names [] = {

[PY_MONITORING_EVENT_PY_START] = "PY_START",

[PY_MONITORING_EVENT_PY_RESUME] = "PY_RESUME",

[PY_MONITORING_EVENT_PY_RETURN] = "PY_RETURN",

[PY_MONITORING_EVENT_PY_YIELD] = "PY_YIELD",

[PY_MONITORING_EVENT_CALL] = "CALL",

[PY_MONITORING_EVENT_LINE] = "LINE",

[PY_MONITORING_EVENT_INSTRUCTION] = "INSTRUCTION",

[PY_MONITORING_EVENT_JUMP] = "JUMP",

[PY_MONITORING_EVENT_BRANCH] = "BRANCH",

[PY_MONITORING_EVENT_C_RETURN] = "C_RETURN",

[PY_MONITORING_EVENT_PY_THROW] = "PY_THROW",

[PY_MONITORING_EVENT_RAISE] = "RAISE",

[PY_MONITORING_EVENT_RERAISE] = "RERAISE",

[PY_MONITORING_EVENT_EXCEPTION_HANDLED] = "EXCEPTION_HANDLED",

[PY_MONITORING_EVENT_C_RAISE] = "C_RAISE",

[PY_MONITORING_EVENT_PY_UNWIND] = "PY_UNWIND",

[PY_MONITORING_EVENT_STOP_ITERATION] = "STOP_ITERATION",

};

static int

call_instrumentation_vector(

PyThreadState *tstate, int event,

_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargs, PyObject *args[])

{

if (tstate->tracing) {

return 0;

}

assert(!_PyErr_Occurred(tstate));

assert(args[0] == NULL);

PyCodeObject *code = frame->f_code;

assert(code->_co_instrumentation_version == tstate->interp->monitoring_version);

assert(is_version_up_to_date(code, tstate->interp));

assert(instrumentation_cross_checks(tstate->interp, code));

assert(args[1] == NULL);

args[1] = (PyObject *)code;

int offset = (int)(instr - _PyCode_CODE(code));

/* Offset visible to user should be the offset in bytes, as that is the

int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT);

PyObject *offset_obj = PyLong_FromLong(bytes_offset);

if (offset_obj == NULL) {

return -1;

}

assert(args[2] == NULL);

args[2] = offset_obj;

PyInterpreterState *interp = tstate->interp;

uint8_t tools = get_tools_for_instruction(code, interp, offset, event);

Py_ssize_t nargsf = nargs | PY_VECTORCALL_ARGUMENTS_OFFSET;

PyObject **callargs = &args[1];

int err = 0;

while (tools) {

int tool = most_significant_bit(tools);

assert(tool >= 0 && tool < 8);

assert(tools & (1 << tool));

tools ^= (1 << tool);

int res = call_one_instrument(interp, tstate, callargs, nargsf, tool, event);

if (res == 0) {

/* Nothing to do */

}

else if (res < 0) {

/* error */

err = -1;

break;

}

else {

/* DISABLE */

if (!PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) {

PyErr_Format(PyExc_ValueError,

"Cannot disable %s events. Callback removed.",

event_names[event]);

/* Clear tool to prevent infinite loop */

Py_CLEAR(interp->monitoring_callables[tool][event]);

err = -1;

break;

}