summaryrefslogtreecommitdiffstats
path: root/contrib/tools/python3/Python/instrumentation.c
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/tools/python3/Python/instrumentation.c')
-rw-r--r--contrib/tools/python3/Python/instrumentation.c1143
1 files changed, 871 insertions, 272 deletions
diff --git a/contrib/tools/python3/Python/instrumentation.c b/contrib/tools/python3/Python/instrumentation.c
index a6ff7a8a985..21d9692b586 100644
--- a/contrib/tools/python3/Python/instrumentation.c
+++ b/contrib/tools/python3/Python/instrumentation.c
@@ -1,32 +1,59 @@
+#include "Python.h"
+#include "opcode_ids.h"
-
-
-#include "Python.h"
+#include "pycore_bitutils.h" // _Py_popcount32
#include "pycore_call.h"
+#include "pycore_ceval.h" // _PY_EVAL_EVENTS_BITS
+#include "pycore_code.h" // _PyCode_Clear_Executors()
+#include "pycore_critical_section.h"
#include "pycore_frame.h"
#include "pycore_interp.h"
#include "pycore_long.h"
+#include "pycore_modsupport.h" // _PyModule_CreateInitialized()
#include "pycore_namespace.h"
#include "pycore_object.h"
-#include "pycore_opcode.h"
+#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE, _PyOpcode_Caches
+#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_UINTPTR_RELEASE
#include "pycore_pyerrors.h"
-#include "pycore_pystate.h"
+#include "pycore_pystate.h" // _PyInterpreterState_GET()
/* Uncomment this to dump debugging output when assertions fail */
// #define INSTRUMENT_DEBUG 1
-PyObject _PyInstrumentation_DISABLE =
-{
- .ob_refcnt = _Py_IMMORTAL_REFCNT,
- .ob_type = &PyBaseObject_Type
-};
+#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)
-PyObject _PyInstrumentation_MISSING =
-{
- .ob_refcnt = _Py_IMMORTAL_REFCNT,
- .ob_type = &PyBaseObject_Type
-};
+#define ASSERT_WORLD_STOPPED_OR_LOCKED(obj) \
+ if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) { \
+ _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); \
+ }
+#define ASSERT_WORLD_STOPPED() assert(_PyInterpreterState_GET()->stoptheworld.world_stopped);
+
+#else
+
+#define ASSERT_WORLD_STOPPED_OR_LOCKED(obj)
+#define ASSERT_WORLD_STOPPED()
+
+#endif
+
+#ifdef Py_GIL_DISABLED
+
+#define LOCK_CODE(code) \
+ assert(!_PyInterpreterState_GET()->stoptheworld.world_stopped); \
+ Py_BEGIN_CRITICAL_SECTION(code)
+
+#define UNLOCK_CODE() Py_END_CRITICAL_SECTION()
+
+#else
+
+#define LOCK_CODE(code)
+#define UNLOCK_CODE()
+
+#endif
+
+PyObject _PyInstrumentation_DISABLE = _PyObject_HEAD_INIT(&PyBaseObject_Type);
+
+PyObject _PyInstrumentation_MISSING = _PyObject_HEAD_INIT(&PyBaseObject_Type);
static const int8_t EVENT_FOR_OPCODE[256] = {
[RETURN_CONST] = PY_MONITORING_EVENT_PY_RETURN,
@@ -35,6 +62,8 @@ static const int8_t EVENT_FOR_OPCODE[256] = {
[INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN,
[CALL] = PY_MONITORING_EVENT_CALL,
[INSTRUMENTED_CALL] = PY_MONITORING_EVENT_CALL,
+ [CALL_KW] = PY_MONITORING_EVENT_CALL,
+ [INSTRUMENTED_CALL_KW] = PY_MONITORING_EVENT_CALL,
[CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL,
[INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL,
[LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL,
@@ -67,6 +96,7 @@ static const uint8_t DE_INSTRUMENT[256] = {
[INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE,
[INSTRUMENTED_RETURN_CONST] = RETURN_CONST,
[INSTRUMENTED_CALL] = CALL,
+ [INSTRUMENTED_CALL_KW] = CALL_KW,
[INSTRUMENTED_CALL_FUNCTION_EX] = CALL_FUNCTION_EX,
[INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE,
[INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD,
@@ -88,6 +118,8 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = {
[INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE,
[CALL] = INSTRUMENTED_CALL,
[INSTRUMENTED_CALL] = INSTRUMENTED_CALL,
+ [CALL_KW] = INSTRUMENTED_CALL_KW,
+ [INSTRUMENTED_CALL_KW] = INSTRUMENTED_CALL_KW,
[CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX,
[INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX,
[YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE,
@@ -123,7 +155,7 @@ static inline bool
opcode_has_event(int opcode)
{
return (
- opcode < INSTRUMENTED_LINE &&
+ opcode != INSTRUMENTED_LINE &&
INSTRUMENTED_OPCODES[opcode] > 0
);
}
@@ -232,57 +264,54 @@ get_events(_Py_GlobalMonitors *m, int tool_id)
return result;
}
-/* Line delta.
- * 8 bit value.
- * if line_delta == -128:
- * line = None # represented as -1
- * elif line_delta == -127:
- * line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT));
+/* Module code can have line 0, even though modules start at line 1,
+ * so -1 is a legal delta. */
+#define NO_LINE (-2)
+
+/* Returns the line delta. Defined as:
+ * if line is None:
+ * line_delta = NO_LINE
* else:
- * line = first_line + (offset >> OFFSET_SHIFT) + line_delta;
+ * line_delta = line - first_line
*/
-
-#define NO_LINE -128
-#define COMPUTED_LINE -127
-
-#define OFFSET_SHIFT 4
-
-static int8_t
-compute_line_delta(PyCodeObject *code, int offset, int line)
+static int
+compute_line_delta(PyCodeObject *code, int line)
{
if (line < 0) {
+ assert(line == -1);
return NO_LINE;
}
- int delta = line - code->co_firstlineno - (offset >> OFFSET_SHIFT);
- if (delta <= INT8_MAX && delta > COMPUTED_LINE) {
- return delta;
- }
- return COMPUTED_LINE;
+ int delta = line - code->co_firstlineno;
+ assert(delta > NO_LINE);
+ return delta;
}
static int
-compute_line(PyCodeObject *code, int offset, int8_t line_delta)
+compute_line(PyCodeObject *code, int line_delta)
{
- if (line_delta > COMPUTED_LINE) {
- return code->co_firstlineno + (offset >> OFFSET_SHIFT) + line_delta;
- }
if (line_delta == NO_LINE) {
-
return -1;
}
- assert(line_delta == COMPUTED_LINE);
- /* Look it up */
- return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT));
+ assert(line_delta > NO_LINE);
+ return code->co_firstlineno + line_delta;
}
-static int
-instruction_length(PyCodeObject *code, int offset)
+static inline uint8_t
+get_original_opcode(_PyCoLineInstrumentationData *line_data, int index)
+{
+ return line_data->data[index*line_data->bytes_per_entry];
+}
+
+int
+_PyInstruction_GetLength(PyCodeObject *code, int offset)
{
+ ASSERT_WORLD_STOPPED_OR_LOCKED(code);
+
int opcode = _PyCode_CODE(code)[offset].op.code;
assert(opcode != 0);
assert(opcode != RESERVED);
if (opcode == INSTRUMENTED_LINE) {
- opcode = code->_co_monitoring->lines[offset].original_opcode;
+ opcode = get_original_opcode(code->_co_monitoring->lines, offset);
}
if (opcode == INSTRUMENTED_INSTRUCTION) {
opcode = code->_co_monitoring->per_instruction_opcodes[offset];
@@ -296,10 +325,62 @@ instruction_length(PyCodeObject *code, int offset)
}
assert(opcode != 0);
assert(!is_instrumented(opcode));
+ if (opcode == ENTER_EXECUTOR) {
+ int exec_index = _PyCode_CODE(code)[offset].op.arg;
+ _PyExecutorObject *exec = code->co_executors->executors[exec_index];
+ opcode = _PyOpcode_Deopt[exec->vm_data.opcode];
+
+ }
+ assert(opcode != ENTER_EXECUTOR);
assert(opcode == _PyOpcode_Deopt[opcode]);
return 1 + _PyOpcode_Caches[opcode];
}
+static inline uint8_t *
+get_original_opcode_ptr(_PyCoLineInstrumentationData *line_data, int index)
+{
+ return &line_data->data[index*line_data->bytes_per_entry];
+}
+
+static inline void
+set_original_opcode(_PyCoLineInstrumentationData *line_data, int index, uint8_t opcode)
+{
+ line_data->data[index*line_data->bytes_per_entry] = opcode;
+}
+
+static inline int
+get_line_delta(_PyCoLineInstrumentationData *line_data, int index)
+{
+ uint8_t *ptr = &line_data->data[index*line_data->bytes_per_entry+1];
+ assert(line_data->bytes_per_entry >= 2);
+ uint32_t value = *ptr;
+ for (int idx = 2; idx < line_data->bytes_per_entry; idx++) {
+ ptr++;
+ int shift = (idx-1)*8;
+ value |= ((uint32_t)(*ptr)) << shift;
+ }
+ assert(value < INT_MAX);
+ /* NO_LINE is stored as zero. */
+ return ((int)value) + NO_LINE;
+}
+
+static inline void
+set_line_delta(_PyCoLineInstrumentationData *line_data, int index, int line_delta)
+{
+ /* Store line_delta + 2 as we need -2 to represent no line number */
+ assert(line_delta >= NO_LINE);
+ uint32_t adjusted = line_delta - NO_LINE;
+ uint8_t *ptr = &line_data->data[index*line_data->bytes_per_entry+1];
+ assert(adjusted < (1ULL << ((line_data->bytes_per_entry-1)*8)));
+ assert(line_data->bytes_per_entry >= 2);
+ *ptr = adjusted & 0xff;
+ for (int idx = 2; idx < line_data->bytes_per_entry; idx++) {
+ ptr++;
+ adjusted >>= 8;
+ *ptr = adjusted & 0xff;
+ }
+}
+
#ifdef INSTRUMENT_DEBUG
static void
@@ -319,11 +400,15 @@ dump_instrumentation_data_lines(PyCodeObject *code, _PyCoLineInstrumentationData
if (lines == NULL) {
fprintf(out, ", lines = NULL");
}
- else if (lines[i].original_opcode == 0) {
- fprintf(out, ", lines = {original_opcode = No LINE (0), line_delta = %d)", lines[i].line_delta);
- }
else {
- fprintf(out, ", lines = {original_opcode = %s, line_delta = %d)", _PyOpcode_OpName[lines[i].original_opcode], lines[i].line_delta);
+ int opcode = get_original_opcode(lines, i);
+ int line_delta = get_line_delta(lines, i);
+ if (opcode == 0) {
+ fprintf(out, ", lines = {original_opcode = No LINE (0), line_delta = %d)", line_delta);
+ }
+ else {
+ fprintf(out, ", lines = {original_opcode = %s, line_delta = %d)", _PyOpcode_OpName[opcode], line_delta);
+ }
}
}
@@ -373,6 +458,12 @@ dump_local_monitors(const char *prefix, _Py_LocalMonitors monitors, FILE*out)
}
}
+/** NOTE:
+ * Do not use PyCode_Addr2Line to determine the line number in instrumentation,
+ * as `PyCode_Addr2Line` uses the monitoring data if it is available.
+ */
+
+
/* No error checking -- Don't use this for anything but experimental debugging */
static void
dump_instrumentation_data(PyCodeObject *code, int star, FILE*out)
@@ -390,14 +481,16 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out)
dump_local_monitors("Active", data->active_monitors, out);
int code_len = (int)Py_SIZE(code);
bool starred = false;
- for (int i = 0; i < code_len; i += instruction_length(code, i)) {
+ PyCodeAddressRange range;
+ _PyCode_InitAddressRange(code, &range);
+ for (int i = 0; i < code_len; i += _PyInstruction_GetLength(code, i)) {
_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];
int opcode = instr->op.code;
if (i == star) {
fprintf(out, "** ");
starred = true;
}
- fprintf(out, "Offset: %d, line: %d %s: ", i, PyCode_Addr2Line(code, i*2), _PyOpcode_OpName[opcode]);
+ fprintf(out, "Offset: %d, line: %d %s: ", i, _PyCode_CheckLineNumber(i*2, &range), _PyOpcode_OpName[opcode]);
dump_instrumentation_data_tools(code, data->tools, i, out);
dump_instrumentation_data_lines(code, data->lines, i, out);
dump_instrumentation_data_line_tools(code, data->line_tools, i, out);
@@ -426,11 +519,13 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out)
static bool
valid_opcode(int opcode)
{
- if (opcode > 0 &&
+ if (opcode == INSTRUMENTED_LINE) {
+ return true;
+ }
+ if (IS_VALID_OPCODE(opcode) &&
+ opcode != CACHE &&
opcode != RESERVED &&
- opcode < 255 &&
- _PyOpcode_OpName[opcode] &&
- _PyOpcode_OpName[opcode][0] != '<')
+ opcode < 255)
{
return true;
}
@@ -440,6 +535,8 @@ valid_opcode(int opcode)
static void
sanity_check_instrumentation(PyCodeObject *code)
{
+ ASSERT_WORLD_STOPPED_OR_LOCKED(code);
+
_PyCoMonitoringData *data = code->_co_monitoring;
if (data == NULL) {
return;
@@ -458,10 +555,12 @@ sanity_check_instrumentation(PyCodeObject *code)
code->_co_monitoring->active_monitors,
active_monitors));
int code_len = (int)Py_SIZE(code);
+ PyCodeAddressRange range;
+ _PyCode_InitAddressRange(co, &range);
for (int i = 0; i < code_len;) {
_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];
int opcode = instr->op.code;
- int base_opcode = _Py_GetBaseOpcode(code, i);
+ int base_opcode = _Py_GetBaseCodeUnit(code, i).op.code;
CHECK(valid_opcode(opcode));
CHECK(valid_opcode(base_opcode));
if (opcode == INSTRUMENTED_INSTRUCTION) {
@@ -469,15 +568,11 @@ sanity_check_instrumentation(PyCodeObject *code)
if (!is_instrumented(opcode)) {
CHECK(_PyOpcode_Deopt[opcode] == opcode);
}
- if (data->per_instruction_tools) {
- uint8_t tools = active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION];
- CHECK((tools & data->per_instruction_tools[i]) == data->per_instruction_tools[i]);
- }
}
if (opcode == INSTRUMENTED_LINE) {
CHECK(data->lines);
- CHECK(valid_opcode(data->lines[i].original_opcode));
- opcode = data->lines[i].original_opcode;
+ opcode = get_original_opcode(data->lines, i);
+ CHECK(valid_opcode(opcode));
CHECK(opcode != END_FOR);
CHECK(opcode != RESUME);
CHECK(opcode != RESUME_CHECK);
@@ -492,7 +587,7 @@ sanity_check_instrumentation(PyCodeObject *code)
* *and* we are executing a INSTRUMENTED_LINE instruction
* that has de-instrumented itself, then we will execute
* an invalid INSTRUMENTED_INSTRUCTION */
- CHECK(data->lines[i].original_opcode != INSTRUMENTED_INSTRUCTION);
+ CHECK(get_original_opcode(data->lines, i) != INSTRUMENTED_INSTRUCTION);
}
if (opcode == INSTRUMENTED_INSTRUCTION) {
CHECK(data->per_instruction_opcodes[i] != 0);
@@ -507,9 +602,9 @@ sanity_check_instrumentation(PyCodeObject *code)
}
CHECK(active_monitors.tools[event] != 0);
}
- if (data->lines && base_opcode != END_FOR) {
- int line1 = compute_line(code, i, data->lines[i].line_delta);
- int line2 = PyCode_Addr2Line(code, i*sizeof(_Py_CODEUNIT));
+ if (data->lines && get_original_opcode(data->lines, i)) {
+ int line1 = compute_line(code, get_line_delta(data->lines, i));
+ int line2 = _PyCode_CheckLineNumber(i*sizeof(_Py_CODEUNIT), &range);
CHECK(line1 == line2);
}
CHECK(valid_opcode(opcode));
@@ -527,7 +622,7 @@ sanity_check_instrumentation(PyCodeObject *code)
CHECK(local_tools == 0xff);
}
}
- i += instruction_length(code, i);
+ i += _PyInstruction_GetLength(code, i);
assert(i <= code_len);
}
}
@@ -542,7 +637,7 @@ int _Py_GetBaseOpcode(PyCodeObject *code, int i)
{
int opcode = _PyCode_CODE(code)[i].op.code;
if (opcode == INSTRUMENTED_LINE) {
- opcode = code->_co_monitoring->lines[i].original_opcode;
+ opcode = get_original_opcode(code->_co_monitoring->lines, i);
}
if (opcode == INSTRUMENTED_INSTRUCTION) {
opcode = code->_co_monitoring->per_instruction_opcodes[i];
@@ -565,8 +660,9 @@ de_instrument(PyCodeObject *code, int i, int event)
_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];
uint8_t *opcode_ptr = &instr->op.code;
int opcode = *opcode_ptr;
+ assert(opcode != ENTER_EXECUTOR);
if (opcode == INSTRUMENTED_LINE) {
- opcode_ptr = &code->_co_monitoring->lines[i].original_opcode;
+ opcode_ptr = get_original_opcode_ptr(code->_co_monitoring->lines, i);
opcode = *opcode_ptr;
}
if (opcode == INSTRUMENTED_INSTRUCTION) {
@@ -578,9 +674,10 @@ de_instrument(PyCodeObject *code, int i, int event)
return;
}
CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented);
- *opcode_ptr = deinstrumented;
+ FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, deinstrumented);
if (_PyOpcode_Caches[deinstrumented]) {
- instr[1].cache = adaptive_counter_warmup();
+ FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.as_counter,
+ adaptive_counter_warmup().as_counter);
}
}
@@ -592,16 +689,16 @@ de_instrument_line(PyCodeObject *code, int i)
if (opcode != INSTRUMENTED_LINE) {
return;
}
- _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i];
- int original_opcode = lines->original_opcode;
+ _PyCoLineInstrumentationData *lines = code->_co_monitoring->lines;
+ int original_opcode = get_original_opcode(lines, i);
if (original_opcode == INSTRUMENTED_INSTRUCTION) {
- lines->original_opcode = code->_co_monitoring->per_instruction_opcodes[i];
+ set_original_opcode(lines, i, code->_co_monitoring->per_instruction_opcodes[i]);
}
CHECK(original_opcode != 0);
CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]);
instr->op.code = original_opcode;
if (_PyOpcode_Caches[original_opcode]) {
- instr[1].cache = adaptive_counter_warmup();
+ instr[1].counter = adaptive_counter_warmup();
}
assert(instr->op.code != INSTRUMENTED_LINE);
}
@@ -613,7 +710,7 @@ de_instrument_per_instruction(PyCodeObject *code, int i)
uint8_t *opcode_ptr = &instr->op.code;
int opcode = *opcode_ptr;
if (opcode == INSTRUMENTED_LINE) {
- opcode_ptr = &code->_co_monitoring->lines[i].original_opcode;
+ opcode_ptr = get_original_opcode_ptr(code->_co_monitoring->lines, i);
opcode = *opcode_ptr;
}
if (opcode != INSTRUMENTED_INSTRUCTION) {
@@ -624,12 +721,10 @@ de_instrument_per_instruction(PyCodeObject *code, int i)
CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]);
*opcode_ptr = original_opcode;
if (_PyOpcode_Caches[original_opcode]) {
- instr[1].cache = adaptive_counter_warmup();
+ instr[1].counter = adaptive_counter_warmup();
}
assert(*opcode_ptr != INSTRUMENTED_INSTRUCTION);
assert(instr->op.code != INSTRUMENTED_INSTRUCTION);
- /* Keep things clean for sanity check */
- code->_co_monitoring->per_instruction_opcodes[i] = 0;
}
@@ -640,8 +735,7 @@ instrument(PyCodeObject *code, int i)
uint8_t *opcode_ptr = &instr->op.code;
int opcode =*opcode_ptr;
if (opcode == INSTRUMENTED_LINE) {
- _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i];
- opcode_ptr = &lines->original_opcode;
+ opcode_ptr = get_original_opcode_ptr(code->_co_monitoring->lines, i);
opcode = *opcode_ptr;
}
if (opcode == INSTRUMENTED_INSTRUCTION) {
@@ -655,9 +749,11 @@ instrument(PyCodeObject *code, int i)
int deopt = _PyOpcode_Deopt[opcode];
int instrumented = INSTRUMENTED_OPCODES[deopt];
assert(instrumented);
- *opcode_ptr = instrumented;
+ FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, instrumented);
if (_PyOpcode_Caches[deopt]) {
- instr[1].cache = adaptive_counter_warmup();
+ FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.as_counter,
+ adaptive_counter_warmup().as_counter);
+ instr[1].counter = adaptive_counter_warmup();
}
}
}
@@ -670,9 +766,8 @@ instrument_line(PyCodeObject *code, int i)
if (opcode == INSTRUMENTED_LINE) {
return;
}
- _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i];
- lines->original_opcode = _PyOpcode_Deopt[opcode];
- CHECK(lines->original_opcode > 0);
+ set_original_opcode(code->_co_monitoring->lines, i, _PyOpcode_Deopt[opcode]);
+ CHECK(get_line_delta(code->_co_monitoring->lines, i) > NO_LINE);
*opcode_ptr = INSTRUMENTED_LINE;
}
@@ -683,8 +778,7 @@ instrument_per_instruction(PyCodeObject *code, int i)
uint8_t *opcode_ptr = &instr->op.code;
int opcode = *opcode_ptr;
if (opcode == INSTRUMENTED_LINE) {
- _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i];
- opcode_ptr = &lines->original_opcode;
+ opcode_ptr = get_original_opcode_ptr(code->_co_monitoring->lines, i);
opcode = *opcode_ptr;
}
if (opcode == INSTRUMENTED_INSTRUCTION) {
@@ -708,6 +802,7 @@ instrument_per_instruction(PyCodeObject *code, int i)
static void
remove_tools(PyCodeObject * code, int offset, int event, int tools)
{
+ ASSERT_WORLD_STOPPED_OR_LOCKED(code);
assert(event != PY_MONITORING_EVENT_LINE);
assert(event != PY_MONITORING_EVENT_INSTRUCTION);
assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event));
@@ -733,7 +828,7 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools)
static bool
tools_is_subset_for_event(PyCodeObject * code, int event, int tools)
{
- int global_tools = PyInterpreterState_Get()->monitors.tools[event];
+ int global_tools = _PyInterpreterState_GET()->monitors.tools[event];
int local_tools = code->_co_monitoring->local_monitors.tools[event];
return tools == ((global_tools | local_tools) & tools);
}
@@ -742,6 +837,8 @@ tools_is_subset_for_event(PyCodeObject * code, int event, int tools)
static void
remove_line_tools(PyCodeObject * code, int offset, int tools)
{
+ ASSERT_WORLD_STOPPED_OR_LOCKED(code);
+
assert(code->_co_monitoring);
if (code->_co_monitoring->line_tools)
{
@@ -764,6 +861,7 @@ remove_line_tools(PyCodeObject * code, int offset, int tools)
static void
add_tools(PyCodeObject * code, int offset, int event, int tools)
{
+ ASSERT_WORLD_STOPPED_OR_LOCKED(code);
assert(event != PY_MONITORING_EVENT_LINE);
assert(event != PY_MONITORING_EVENT_INSTRUCTION);
assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event));
@@ -784,6 +882,8 @@ add_tools(PyCodeObject * code, int offset, int event, int tools)
static void
add_line_tools(PyCodeObject * code, int offset, int tools)
{
+ ASSERT_WORLD_STOPPED_OR_LOCKED(code);
+
assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_LINE, tools));
assert(code->_co_monitoring);
if (code->_co_monitoring->line_tools) {
@@ -800,6 +900,8 @@ add_line_tools(PyCodeObject * code, int offset, int tools)
static void
add_per_instruction_tools(PyCodeObject * code, int offset, int tools)
{
+ ASSERT_WORLD_STOPPED_OR_LOCKED(code);
+
assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_INSTRUCTION, tools));
assert(code->_co_monitoring);
if (code->_co_monitoring->per_instruction_tools) {
@@ -816,6 +918,8 @@ add_per_instruction_tools(PyCodeObject * code, int offset, int tools)
static void
remove_per_instruction_tools(PyCodeObject * code, int offset, int tools)
{
+ ASSERT_WORLD_STOPPED_OR_LOCKED(code);
+
assert(code->_co_monitoring);
if (code->_co_monitoring->per_instruction_tools) {
uint8_t *toolsptr = &code->_co_monitoring->per_instruction_tools[offset];
@@ -839,7 +943,7 @@ remove_per_instruction_tools(PyCodeObject * code, int offset, int tools)
static int
call_one_instrument(
PyInterpreterState *interp, PyThreadState *tstate, PyObject **args,
- Py_ssize_t nargsf, int8_t tool, int event)
+ size_t nargsf, int8_t tool, int event)
{
assert(0 <= tool && tool < 8);
assert(tstate->tracing == 0);
@@ -878,21 +982,87 @@ static inline int most_significant_bit(uint8_t bits) {
return MOST_SIGNIFICANT_BITS[bits];
}
+static uint32_t
+global_version(PyInterpreterState *interp)
+{
+ uint32_t version = (uint32_t)_Py_atomic_load_uintptr_relaxed(
+ &interp->ceval.instrumentation_version);
+#ifdef Py_DEBUG
+ PyThreadState *tstate = _PyThreadState_GET();
+ uint32_t thread_version =
+ (uint32_t)(_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) &
+ ~_PY_EVAL_EVENTS_MASK);
+ assert(thread_version == version);
+#endif
+ return version;
+}
+
+/* Atomically set the given version in the given location, without touching
+ anything in _PY_EVAL_EVENTS_MASK. */
+static void
+set_version_raw(uintptr_t *ptr, uint32_t version)
+{
+ uintptr_t old = _Py_atomic_load_uintptr_relaxed(ptr);
+ uintptr_t new;
+ do {
+ new = (old & _PY_EVAL_EVENTS_MASK) | version;
+ } while (!_Py_atomic_compare_exchange_uintptr(ptr, &old, new));
+}
+
+static void
+set_global_version(PyThreadState *tstate, uint32_t version)
+{
+ ASSERT_WORLD_STOPPED();
+
+ assert((version & _PY_EVAL_EVENTS_MASK) == 0);
+ PyInterpreterState *interp = tstate->interp;
+ set_version_raw(&interp->ceval.instrumentation_version, version);
+
+#ifdef Py_GIL_DISABLED
+ // Set the version on all threads in free-threaded builds.
+ _PyRuntimeState *runtime = &_PyRuntime;
+ HEAD_LOCK(runtime);
+ for (tstate = interp->threads.head; tstate;
+ tstate = PyThreadState_Next(tstate)) {
+ set_version_raw(&tstate->eval_breaker, version);
+ };
+ HEAD_UNLOCK(runtime);
+#else
+ // Normal builds take the current version from instrumentation_version when
+ // attaching a thread, so we only have to set the current thread's version.
+ set_version_raw(&tstate->eval_breaker, version);
+#endif
+}
+
static bool
is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp)
{
- return interp->monitoring_version == code->_co_instrumentation_version;
+ ASSERT_WORLD_STOPPED_OR_LOCKED(code);
+ return global_version(interp) == code->_co_instrumentation_version;
}
#ifndef NDEBUG
static bool
instrumentation_cross_checks(PyInterpreterState *interp, PyCodeObject *code)
{
+ ASSERT_WORLD_STOPPED_OR_LOCKED(code);
_Py_LocalMonitors expected = local_union(
interp->monitors,
code->_co_monitoring->local_monitors);
return monitors_equals(code->_co_monitoring->active_monitors, expected);
}
+
+static int
+debug_check_sanity(PyInterpreterState *interp, PyCodeObject *code)
+{
+ int res;
+ LOCK_CODE(code);
+ res = is_version_up_to_date(code, interp) &&
+ instrumentation_cross_checks(interp, code);
+ UNLOCK_CODE();
+ return res;
+}
+
#endif
static inline uint8_t
@@ -907,8 +1077,7 @@ get_tools_for_instruction(PyCodeObject *code, PyInterpreterState *interp, int i,
event = PY_MONITORING_EVENT_CALL;
}
if (PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) {
- CHECK(is_version_up_to_date(code, interp));
- CHECK(instrumentation_cross_checks(interp, code));
+ CHECK(debug_check_sanity(interp, code));
if (code->_co_monitoring->tools) {
tools = code->_co_monitoring->tools[i];
}
@@ -952,17 +1121,14 @@ call_instrumentation_vector(
}
assert(!_PyErr_Occurred(tstate));
assert(args[0] == NULL);
- PyCodeObject *code = frame->f_code;
- assert(code->_co_instrumentation_version == tstate->interp->monitoring_version);
- assert(is_version_up_to_date(code, tstate->interp));
- assert(instrumentation_cross_checks(tstate->interp, code));
+ PyCodeObject *code = _PyFrame_GetCode(frame);
assert(args[1] == NULL);
args[1] = (PyObject *)code;
int offset = (int)(instr - _PyCode_CODE(code));
/* Offset visible to user should be the offset in bytes, as that is the
* convention for APIs involving code offsets. */
int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT);
- PyObject *offset_obj = PyLong_FromSsize_t(bytes_offset);
+ PyObject *offset_obj = PyLong_FromLong(bytes_offset);
if (offset_obj == NULL) {
return -1;
}
@@ -970,7 +1136,7 @@ call_instrumentation_vector(
args[2] = offset_obj;
PyInterpreterState *interp = tstate->interp;
uint8_t tools = get_tools_for_instruction(code, interp, offset, event);
- Py_ssize_t nargsf = nargs | PY_VECTORCALL_ARGUMENTS_OFFSET;
+ size_t nargsf = (size_t) nargs | PY_VECTORCALL_ARGUMENTS_OFFSET;
PyObject **callargs = &args[1];
int err = 0;
while (tools) {
@@ -999,7 +1165,9 @@ call_instrumentation_vector(
break;
}
else {
+ LOCK_CODE(code);
remove_tools(code, offset, event, 1 << tool);
+ UNLOCK_CODE();
}
}
}
@@ -1041,10 +1209,8 @@ _Py_call_instrumentation_jump(
{
assert(event == PY_MONITORING_EVENT_JUMP ||
event == PY_MONITORING_EVENT_BRANCH);
- assert(frame->prev_instr == instr);
- /* Event should occur after the jump */
- frame->prev_instr = target;
- PyCodeObject *code = frame->f_code;
+ assert(frame->instr_ptr == instr);
+ PyCodeObject *code = _PyFrame_GetCode(frame);
int to = (int)(target - _PyCode_CODE(code));
PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT));
if (to_obj == NULL) {
@@ -1056,12 +1222,10 @@ _Py_call_instrumentation_jump(
if (err) {
return NULL;
}
- if (frame->prev_instr != target) {
+ if (frame->instr_ptr != instr) {
/* The callback has caused a jump (by setting the line number) */
- return frame->prev_instr;
+ return frame->instr_ptr;
}
- /* Reset prev_instr for INSTRUMENTED_LINE */
- frame->prev_instr = instr;
return target;
}
@@ -1092,63 +1256,90 @@ _Py_call_instrumentation_exc2(
call_instrumentation_vector_protected(tstate, event, frame, instr, 4, args);
}
-
int
_Py_Instrumentation_GetLine(PyCodeObject *code, int index)
{
_PyCoMonitoringData *monitoring = code->_co_monitoring;
assert(monitoring != NULL);
assert(monitoring->lines != NULL);
- assert(index >= code->_co_firsttraceable);
assert(index < Py_SIZE(code));
- _PyCoLineInstrumentationData *line_data = &monitoring->lines[index];
- int8_t line_delta = line_data->line_delta;
- int line = compute_line(code, index, line_delta);
+ _PyCoLineInstrumentationData *line_data = monitoring->lines;
+ int line_delta = get_line_delta(line_data, index);
+ int line = compute_line(code, line_delta);
return line;
}
int
_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev)
{
- frame->prev_instr = instr;
- PyCodeObject *code = frame->f_code;
- assert(is_version_up_to_date(code, tstate->interp));
- assert(instrumentation_cross_checks(tstate->interp, code));
+ PyCodeObject *code = _PyFrame_GetCode(frame);
+ assert(tstate->tracing == 0);
+ assert(debug_check_sanity(tstate->interp, code));
int i = (int)(instr - _PyCode_CODE(code));
_PyCoMonitoringData *monitoring = code->_co_monitoring;
- _PyCoLineInstrumentationData *line_data = &monitoring->lines[i];
- if (tstate->tracing) {
- goto done;
- }
+ _PyCoLineInstrumentationData *line_data = monitoring->lines;
PyInterpreterState *interp = tstate->interp;
- int8_t line_delta = line_data->line_delta;
- int line = compute_line(code, i, line_delta);
+ int line = _Py_Instrumentation_GetLine(code, i);
assert(line >= 0);
+ assert(prev != NULL);
int prev_index = (int)(prev - _PyCode_CODE(code));
int prev_line = _Py_Instrumentation_GetLine(code, prev_index);
if (prev_line == line) {
int prev_opcode = _PyCode_CODE(code)[prev_index].op.code;
/* RESUME and INSTRUMENTED_RESUME are needed for the operation of
- * instrumentation, so must never be hidden by an INSTRUMENTED_LINE.
- */
+ * instrumentation, so must never be hidden by an INSTRUMENTED_LINE.
+ */
if (prev_opcode != RESUME && prev_opcode != INSTRUMENTED_RESUME) {
goto done;
}
}
+
uint8_t tools = code->_co_monitoring->line_tools != NULL ?
code->_co_monitoring->line_tools[i] :
(interp->monitors.tools[PY_MONITORING_EVENT_LINE] |
code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_LINE]
);
- PyObject *line_obj = PyLong_FromSsize_t(line);
+ /* Special case sys.settrace to avoid boxing the line number,
+ * only to immediately unbox it. */
+ if (tools & (1 << PY_MONITORING_SYS_TRACE_ID)) {
+ if (tstate->c_tracefunc != NULL) {
+ PyFrameObject *frame_obj = _PyFrame_GetFrameObject(frame);
+ if (frame_obj == NULL) {
+ return -1;
+ }
+ if (frame_obj->f_trace_lines) {
+ /* Need to set tracing and what_event as if using
+ * the instrumentation call. */
+ int old_what = tstate->what_event;
+ tstate->what_event = PY_MONITORING_EVENT_LINE;
+ tstate->tracing++;
+ /* Call c_tracefunc directly, having set the line number. */
+ Py_INCREF(frame_obj);
+ frame_obj->f_lineno = line;
+ int err = tstate->c_tracefunc(tstate->c_traceobj, frame_obj, PyTrace_LINE, Py_None);
+ frame_obj->f_lineno = 0;
+ tstate->tracing--;
+ tstate->what_event = old_what;
+ Py_DECREF(frame_obj);
+ if (err) {
+ return -1;
+ }
+ }
+ }
+ tools &= (255 - (1 << PY_MONITORING_SYS_TRACE_ID));
+ }
+ if (tools == 0) {
+ goto done;
+ }
+ PyObject *line_obj = PyLong_FromLong(line);
if (line_obj == NULL) {
return -1;
}
PyObject *args[3] = { NULL, (PyObject *)code, line_obj };
- while (tools) {
+ do {
int tool = most_significant_bit(tools);
- assert(tool >= 0 && tool < 8);
+ assert(tool >= 0 && tool < PY_MONITORING_SYS_PROFILE_ID);
assert(tools & (1 << tool));
tools &= ~(1 << tool);
int res = call_one_instrument(interp, tstate, &args[1],
@@ -1164,15 +1355,17 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame,
}
else {
/* DISABLE */
+ LOCK_CODE(code);
remove_line_tools(code, i, 1 << tool);
+ UNLOCK_CODE();
}
- }
+ } while (tools);
Py_DECREF(line_obj);
uint8_t original_opcode;
done:
- original_opcode = line_data->original_opcode;
+ original_opcode = get_original_opcode(line_data, i);
assert(original_opcode != 0);
- assert(original_opcode < INSTRUMENTED_LINE);
+ assert(original_opcode != INSTRUMENTED_LINE);
assert(_PyOpcode_Deopt[original_opcode] == original_opcode);
return original_opcode;
}
@@ -1180,9 +1373,7 @@ done:
int
_Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr)
{
- PyCodeObject *code = frame->f_code;
- assert(is_version_up_to_date(code, tstate->interp));
- assert(instrumentation_cross_checks(tstate->interp, code));
+ PyCodeObject *code = _PyFrame_GetCode(frame);
int offset = (int)(instr - _PyCode_CODE(code));
_PyCoMonitoringData *instrumentation_data = code->_co_monitoring;
assert(instrumentation_data->per_instruction_opcodes);
@@ -1190,6 +1381,7 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame*
if (tstate->tracing) {
return next_opcode;
}
+ assert(debug_check_sanity(tstate->interp, code));
PyInterpreterState *interp = tstate->interp;
uint8_t tools = instrumentation_data->per_instruction_tools != NULL ?
instrumentation_data->per_instruction_tools[offset] :
@@ -1197,7 +1389,7 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame*
code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION]
);
int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT);
- PyObject *offset_obj = PyLong_FromSsize_t(bytes_offset);
+ PyObject *offset_obj = PyLong_FromLong(bytes_offset);
if (offset_obj == NULL) {
return -1;
}
@@ -1220,7 +1412,9 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame*
}
else {
/* DISABLE */
+ LOCK_CODE(code);
remove_per_instruction_tools(code, offset, 1 << tool);
+ UNLOCK_CODE();
}
}
Py_DECREF(offset_obj);
@@ -1232,25 +1426,30 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame*
PyObject *
_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj)
{
- PyInterpreterState *is = _PyInterpreterState_Get();
+ PyInterpreterState *is = _PyInterpreterState_GET();
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
assert(0 <= event_id && event_id < _PY_MONITORING_EVENTS);
+ _PyEval_StopTheWorld(is);
PyObject *callback = is->monitoring_callables[tool_id][event_id];
is->monitoring_callables[tool_id][event_id] = Py_XNewRef(obj);
+ _PyEval_StartTheWorld(is);
return callback;
}
static void
initialize_tools(PyCodeObject *code)
{
+ ASSERT_WORLD_STOPPED_OR_LOCKED(code);
uint8_t* tools = code->_co_monitoring->tools;
+
assert(tools != NULL);
int code_len = (int)Py_SIZE(code);
for (int i = 0; i < code_len; i++) {
_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];
int opcode = instr->op.code;
+ assert(opcode != ENTER_EXECUTOR);
if (opcode == INSTRUMENTED_LINE) {
- opcode = code->_co_monitoring->lines[i].original_opcode;
+ opcode = get_original_opcode(code->_co_monitoring->lines, i);
}
if (opcode == INSTRUMENTED_INSTRUCTION) {
opcode = code->_co_monitoring->per_instruction_opcodes[i];
@@ -1293,54 +1492,57 @@ initialize_tools(PyCodeObject *code)
}
}
-#define NO_LINE -128
-
static void
-initialize_lines(PyCodeObject *code)
+initialize_lines(PyCodeObject *code, int bytes_per_entry)
{
+ ASSERT_WORLD_STOPPED_OR_LOCKED(code);
_PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines;
+
assert(line_data != NULL);
+ line_data->bytes_per_entry = bytes_per_entry;
int code_len = (int)Py_SIZE(code);
PyCodeAddressRange range;
_PyCode_InitAddressRange(code, &range);
- for (int i = 0; i < code->_co_firsttraceable && i < code_len; i++) {
- line_data[i].original_opcode = 0;
- line_data[i].line_delta = -127;
- }
int current_line = -1;
- for (int i = code->_co_firsttraceable; i < code_len; ) {
+ for (int i = 0; i < code_len; ) {
int opcode = _Py_GetBaseOpcode(code, i);
int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range);
- line_data[i].line_delta = compute_line_delta(code, i, line);
- int length = instruction_length(code, i);
- switch (opcode) {
- case END_ASYNC_FOR:
- case END_FOR:
- case END_SEND:
- case RESUME:
- /* END_FOR cannot start a line, as it is skipped by FOR_ITER
- * END_SEND cannot start a line, as it is skipped by SEND
- * RESUME must not be instrumented with INSTRUMENT_LINE */
- line_data[i].original_opcode = 0;
- break;
- default:
- /* Set original_opcode to the opcode iff the instruction
- * starts a line, and thus should be instrumented.
- * This saves having to perform this check every time the
- * we turn instrumentation on or off, and serves as a sanity
- * check when debugging.
- */
- if (line != current_line && line >= 0) {
- line_data[i].original_opcode = opcode;
- }
- else {
- line_data[i].original_opcode = 0;
- }
- current_line = line;
+ set_line_delta(line_data, i, compute_line_delta(code, line));
+ int length = _PyInstruction_GetLength(code, i);
+ if (i < code->_co_firsttraceable) {
+ set_original_opcode(line_data, i, 0);
+ }
+ else {
+ switch (opcode) {
+ case END_ASYNC_FOR:
+ case END_FOR:
+ case END_SEND:
+ case RESUME:
+ /* END_FOR cannot start a line, as it is skipped by FOR_ITER
+ * END_SEND cannot start a line, as it is skipped by SEND
+ * RESUME and POP_ITER must not be instrumented with INSTRUMENTED_LINE */
+ set_original_opcode(line_data, i, 0);
+ break;
+ default:
+ /* Set original_opcode to the opcode iff the instruction
+ * starts a line, and thus should be instrumented.
+ * This saves having to perform this check every time the
+ * we turn instrumentation on or off, and serves as a sanity
+ * check when debugging.
+ */
+ if (line != current_line && line >= 0) {
+ set_original_opcode(line_data, i, opcode);
+ CHECK(get_line_delta(line_data, i) != NO_LINE);
+ }
+ else {
+ set_original_opcode(line_data, i, 0);
+ }
+ current_line = line;
+ }
}
for (int j = 1; j < length; j++) {
- line_data[i+j].original_opcode = 0;
- line_data[i+j].line_delta = NO_LINE;
+ set_original_opcode(line_data, i+j, 0);
+ set_line_delta(line_data, i+j, NO_LINE);
}
i += length;
}
@@ -1353,7 +1555,7 @@ initialize_lines(PyCodeObject *code)
opcode = _Py_GetBaseOpcode(code, i);
}
oparg = (oparg << 8) | _PyCode_CODE(code)[i].op.arg;
- i += instruction_length(code, i);
+ i += _PyInstruction_GetLength(code, i);
int target = -1;
switch (opcode) {
case POP_JUMP_IF_FALSE:
@@ -1382,8 +1584,9 @@ initialize_lines(PyCodeObject *code)
continue;
}
assert(target >= 0);
- if (line_data[target].line_delta != NO_LINE) {
- line_data[target].original_opcode = _Py_GetBaseOpcode(code, target);
+ if (get_line_delta(line_data, target) != NO_LINE) {
+ int opcode = _Py_GetBaseOpcode(code, target);
+ set_original_opcode(line_data, target, opcode);
}
}
/* Scan exception table */
@@ -1405,9 +1608,8 @@ initialize_lines(PyCodeObject *code)
* END_ASYNC_FOR is a bit special as it marks the end of
* an `async for` loop, which should not generate its own
* line event. */
- if (line_data[handler].line_delta != NO_LINE &&
- original_opcode != END_ASYNC_FOR) {
- line_data[handler].original_opcode = original_opcode;
+ if (get_line_delta(line_data, handler) != NO_LINE && original_opcode != END_ASYNC_FOR) {
+ set_original_opcode(line_data, handler, original_opcode);
}
}
}
@@ -1415,7 +1617,9 @@ initialize_lines(PyCodeObject *code)
static void
initialize_line_tools(PyCodeObject *code, _Py_LocalMonitors *all_events)
{
+ ASSERT_WORLD_STOPPED_OR_LOCKED(code);
uint8_t *line_tools = code->_co_monitoring->line_tools;
+
assert(line_tools != NULL);
int code_len = (int)Py_SIZE(code);
for (int i = 0; i < code_len; i++) {
@@ -1426,6 +1630,7 @@ initialize_line_tools(PyCodeObject *code, _Py_LocalMonitors *all_events)
static int
allocate_instrumentation_data(PyCodeObject *code)
{
+ ASSERT_WORLD_STOPPED_OR_LOCKED(code);
if (code->_co_monitoring == NULL) {
code->_co_monitoring = PyMem_Malloc(sizeof(_PyCoMonitoringData));
@@ -1447,6 +1652,8 @@ allocate_instrumentation_data(PyCodeObject *code)
static int
update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp)
{
+ ASSERT_WORLD_STOPPED_OR_LOCKED(code);
+
int code_len = (int)Py_SIZE(code);
if (allocate_instrumentation_data(code)) {
return -1;
@@ -1465,12 +1672,39 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp)
}
if (all_events.tools[PY_MONITORING_EVENT_LINE]) {
if (code->_co_monitoring->lines == NULL) {
- code->_co_monitoring->lines = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData));
+ PyCodeAddressRange range;
+ _PyCode_InitAddressRange(code, &range);
+ int max_line = code->co_firstlineno + 1;
+ _PyCode_InitAddressRange(code, &range);
+ for (int i = code->_co_firsttraceable; i < code_len; ) {
+ int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range);
+ if (line > max_line) {
+ max_line = line;
+ }
+ int length = _PyInstruction_GetLength(code, i);
+ i += length;
+ }
+ int bytes_per_entry;
+ int max_delta = max_line - code->co_firstlineno;
+ /* We store delta+2 in the table, so 253 is max for one byte */
+ if (max_delta < 256+NO_LINE) {
+ bytes_per_entry = 2;
+ }
+ else if (max_delta < (1 << 16)+NO_LINE) {
+ bytes_per_entry = 3;
+ }
+ else if (max_delta < (1 << 24)+NO_LINE) {
+ bytes_per_entry = 4;
+ }
+ else {
+ bytes_per_entry = 5;
+ }
+ code->_co_monitoring->lines = PyMem_Malloc(1 + code_len * bytes_per_entry);
if (code->_co_monitoring->lines == NULL) {
PyErr_NoMemory();
return -1;
}
- initialize_lines(code);
+ initialize_lines(code, bytes_per_entry);
}
if (multitools && code->_co_monitoring->line_tools == NULL) {
code->_co_monitoring->line_tools = PyMem_Malloc(code_len);
@@ -1488,9 +1722,11 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp)
PyErr_NoMemory();
return -1;
}
- /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */
+ // Initialize all of the instructions so if local events change while another thread is executing
+ // we know what the original opcode was.
for (int i = 0; i < code_len; i++) {
- code->_co_monitoring->per_instruction_opcodes[i] = 0;
+ int opcode = _PyCode_CODE(code)[i].op.code;
+ code->_co_monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode];
}
}
if (multitools && code->_co_monitoring->per_instruction_tools == NULL) {
@@ -1499,7 +1735,6 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp)
PyErr_NoMemory();
return -1;
}
- /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */
for (int i = 0; i < code_len; i++) {
code->_co_monitoring->per_instruction_tools[i] = 0;
}
@@ -1508,39 +1743,23 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp)
return 0;
}
-static const uint8_t super_instructions[256] = {
- [LOAD_FAST__LOAD_FAST] = 1,
- [LOAD_FAST__LOAD_CONST] = 1,
- [STORE_FAST__LOAD_FAST] = 1,
- [STORE_FAST__STORE_FAST] = 1,
- [LOAD_CONST__LOAD_FAST] = 1,
-};
-
-/* Should use instruction metadata for this */
-static bool
-is_super_instruction(uint8_t opcode) {
- return super_instructions[opcode] != 0;
-}
-
-int
-_Py_Instrument(PyCodeObject *code, PyInterpreterState *interp)
+static int
+force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp)
{
+ ASSERT_WORLD_STOPPED_OR_LOCKED(code);
- if (is_version_up_to_date(code, interp)) {
- assert(
- interp->monitoring_version == 0 ||
- instrumentation_cross_checks(interp, code)
- );
- return 0;
+#ifdef _Py_TIER2
+ if (code->co_executors != NULL) {
+ _PyCode_Clear_Executors(code);
}
+ _Py_Executors_InvalidateDependency(interp, code, 1);
+#endif
int code_len = (int)Py_SIZE(code);
- /* code->_co_firsttraceable >= code_len indicates
- * that no instrumentation can be inserted.
- * Exit early to avoid creating instrumentation
+ /* Exit early to avoid creating instrumentation
* data for potential statically allocated code
* objects.
* See https://github.com/python/cpython/issues/108390 */
- if (code->_co_firsttraceable >= code_len) {
+ if (code->co_flags & CO_NO_MONITORING_EVENTS) {
return 0;
}
if (update_instrumentation_data(code, interp)) {
@@ -1563,21 +1782,16 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp)
assert(monitors_are_empty(monitors_and(new_events, removed_events)));
}
code->_co_monitoring->active_monitors = active_events;
- code->_co_instrumentation_version = interp->monitoring_version;
if (monitors_are_empty(new_events) && monitors_are_empty(removed_events)) {
-#ifdef INSTRUMENT_DEBUG
- sanity_check_instrumentation(code);
-#endif
- return 0;
+ goto done;
}
/* Insert instrumentation */
- for (int i = 0; i < code_len; i+= instruction_length(code, i)) {
+ for (int i = code->_co_firsttraceable; i < code_len; i+= _PyInstruction_GetLength(code, i)) {
_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];
- if (is_super_instruction(instr->op.code)) {
- instr->op.code = _PyOpcode_Deopt[instr->op.code];
- }
CHECK(instr->op.code != 0);
+ assert(instr->op.code != ENTER_EXECUTOR);
int base_opcode = _Py_GetBaseOpcode(code, i);
+ assert(base_opcode != ENTER_EXECUTOR);
if (opcode_has_event(base_opcode)) {
int8_t event;
if (base_opcode == RESUME) {
@@ -1600,70 +1814,93 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp)
// GH-103845: We need to remove both the line and instruction instrumentation before
// adding new ones, otherwise we may remove the newly added instrumentation.
-
uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE];
uint8_t removed_per_instruction_tools = removed_events.tools[PY_MONITORING_EVENT_INSTRUCTION];
if (removed_line_tools) {
_PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines;
for (int i = code->_co_firsttraceable; i < code_len;) {
- if (line_data[i].original_opcode) {
- if (removed_line_tools) {
- remove_line_tools(code, i, removed_line_tools);
- }
+ if (get_original_opcode(line_data, i)) {
+ remove_line_tools(code, i, removed_line_tools);
}
- i += instruction_length(code, i);
+ i += _PyInstruction_GetLength(code, i);
}
}
if (removed_per_instruction_tools) {
for (int i = code->_co_firsttraceable; i < code_len;) {
int opcode = _Py_GetBaseOpcode(code, i);
if (opcode == RESUME || opcode == END_FOR) {
- i += instruction_length(code, i);
+ i += _PyInstruction_GetLength(code, i);
continue;
}
- if (removed_per_instruction_tools) {
- remove_per_instruction_tools(code, i, removed_per_instruction_tools);
- }
- i += instruction_length(code, i);
+ remove_per_instruction_tools(code, i, removed_per_instruction_tools);
+ i += _PyInstruction_GetLength(code, i);
}
}
#ifdef INSTRUMENT_DEBUG
sanity_check_instrumentation(code);
#endif
+
uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE];
uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION];
if (new_line_tools) {
_PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines;
for (int i = code->_co_firsttraceable; i < code_len;) {
- if (line_data[i].original_opcode) {
- if (new_line_tools) {
- add_line_tools(code, i, new_line_tools);
- }
+ if (get_original_opcode(line_data, i)) {
+ add_line_tools(code, i, new_line_tools);
}
- i += instruction_length(code, i);
+ i += _PyInstruction_GetLength(code, i);
}
}
if (new_per_instruction_tools) {
for (int i = code->_co_firsttraceable; i < code_len;) {
int opcode = _Py_GetBaseOpcode(code, i);
if (opcode == RESUME || opcode == END_FOR) {
- i += instruction_length(code, i);
+ i += _PyInstruction_GetLength(code, i);
continue;
}
- if (new_per_instruction_tools) {
- add_per_instruction_tools(code, i, new_per_instruction_tools);
- }
- i += instruction_length(code, i);
+ add_per_instruction_tools(code, i, new_per_instruction_tools);
+ i += _PyInstruction_GetLength(code, i);
}
}
+
+done:
+ FT_ATOMIC_STORE_UINTPTR_RELEASE(code->_co_instrumentation_version,
+ global_version(interp));
+
#ifdef INSTRUMENT_DEBUG
sanity_check_instrumentation(code);
#endif
return 0;
}
+static int
+instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp)
+{
+ ASSERT_WORLD_STOPPED_OR_LOCKED(code);
+
+ if (is_version_up_to_date(code, interp)) {
+ assert(
+ interp->ceval.instrumentation_version == 0 ||
+ instrumentation_cross_checks(interp, code)
+ );
+ return 0;
+ }
+
+ return force_instrument_lock_held(code, interp);
+}
+
+int
+_Py_Instrument(PyCodeObject *code, PyInterpreterState *interp)
+{
+ int res;
+ LOCK_CODE(code);
+ res = instrument_lock_held(code, interp);
+ UNLOCK_CODE();
+ return res;
+}
+
#define C_RETURN_EVENTS \
((1 << PY_MONITORING_EVENT_C_RETURN) | \
(1 << PY_MONITORING_EVENT_C_RAISE))
@@ -1673,26 +1910,27 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp)
static int
-instrument_all_executing_code_objects(PyInterpreterState *interp) {
- _PyRuntimeState *runtime = &_PyRuntime;
- HEAD_LOCK(runtime);
- PyThreadState* ts = PyInterpreterState_ThreadHead(interp);
- HEAD_UNLOCK(runtime);
- while (ts) {
- _PyInterpreterFrame *frame = ts->cframe->current_frame;
+instrument_all_executing_code_objects(PyInterpreterState *interp)
+{
+ ASSERT_WORLD_STOPPED();
+
+ int err = 0;
+ HEAD_LOCK(&_PyRuntime);
+ for (PyThreadState *ts = interp->threads.head; ts != NULL; ts = ts->next) {
+ _PyInterpreterFrame *frame = ts->current_frame;
while (frame) {
if (frame->owner != FRAME_OWNED_BY_CSTACK) {
- if (_Py_Instrument(frame->f_code, interp)) {
- return -1;
+ err = instrument_lock_held(_PyFrame_GetCode(frame), interp);
+ if (err) {
+ goto done;
}
}
frame = frame->previous;
}
- HEAD_LOCK(runtime);
- ts = PyThreadState_Next(ts);
- HEAD_UNLOCK(runtime);
}
- return 0;
+done:
+ HEAD_UNLOCK(&_PyRuntime);
+ return err;
}
static void
@@ -1731,29 +1969,46 @@ check_tool(PyInterpreterState *interp, int tool_id)
return 0;
}
+/* We share the eval-breaker with flags, so the monitoring
+ * version goes in the top 24 bits */
+#define MONITORING_VERSION_INCREMENT (1 << _PY_EVAL_EVENTS_BITS)
+
int
_PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events)
{
+ ASSERT_WORLD_STOPPED();
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
- PyInterpreterState *interp = _PyInterpreterState_Get();
+ PyThreadState *tstate = _PyThreadState_GET();
+ PyInterpreterState *interp = tstate->interp;
assert(events < (1 << _PY_MONITORING_UNGROUPED_EVENTS));
if (check_tool(interp, tool_id)) {
return -1;
}
+
uint32_t existing_events = get_events(&interp->monitors, tool_id);
if (existing_events == events) {
return 0;
}
+ uint32_t new_version = global_version(interp) + MONITORING_VERSION_INCREMENT;
+ if (new_version == 0) {
+ PyErr_Format(PyExc_OverflowError, "events set too many times");
+ return -1;
+ }
set_events(&interp->monitors, tool_id, events);
- interp->monitoring_version++;
+ set_global_version(tstate, new_version);
+#ifdef _Py_TIER2
+ _Py_Executors_InvalidateAll(interp, 1);
+#endif
return instrument_all_executing_code_objects(interp);
}
int
_PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet events)
{
+ ASSERT_WORLD_STOPPED();
+
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
- PyInterpreterState *interp = _PyInterpreterState_Get();
+ PyInterpreterState *interp = _PyInterpreterState_GET();
assert(events < (1 << _PY_MONITORING_LOCAL_EVENTS));
if (code->_co_firsttraceable >= Py_SIZE(code)) {
PyErr_Format(PyExc_SystemError, "cannot instrument shim code object '%U'", code->co_name);
@@ -1762,22 +2017,35 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent
if (check_tool(interp, tool_id)) {
return -1;
}
+
if (allocate_instrumentation_data(code)) {
return -1;
}
+
_Py_LocalMonitors *local = &code->_co_monitoring->local_monitors;
uint32_t existing_events = get_local_events(local, tool_id);
if (existing_events == events) {
return 0;
}
set_local_events(local, tool_id, events);
- if (is_version_up_to_date(code, interp)) {
- /* Force instrumentation update */
- code->_co_instrumentation_version = UINT64_MAX;
- }
- if (_Py_Instrument(code, interp)) {
+
+ return force_instrument_lock_held(code, interp);
+}
+
+int
+_PyMonitoring_GetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet *events)
+{
+ assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
+ PyInterpreterState *interp = _PyInterpreterState_GET();
+ if (check_tool(interp, tool_id)) {
return -1;
}
+ if (code->_co_monitoring == NULL) {
+ *events = 0;
+ return 0;
+ }
+ _Py_LocalMonitors *local = &code->_co_monitoring->local_monitors;
+ *events = get_local_events(local, tool_id);
return 0;
}
@@ -1819,7 +2087,7 @@ monitoring_use_tool_id_impl(PyObject *module, int tool_id, PyObject *name)
PyErr_SetString(PyExc_ValueError, "tool name must be a str");
return NULL;
}
- PyInterpreterState *interp = _PyInterpreterState_Get();
+ PyInterpreterState *interp = _PyInterpreterState_GET();
if (interp->monitoring_tool_names[tool_id] != NULL) {
PyErr_Format(PyExc_ValueError, "tool %d is already in use", tool_id);
return NULL;
@@ -1843,7 +2111,7 @@ monitoring_free_tool_id_impl(PyObject *module, int tool_id)
if (check_valid_tool(tool_id)) {
return NULL;
}
- PyInterpreterState *interp = _PyInterpreterState_Get();
+ PyInterpreterState *interp = _PyInterpreterState_GET();
Py_CLEAR(interp->monitoring_tool_names[tool_id]);
Py_RETURN_NONE;
}
@@ -1865,7 +2133,7 @@ monitoring_get_tool_impl(PyObject *module, int tool_id)
if (check_valid_tool(tool_id)) {
return NULL;
}
- PyInterpreterState *interp = _PyInterpreterState_Get();
+ PyInterpreterState *interp = _PyInterpreterState_GET();
PyObject *name = interp->monitoring_tool_names[tool_id];
if (name == NULL) {
Py_RETURN_NONE;
@@ -1959,7 +2227,11 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set)
return NULL;
}
event_set &= ~C_RETURN_EVENTS;
- if (_PyMonitoring_SetEvents(tool_id, event_set)) {
+ PyInterpreterState *interp = _PyInterpreterState_GET();
+ _PyEval_StopTheWorld(interp);
+ int err = _PyMonitoring_SetEvents(tool_id, event_set);
+ _PyEval_StartTheWorld(interp);
+ if (err) {
return NULL;
}
Py_RETURN_NONE;
@@ -2036,7 +2308,11 @@ monitoring_set_local_events_impl(PyObject *module, int tool_id,
return NULL;
}
- if (_PyMonitoring_SetLocalEvents((PyCodeObject*)code, tool_id, event_set)) {
+ PyInterpreterState *interp = _PyInterpreterState_GET();
+ _PyEval_StopTheWorld(interp);
+ int err = _PyMonitoring_SetLocalEvents((PyCodeObject*)code, tool_id, event_set);
+ _PyEval_StartTheWorld(interp);
+ if (err) {
return NULL;
}
Py_RETURN_NONE;
@@ -2055,10 +2331,23 @@ monitoring_restart_events_impl(PyObject *module)
* last restart version > instrumented version for all code objects
* last restart version < current version
*/
- PyInterpreterState *interp = _PyInterpreterState_Get();
- interp->last_restart_version = interp->monitoring_version + 1;
- interp->monitoring_version = interp->last_restart_version + 1;
- if (instrument_all_executing_code_objects(interp)) {
+ PyThreadState *tstate = _PyThreadState_GET();
+ PyInterpreterState *interp = tstate->interp;
+
+ _PyEval_StopTheWorld(interp);
+ uint32_t restart_version = global_version(interp) + MONITORING_VERSION_INCREMENT;
+ uint32_t new_version = restart_version + MONITORING_VERSION_INCREMENT;
+ if (new_version <= MONITORING_VERSION_INCREMENT) {
+ _PyEval_StartTheWorld(interp);
+ PyErr_Format(PyExc_OverflowError, "events set too many times");
+ return NULL;
+ }
+ interp->last_restart_version = restart_version;
+ set_global_version(tstate, new_version);
+ int res = instrument_all_executing_code_objects(interp);
+ _PyEval_StartTheWorld(interp);
+
+ if (res) {
return NULL;
}
Py_RETURN_NONE;
@@ -2084,7 +2373,7 @@ static PyObject *
monitoring__all_events_impl(PyObject *module)
/*[clinic end generated code: output=6b7581e2dbb690f6 input=62ee9672c17b7f0e]*/
{
- PyInterpreterState *interp = _PyInterpreterState_Get();
+ PyInterpreterState *interp = _PyInterpreterState_GET();
PyObject *res = PyDict_New();
if (res == NULL) {
return NULL;
@@ -2156,18 +2445,22 @@ PyObject *_Py_CreateMonitoringObject(void)
err = PyObject_SetAttrString(events, "NO_EVENTS", _PyLong_GetZero());
if (err) goto error;
PyObject *val = PyLong_FromLong(PY_MONITORING_DEBUGGER_ID);
+ assert(val != NULL); /* Can't return NULL because the int is small. */
err = PyObject_SetAttrString(mod, "DEBUGGER_ID", val);
Py_DECREF(val);
if (err) goto error;
val = PyLong_FromLong(PY_MONITORING_COVERAGE_ID);
+ assert(val != NULL);
err = PyObject_SetAttrString(mod, "COVERAGE_ID", val);
Py_DECREF(val);
if (err) goto error;
val = PyLong_FromLong(PY_MONITORING_PROFILER_ID);
+ assert(val != NULL);
err = PyObject_SetAttrString(mod, "PROFILER_ID", val);
Py_DECREF(val);
if (err) goto error;
val = PyLong_FromLong(PY_MONITORING_OPTIMIZER_ID);
+ assert(val != NULL);
err = PyObject_SetAttrString(mod, "OPTIMIZER_ID", val);
Py_DECREF(val);
if (err) goto error;
@@ -2176,3 +2469,309 @@ error:
Py_DECREF(mod);
return NULL;
}
+
+
+static int
+capi_call_instrumentation(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+ PyObject **args, Py_ssize_t nargs, int event)
+{
+ PyThreadState *tstate = _PyThreadState_GET();
+ PyInterpreterState *interp = tstate->interp;
+
+ uint8_t tools = state->active;
+ assert(args[1] == NULL);
+ args[1] = codelike;
+ if (offset < 0) {
+ PyErr_SetString(PyExc_ValueError, "offset must be non-negative");
+ return -1;
+ }
+ if (event != PY_MONITORING_EVENT_LINE) {
+ PyObject *offset_obj = PyLong_FromLong(offset);
+ if (offset_obj == NULL) {
+ return -1;
+ }
+ assert(args[2] == NULL);
+ args[2] = offset_obj;
+ }
+ size_t nargsf = (size_t) nargs | PY_VECTORCALL_ARGUMENTS_OFFSET;
+ PyObject **callargs = &args[1];
+ int err = 0;
+
+ while (tools) {
+ int tool = most_significant_bit(tools);
+ assert(tool >= 0 && tool < 8);
+ assert(tools & (1 << tool));
+ tools ^= (1 << tool);
+ int res = call_one_instrument(interp, tstate, callargs, nargsf, tool, event);
+ if (res == 0) {
+ /* Nothing to do */
+ }
+ else if (res < 0) {
+ /* error */
+ err = -1;
+ break;
+ }
+ else {
+ /* DISABLE */
+ if (!PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) {
+ PyErr_Format(PyExc_ValueError,
+ "Cannot disable %s events. Callback removed.",
+ event_names[event]);
+ /* Clear tool to prevent infinite loop */
+ Py_CLEAR(interp->monitoring_callables[tool][event]);
+ err = -1;
+ break;
+ }
+ else {
+ state->active &= ~(1 << tool);
+ }
+ }
+ }
+ return err;
+}
+
+int
+PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version,
+ const uint8_t *event_types, Py_ssize_t length)
+{
+ PyInterpreterState *interp = _PyInterpreterState_GET();
+ if (global_version(interp) == *version) {
+ return 0;
+ }
+
+ _Py_GlobalMonitors *m = &interp->monitors;
+ for (Py_ssize_t i = 0; i < length; i++) {
+ int event = event_types[i];
+ state_array[i].active = m->tools[event];
+ }
+ *version = global_version(interp);
+ return 0;
+}
+
+int
+PyMonitoring_ExitScope(void)
+{
+ return 0;
+}
+
+int
+_PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+ assert(state->active);
+ PyObject *args[3] = { NULL, NULL, NULL };
+ return capi_call_instrumentation(state, codelike, offset, args, 2,
+ PY_MONITORING_EVENT_PY_START);
+}
+
+int
+_PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+ assert(state->active);
+ PyObject *args[3] = { NULL, NULL, NULL };
+ return capi_call_instrumentation(state, codelike, offset, args, 2,
+ PY_MONITORING_EVENT_PY_RESUME);
+}
+
+
+
+int
+_PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+ PyObject* retval)
+{
+ assert(state->active);
+ PyObject *args[4] = { NULL, NULL, NULL, retval };
+ return capi_call_instrumentation(state, codelike, offset, args, 3,
+ PY_MONITORING_EVENT_PY_RETURN);
+}
+
+int
+_PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+ PyObject* retval)
+{
+ assert(state->active);
+ PyObject *args[4] = { NULL, NULL, NULL, retval };
+ return capi_call_instrumentation(state, codelike, offset, args, 3,
+ PY_MONITORING_EVENT_PY_YIELD);
+}
+
+int
+_PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+ PyObject* callable, PyObject *arg0)
+{
+ assert(state->active);
+ PyObject *args[5] = { NULL, NULL, NULL, callable, arg0 };
+ return capi_call_instrumentation(state, codelike, offset, args, 4,
+ PY_MONITORING_EVENT_CALL);
+}
+
+int
+_PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+ int lineno)
+{
+ assert(state->active);
+ PyObject *lno = PyLong_FromLong(lineno);
+ if (lno == NULL) {
+ return -1;
+ }
+ PyObject *args[3] = { NULL, NULL, lno };
+ int res= capi_call_instrumentation(state, codelike, offset, args, 2,
+ PY_MONITORING_EVENT_LINE);
+ Py_DECREF(lno);
+ return res;
+}
+
+int
+_PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+ PyObject *target_offset)
+{
+ assert(state->active);
+ PyObject *args[4] = { NULL, NULL, NULL, target_offset };
+ return capi_call_instrumentation(state, codelike, offset, args, 3,
+ PY_MONITORING_EVENT_JUMP);
+}
+
+int
+_PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+ PyObject *target_offset)
+{
+ assert(state->active);
+ PyObject *args[4] = { NULL, NULL, NULL, target_offset };
+ return capi_call_instrumentation(state, codelike, offset, args, 3,
+ PY_MONITORING_EVENT_BRANCH);
+}
+
+int
+_PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+ PyObject *retval)
+{
+ assert(state->active);
+ PyObject *args[4] = { NULL, NULL, NULL, retval };
+ return capi_call_instrumentation(state, codelike, offset, args, 3,
+ PY_MONITORING_EVENT_C_RETURN);
+}
+
+static inline int
+exception_event_setup(PyObject **exc, int event) {
+ *exc = PyErr_GetRaisedException();
+ if (*exc == NULL) {
+ PyErr_Format(PyExc_ValueError,
+ "Firing event %d with no exception set",
+ event);
+ return -1;
+ }
+ return 0;
+}
+
+
+static inline int
+exception_event_teardown(int err, PyObject *exc) {
+ if (err == 0) {
+ PyErr_SetRaisedException(exc);
+ }
+ else {
+ assert(PyErr_Occurred());
+ Py_XDECREF(exc);
+ }
+ return err;
+}
+
+int
+_PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+ int event = PY_MONITORING_EVENT_PY_THROW;
+ assert(state->active);
+ PyObject *exc;
+ if (exception_event_setup(&exc, event) < 0) {
+ return -1;
+ }
+ PyObject *args[4] = { NULL, NULL, NULL, exc };
+ int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
+ return exception_event_teardown(err, exc);
+}
+
+int
+_PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+ int event = PY_MONITORING_EVENT_RAISE;
+ assert(state->active);
+ PyObject *exc;
+ if (exception_event_setup(&exc, event) < 0) {
+ return -1;
+ }
+ PyObject *args[4] = { NULL, NULL, NULL, exc };
+ int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
+ return exception_event_teardown(err, exc);
+}
+
+int
+_PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+ int event = PY_MONITORING_EVENT_C_RAISE;
+ assert(state->active);
+ PyObject *exc;
+ if (exception_event_setup(&exc, event) < 0) {
+ return -1;
+ }
+ PyObject *args[4] = { NULL, NULL, NULL, exc };
+ int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
+ return exception_event_teardown(err, exc);
+}
+
+int
+_PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+ int event = PY_MONITORING_EVENT_RERAISE;
+ assert(state->active);
+ PyObject *exc;
+ if (exception_event_setup(&exc, event) < 0) {
+ return -1;
+ }
+ PyObject *args[4] = { NULL, NULL, NULL, exc };
+ int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
+ return exception_event_teardown(err, exc);
+}
+
+int
+_PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+ int event = PY_MONITORING_EVENT_EXCEPTION_HANDLED;
+ assert(state->active);
+ PyObject *exc;
+ if (exception_event_setup(&exc, event) < 0) {
+ return -1;
+ }
+ PyObject *args[4] = { NULL, NULL, NULL, exc };
+ int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
+ return exception_event_teardown(err, exc);
+}
+
+int
+_PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+ int event = PY_MONITORING_EVENT_PY_UNWIND;
+ assert(state->active);
+ PyObject *exc;
+ if (exception_event_setup(&exc, event) < 0) {
+ return -1;
+ }
+ PyObject *args[4] = { NULL, NULL, NULL, exc };
+ int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
+ return exception_event_teardown(err, exc);
+}
+
+int
+_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value)
+{
+ int event = PY_MONITORING_EVENT_STOP_ITERATION;
+ assert(state->active);
+ assert(!PyErr_Occurred());
+ PyErr_SetObject(PyExc_StopIteration, value);
+ PyObject *exc;
+ if (exception_event_setup(&exc, event) < 0) {
+ return -1;
+ }
+ PyObject *args[4] = { NULL, NULL, NULL, exc };
+ int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
+ Py_DECREF(exc);
+ return exception_event_teardown(err, NULL);
+}
generated by cgit v1.3 (git 2.53.0) at 2026-06-09 00:29:18 +0000