#include "Python.h"

#include "pycore_ceval.h" // _PyPerf_Callbacks

#include "pycore_frame.h"

#include "pycore_interp.h"

#include "pycore_mmap.h" // _PyAnnotateMemoryMap()

#include "pycore_jit_unwind.h"

#include "pycore_runtime.h" // _PyRuntime

#ifdef PY_HAVE_PERF_TRAMPOLINE

#if defined(__linux__)

# include <elf.h> // ELF architecture constants

#include <fcntl.h> // File control operations

#include <stdio.h> // Standard I/O operations

#include <stdlib.h> // Standard library functions

#include <string.h> // memcpy, strlen

#include <sys/mman.h> // Memory mapping functions (mmap)

#include <sys/types.h> // System data types

#include <unistd.h> // System calls (sysconf, getpid)

#include <sys/time.h> // Time functions (gettimeofday)

#if defined(__linux__)

# include <sys/syscall.h> // System call interface

#if defined(__APPLE__)

# include <mach/mach_time.h> // mach_absolute_time, mach_timebase_info

#if !defined(__linux__)

# if defined(__i386__) || defined(_M_IX86)

# define EM_386 3

# elif defined(__arm__) || defined(_M_ARM)

# define EM_ARM 40

# elif defined(__x86_64__) || defined(_M_X64)

# define EM_X86_64 62

# elif defined(__aarch64__)

# define EM_AARCH64 183

# elif defined(__riscv)

# define EM_RISCV 243

#define trampoline_api _PyRuntime.ceval.perf.trampoline_api

typedef uint64_t uword; // Word-sized unsigned integer

typedef const char* CodeComments; // Code comment strings

#define MB (1024 * 1024) // 1 Megabyte for buffer sizing

static uint64_t GetElfMachineArchitecture(void) {

#if defined(__x86_64__) || defined(_M_X64)

return EM_X86_64;

#elif defined(__i386__) || defined(_M_IX86)

return EM_386;

#elif defined(__aarch64__)

return EM_AARCH64;

#elif defined(__arm__) || defined(_M_ARM)

return EM_ARM;

#elif defined(__riscv)

return EM_RISCV;

Py_UNREACHABLE(); // Unsupported architecture - should never reach here

return 0;

}

typedef struct {

uint32_t magic; // Magic number (0x4A695444 = "JiTD")

uint32_t version; // Jitdump format version (currently 1)

uint32_t size; // Size of this header structure

uint32_t elf_mach_target; // Target architecture (from GetElfMachineArchitecture)

uint32_t reserved; // Reserved field (must be 0)

uint32_t process_id; // Process ID of the JIT compiler

uint64_t time_stamp; // Timestamp when jitdump was created

uint64_t flags; // Feature flags (currently unused)

} Header;

enum PerfEvent {

};

struct BaseEvent {

};

typedef struct {

struct BaseEvent base; // Common event header

uint32_t process_id; // Process ID where code was generated

uint32_t thread_id; // Thread ID where code was generated

uint64_t vma; // Virtual memory address where code is loaded

uint64_t code_address; // Address of the actual machine code

uint64_t code_size; // Size of the machine code in bytes

uint64_t code_id; // Unique identifier for this code region

/* Followed by:

} CodeLoadEvent;

typedef struct {

struct BaseEvent base; // Common event header

uint64_t unwind_data_size; // Size of the unwinding data

uint64_t eh_frame_hdr_size; // Size of the EH frame header

uint64_t mapped_size; // Total mapped size (with padding)

/* Followed by:

} CodeUnwindingInfoEvent;

typedef struct __attribute__((packed)) {

uint8_t version;

uint8_t eh_frame_ptr_enc;

uint8_t fde_count_enc;

uint8_t table_enc;

int32_t eh_frame_ptr;

uint32_t eh_fde_count;

int32_t from;

int32_t to;

} EhFrameHeader;

_Static_assert(sizeof(EhFrameHeader) == 20, "EhFrameHeader layout mismatch");

typedef struct {

FILE* perf_map; // File handle for the jitdump file

PyMutex map_lock; // Thread synchronization lock

void* mapped_buffer; // Memory-mapped region (signals perf we're active)

size_t mapped_size; // Size of the mapped region

uint32_t code_id; // Counter for unique code region identifiers

uint64_t build_id_salt; // Per-process salt for unique synthetic DSOs

} PerfMapJitState;

static PerfMapJitState perf_jit_map_state;

#if !defined(__APPLE__)

static const intptr_t nanoseconds_per_second = 1000000000;

static int64_t get_current_monotonic_ticks(void) {

#if defined(__APPLE__)

// On macOS the jitdump file is consumed by profilers (such as samply) that

// timestamp their samples using mach_absolute_time(). The jitdump event

// timestamps must use the same clock domain, otherwise the JIT code

// mappings cannot be lined up with the samples.

static mach_timebase_info_data_t timebase = {0, 0};

if (timebase.denom == 0) {

(void)mach_timebase_info(&timebase);

}

uint64_t ticks = mach_absolute_time();

return (int64_t)(ticks * timebase.numer / timebase.denom);

struct timespec ts;

if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) {

Py_UNREACHABLE(); // Should never fail on supported systems

return 0;

}

/* Convert to nanoseconds for maximum precision */

int64_t result = ts.tv_sec;

result *= nanoseconds_per_second;

result += ts.tv_nsec;

return result;

}

static int64_t get_current_time_microseconds(void) {

struct timeval tv;

if (gettimeofday(&tv, NULL) < 0) {

Py_UNREACHABLE(); // Should never fail on supported systems

return 0;

}

return ((int64_t)(tv.tv_sec) * 1000000) + tv.tv_usec;

}

static void perf_map_jit_write_fully(const void* buffer, size_t size) {

FILE* out_file = perf_jit_map_state.perf_map;

const char* ptr = (const char*)(buffer);

while (size > 0) {

const size_t written = fwrite(ptr, 1, size, out_file);

if (written == 0) {

Py_UNREACHABLE(); // Write failure - should be very rare

break;

}

size -= written;

ptr += written;

}

}

static void perf_map_jit_write_header(int pid, FILE* out_file) {

Header header;

/* Initialize header with required values */

header.magic = 0x4A695444; // "JiTD" magic number

header.version = 1; // Current jitdump version

header.size = sizeof(Header); // Header size for validation

header.elf_mach_target = GetElfMachineArchitecture(); // Target architecture

header.reserved = 0; // padding reserved for future use

header.process_id = pid; // Process identifier

header.time_stamp = get_current_time_microseconds(); // Creation time

header.flags = 0; // No special flags currently used

perf_map_jit_write_fully(&header, sizeof(header));

}

static void* perf_map_jit_init(void) {

PyMutex_Lock(&perf_jit_map_state.map_lock);

if (perf_jit_map_state.perf_map != NULL) {

PyMutex_Unlock(&perf_jit_map_state.map_lock);

return &perf_jit_map_state;

}

char filename[100];

int pid = getpid();

/* Create unique filename based on process ID */

snprintf(filename, sizeof(filename) - 1, "/tmp/jit-%d.dump", pid);

/* Create/open the jitdump file with appropriate permissions */

const int fd = open(filename, O_CREAT | O_TRUNC | O_RDWR, 0666);

if (fd == -1) {

PyMutex_Unlock(&perf_jit_map_state.map_lock);

return NULL; // Failed to create file

}

/* Get system page size for memory mapping */

const long page_size = sysconf(_SC_PAGESIZE);

if (page_size == -1) {

close(fd);

PyMutex_Unlock(&perf_jit_map_state.map_lock);

return NULL; // Failed to get page size

}

#if defined(__APPLE__)

// On macOS, samply uses a preload to find jitdumps and this mmap can be slow.

perf_jit_map_state.mapped_buffer = NULL;

/*

perf_jit_map_state.mapped_buffer = mmap(

NULL, // Let kernel choose address

page_size, // Map one page

PROT_READ | PROT_EXEC, // Read and execute permissions (required by perf)

MAP_PRIVATE, // Private mapping

fd, // File descriptor

0 // Offset 0 (first page)

);

if (perf_jit_map_state.mapped_buffer == MAP_FAILED) {

perf_jit_map_state.mapped_buffer = NULL;

close(fd);

PyMutex_Unlock(&perf_jit_map_state.map_lock);

return NULL; // Memory mapping failed

}

(void)_PyAnnotateMemoryMap(perf_jit_map_state.mapped_buffer, page_size,

"cpython:perf_jit_trampoline");

perf_jit_map_state.mapped_size = page_size;

/* Convert file descriptor to FILE* for easier I/O operations */

perf_jit_map_state.perf_map = fdopen(fd, "w+");

if (perf_jit_map_state.perf_map == NULL) {

close(fd);

PyMutex_Unlock(&perf_jit_map_state.map_lock);

return NULL; // Failed to create FILE*

}

/*

setvbuf(perf_jit_map_state.perf_map, NULL, _IOFBF, 2 * MB);

/* Write the jitdump file header */

perf_map_jit_write_header(pid, perf_jit_map_state.perf_map);

/* Initialize code ID counter */

perf_jit_map_state.code_id = 0;

perf_jit_map_state.build_id_salt =

((uint64_t)pid << 32) ^ (uint64_t)get_current_monotonic_ticks();

/* Calculate padding size based on actual unwind info requirements */

size_t eh_frame_size = _PyJitUnwind_EhFrameSize(0);

size_t unwind_data_size = sizeof(EhFrameHeader) + eh_frame_size;

trampoline_api.code_padding = _Py_SIZE_ROUND_UP(unwind_data_size, 16);

trampoline_api.code_alignment = 32;

PyMutex_Unlock(&perf_jit_map_state.map_lock);

return &perf_jit_map_state;

}

static void perf_map_jit_write_entry_with_name(

)

{

/* Initialize jitdump system on first use */

void* ret = perf_map_jit_init();

if (ret == NULL) {

return; // Initialization failed, silently abort

}

if (entry == NULL) {

entry = "";

}

if (filename == NULL) {

filename = "";

}

/*

size_t perf_map_entry_size = snprintf(NULL, 0, "py::%s:%s", entry, filename) + 1;

char* perf_map_entry = (char*) PyMem_RawMalloc(perf_map_entry_size);

if (perf_map_entry == NULL) {

return; // Memory allocation failed

}

snprintf(perf_map_entry, perf_map_entry_size, "py::%s:%s", entry, filename);

const size_t name_length = strlen(perf_map_entry);

uword base = (uword)code_addr;

uword size = code_size;

/*

uint8_t buffer[1024]; // Buffer for DWARF data (1KB should be sufficient)

size_t eh_frame_size = _PyJitUnwind_BuildEhFrame(

buffer, sizeof(buffer), code_addr, code_size, 0);

if (eh_frame_size == 0) {

PyMem_RawFree(perf_map_entry);

return;

}

/*

PyMutex_Lock(&perf_jit_map_state.map_lock);

/*

CodeUnwindingInfoEvent ev2;

ev2.base.event = PerfUnwindingInfo;

ev2.base.time_stamp = get_current_monotonic_ticks();

ev2.unwind_data_size = sizeof(EhFrameHeader) + eh_frame_size;

/* Verify we don't exceed our padding budget */

assert(ev2.unwind_data_size <= (uint64_t)trampoline_api.code_padding);

ev2.eh_frame_hdr_size = sizeof(EhFrameHeader);

ev2.mapped_size = _Py_SIZE_ROUND_UP(ev2.unwind_data_size, 16); // 16-byte alignment

/* Calculate total event size with padding */

int content_size = (int)(sizeof(ev2) + sizeof(EhFrameHeader) + eh_frame_size);

int padding_size = (int)_Py_SIZE_ROUND_UP((size_t)content_size, 8) - content_size; // 8-byte align

ev2.base.size = (uint32_t)(content_size + padding_size);

/* Write the unwinding info event header */

perf_map_jit_write_fully(&ev2, sizeof(ev2));

/*

EhFrameHeader f;

f.version = 1;

f.eh_frame_ptr_enc = DWRF_EH_PE_sdata4 | DWRF_EH_PE_pcrel;

f.fde_count_enc = DWRF_EH_PE_udata4;

f.table_enc = DWRF_EH_PE_sdata4 | DWRF_EH_PE_datarel;

/* Calculate relative offsets for EH frame navigation */

f.eh_frame_ptr = -(int32_t)(eh_frame_size + 4 * sizeof(unsigned char));

f.eh_fde_count = 1; // We generate exactly one FDE per function

f.from = -(int32_t)(_Py_SIZE_ROUND_UP(code_size, 8) + eh_frame_size);

uint32_t cie_payload_size;

memcpy(&cie_payload_size, buffer, sizeof(cie_payload_size));

int cie_size = (int)(sizeof(cie_payload_size) + cie_payload_size);

f.to = -(int32_t)(eh_frame_size - cie_size);

/* Write EH frame data and header */

perf_map_jit_write_fully(buffer, eh_frame_size);

perf_map_jit_write_fully(&f, sizeof(f));

/* Write padding to maintain alignment */

char padding_bytes[] = "\0\0\0\0\0\0\0\0";

perf_map_jit_write_fully(&padding_bytes, padding_size);

/*

CodeLoadEvent ev;

ev.base.event = PerfLoad;

ev.base.size = sizeof(ev) + (name_length+1) + size;

ev.base.time_stamp = get_current_monotonic_ticks();

ev.process_id = getpid();

#if defined(__APPLE__)

// The jitdump format defines the thread id field as a 32-bit value, but

// pthread_threadid_np() returns a 64-bit id. Truncate it to 32 bits to

// keep the record layout identical to other platforms.

uint64_t thread_id = 0;

pthread_threadid_np(NULL, &thread_id);

ev.thread_id = (uint32_t)thread_id;

#elif defined(HAVE_GETTID)

ev.thread_id = gettid();

ev.thread_id = syscall(SYS_gettid); // Get thread ID via system call

ev.vma = base; // Virtual memory address

ev.code_address = base; // Same as VMA for our use case

ev.code_size = size;

/* Assign unique code ID and increment counter */

perf_jit_map_state.code_id += 1;

ev.code_id = perf_jit_map_state.code_id;

/* Write code load event and associated data */

perf_map_jit_write_fully(&ev, sizeof(ev));

perf_map_jit_write_fully(perf_map_entry, name_length+1); // Include null terminator

/*

uint64_t marker = perf_jit_map_state.build_id_salt ^

((uint64_t)perf_jit_map_state.code_id << 32) ^

(uint64_t)code_size;

if (size >= sizeof(marker)) {

size_t prefix = size - sizeof(marker);

perf_map_jit_write_fully((void *)(base), prefix);

perf_map_jit_write_fully(&marker, sizeof(marker));

}

else if (size > 0) {

uint8_t tmp[sizeof(marker)];

memcpy(tmp, (void *)(base), size);

for (size_t i = 0; i < size; i++) {

tmp[i] ^= (uint8_t)(marker >> (i * 8));

}

perf_map_jit_write_fully(tmp, size);

}

/* Clean up allocated memory */

PyMutex_Unlock(&perf_jit_map_state.map_lock);

PyMem_RawFree(perf_map_entry);

}

static void perf_map_jit_write_entry(void *state, const void *code_addr,

size_t code_size, PyCodeObject *co)

{

const char *entry = "";

const char *filename = "";

if (co != NULL) {

if (co->co_qualname != NULL) {

entry = PyUnicode_AsUTF8(co->co_qualname);

}

if (co->co_filename != NULL) {

filename = PyUnicode_AsUTF8(co->co_filename);

}

}

perf_map_jit_write_entry_with_name(state, code_addr, code_size,

entry, filename);

}

_PyPerfJit_WriteNamedCode(const void *code_addr, size_t code_size,

const char *entry, const char *filename)

{

perf_map_jit_write_entry_with_name(

NULL, code_addr, code_size, entry, filename);

}

static int perf_map_jit_fini(void* state) {

/*

PyMutex_Lock(&perf_jit_map_state.map_lock);

if (perf_jit_map_state.perf_map != NULL) {

fclose(perf_jit_map_state.perf_map); // This also flushes buffers

perf_jit_map_state.perf_map = NULL;

}

PyMutex_Unlock(&perf_jit_map_state.map_lock);

/*

if (perf_jit_map_state.mapped_buffer != NULL) {

munmap(perf_jit_map_state.mapped_buffer, perf_jit_map_state.mapped_size);

perf_jit_map_state.mapped_buffer = NULL;

}

/* Clear global state reference */

trampoline_api.state = NULL;

return 0; // Success

}

_PyPerf_Callbacks _Py_perfmap_jit_callbacks = {

&perf_map_jit_init, // Initialization function

&perf_map_jit_write_entry, // Event writing function

&perf_map_jit_fini, // Cleanup function

};

#endif /* PY_HAVE_PERF_TRAMPOLINE */