perf_jit_trampoline.c

#include"Python.h"
#include"pycore_ceval.h"// _PyPerf_Callbacks
#include"pycore_frame.h"
#include"pycore_interp.h"


#ifdefPY_HAVE_PERF_TRAMPOLINE

#include<fcntl.h>
#include<stdio.h>
#include<stdlib.h>
#include<sys/mman.h>// mmap()
#include<sys/types.h>
#include<unistd.h>// sysconf()
#include<sys/time.h>// gettimeofday()
#include<sys/syscall.h>

// ----------------------------------
// Perf jitdump API
// ----------------------------------

typedefstruct {
FILE*perf_map;
PyThread_type_lockmap_lock;
void*mapped_buffer;
size_tmapped_size;
intcode_id;
} PerfMapJitState;

staticPerfMapJitStateperf_jit_map_state;

/*
Usually the binary and libraries are mapped in separate region like below:

 address ->
 --+---------------------+--//--+---------------------+--
 | .text | .data | ... | | .text | .data | ... |
 --+---------------------+--//--+---------------------+--
 myprog libc.so

So it'd be easy and straight-forward to find a mapped binary or library from an
address.

But for JIT code, the code arena only cares about the code section. But the
resulting DSOs (which is generated by perf inject -j) contain ELF headers and
unwind info too. Then it'd generate following address space with synthesized
MMAP events. Let's say it has a sample between address B and C.

 sample
 |
 address -> A B v C
 ---------------------------------------------------------------------------------------------------
 /tmp/jitted-PID-0.so | (headers) | .text | unwind info |
 /tmp/jitted-PID-1.so | (headers) | .text | unwind info |
 /tmp/jitted-PID-2.so | (headers) | .text | unwind info |
 ...
 ---------------------------------------------------------------------------------------------------

If it only maps the .text section, it'd find the jitted-PID-1.so but cannot see
the unwind info. If it maps both .text section and unwind sections, the sample
could be mapped to either jitted-PID-0.so or jitted-PID-1.so and it's confusing
which one is right. So to make perf happy we have non-overlapping ranges for each
DSO:

 address ->
 -------------------------------------------------------------------------------------------------------
 /tmp/jitted-PID-0.so | (headers) | .text | unwind info |
 /tmp/jitted-PID-1.so | (headers) | .text | unwind info |
 /tmp/jitted-PID-2.so | (headers) | .text | unwind info |
 ...
 -------------------------------------------------------------------------------------------------------

As the trampolines are constant, we add a constant padding but in general the padding needs to have the
size of the unwind info rounded to 16 bytes. In general, for our trampolines this is 0x50
 */

#definePERF_JIT_CODE_PADDING 0x100
#definetrampoline_api _PyRuntime.ceval.perf.trampoline_api

typedefuint64_tuword;
typedefconstchar*CodeComments;

#definePd "d"
#defineMB (1024 * 1024)

#defineEM_386 3
#defineEM_X86_64 62
#defineEM_ARM 40
#defineEM_AARCH64 183
#defineEM_RISCV 243

#defineTARGET_ARCH_IA32 0
#defineTARGET_ARCH_X64 0
#defineTARGET_ARCH_ARM 0
#defineTARGET_ARCH_ARM64 0
#defineTARGET_ARCH_RISCV32 0
#defineTARGET_ARCH_RISCV64 0

#defineFLAG_generate_perf_jitdump 0
#defineFLAG_write_protect_code 0
#defineFLAG_write_protect_vm_isolate 0
#defineFLAG_code_comments 0

#defineUNREACHABLE()

staticuwordGetElfMachineArchitecture(void) {
#ifTARGET_ARCH_IA32
returnEM_386;
#elifTARGET_ARCH_X64
returnEM_X86_64;
#elifTARGET_ARCH_ARM
returnEM_ARM;
#elifTARGET_ARCH_ARM64
returnEM_AARCH64;
#elifTARGET_ARCH_RISCV32||TARGET_ARCH_RISCV64
returnEM_RISCV;
#else
UNREACHABLE();
return0;
#endif
}

typedefstruct {
uint32_tmagic;
uint32_tversion;
uint32_tsize;
uint32_telf_mach_target;
uint32_treserved;
uint32_tprocess_id;
uint64_ttime_stamp;
uint64_tflags;
} Header;

enumPerfEvent {
PerfLoad=0,
PerfMove=1,
PerfDebugInfo=2,
PerfClose=3,
PerfUnwindingInfo=4
};

structBaseEvent {
uint32_tevent;
uint32_tsize;
uint64_ttime_stamp;
 };

typedefstruct {
structBaseEventbase;
uint32_tprocess_id;
uint32_tthread_id;
uint64_tvma;
uint64_tcode_address;
uint64_tcode_size;
uint64_tcode_id;
} CodeLoadEvent;

typedefstruct {
structBaseEventbase;
uint64_tunwind_data_size;
uint64_teh_frame_hdr_size;
uint64_tmapped_size;
} CodeUnwindingInfoEvent;

staticconstintptr_tnanoseconds_per_second=1000000000;

// Dwarf encoding constants

staticconstuint8_tDwarfUData4=0x03;
staticconstuint8_tDwarfSData4=0x0b;
staticconstuint8_tDwarfPcRel=0x10;
staticconstuint8_tDwarfDataRel=0x30;
// static uint8_t DwarfOmit = 0xff;
typedefstruct {
unsigned charversion;
unsigned chareh_frame_ptr_enc;
unsigned charfde_count_enc;
unsigned chartable_enc;
int32_teh_frame_ptr;
int32_teh_fde_count;
int32_tfrom;
int32_tto;
} EhFrameHeader;

staticint64_tget_current_monotonic_ticks(void) {
structtimespects;
if (clock_gettime(CLOCK_MONOTONIC, &ts) !=0) {
UNREACHABLE();
return0;
 }
// Convert to nanoseconds.
int64_tresult=ts.tv_sec;
result *= nanoseconds_per_second;
result+=ts.tv_nsec;
returnresult;
}

staticint64_tget_current_time_microseconds(void) {
// gettimeofday has microsecond resolution.
structtimevaltv;
if (gettimeofday(&tv, NULL) <0) {
UNREACHABLE();
return0;
 }
return ((int64_t)(tv.tv_sec) *1000000) +tv.tv_usec;
}


staticsize_tround_up(int64_tvalue, int64_tmultiple) {
if (multiple==0) {
// Avoid division by zero
returnvalue;
 }

int64_tremainder=value % multiple;
if (remainder==0) {
// Value is already a multiple of 'multiple'
returnvalue;
 }

// Calculate the difference to the next multiple
int64_tdifference=multiple-remainder;

// Add the difference to the value
int64_trounded_up_value=value+difference;

returnrounded_up_value;
}


staticvoidperf_map_jit_write_fully(constvoid*buffer, size_tsize) {
FILE*out_file=perf_jit_map_state.perf_map;
constchar*ptr= (constchar*)(buffer);
while (size>0) {
constsize_twritten=fwrite(ptr, 1, size, out_file);
if (written==0) {
UNREACHABLE();
break;
 }
size-=written;
ptr+=written;
 }
}

staticvoidperf_map_jit_write_header(intpid, FILE*out_file) {
Headerheader;
header.magic=0x4A695444;
header.version=1;
header.size=sizeof(Header);
header.elf_mach_target=GetElfMachineArchitecture();
header.process_id=pid;
header.time_stamp=get_current_time_microseconds();
header.flags=0;
perf_map_jit_write_fully(&header, sizeof(header));
}

staticvoid*perf_map_jit_init(void) {
charfilename[100];
intpid=getpid();
snprintf(filename, sizeof(filename) -1, "/tmp/jit-%d.dump", pid);
constintfd=open(filename, O_CREAT | O_TRUNC | O_RDWR, 0666);
if (fd==-1) {
returnNULL;
 }

constlongpage_size=sysconf(_SC_PAGESIZE); // NOLINT(runtime/int)
if (page_size==-1) {
close(fd);
returnNULL;
 }

// The perf jit interface forces us to map the first page of the file
// to signal that we are using the interface.
perf_jit_map_state.mapped_buffer=mmap(NULL, page_size, PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, 0);
if (perf_jit_map_state.mapped_buffer==NULL) {
close(fd);
returnNULL;
 }
perf_jit_map_state.mapped_size=page_size;
perf_jit_map_state.perf_map=fdopen(fd, "w+");
if (perf_jit_map_state.perf_map==NULL) {
close(fd);
returnNULL;
 }
setvbuf(perf_jit_map_state.perf_map, NULL, _IOFBF, 2*MB);
perf_map_jit_write_header(pid, perf_jit_map_state.perf_map);

perf_jit_map_state.map_lock=PyThread_allocate_lock();
if (perf_jit_map_state.map_lock==NULL) {
fclose(perf_jit_map_state.perf_map);
returnNULL;
 }
perf_jit_map_state.code_id=0;

trampoline_api.code_padding=PERF_JIT_CODE_PADDING;
return&perf_jit_map_state;
}

/* DWARF definitions. */

#defineDWRF_CIE_VERSION 1

enum {
DWRF_CFA_nop=0x0,
DWRF_CFA_offset_extended=0x5,
DWRF_CFA_def_cfa=0xc,
DWRF_CFA_def_cfa_offset=0xe,
DWRF_CFA_offset_extended_sf=0x11,
DWRF_CFA_advance_loc=0x40,
DWRF_CFA_offset=0x80
};

enum
 {
DWRF_EH_PE_absptr=0x00,
DWRF_EH_PE_omit=0xff,

/* FDE data encoding. */
DWRF_EH_PE_uleb128=0x01,
DWRF_EH_PE_udata2=0x02,
DWRF_EH_PE_udata4=0x03,
DWRF_EH_PE_udata8=0x04,
DWRF_EH_PE_sleb128=0x09,
DWRF_EH_PE_sdata2=0x0a,
DWRF_EH_PE_sdata4=0x0b,
DWRF_EH_PE_sdata8=0x0c,
DWRF_EH_PE_signed=0x08,

/* FDE flags. */
DWRF_EH_PE_pcrel=0x10,
DWRF_EH_PE_textrel=0x20,
DWRF_EH_PE_datarel=0x30,
DWRF_EH_PE_funcrel=0x40,
DWRF_EH_PE_aligned=0x50,

DWRF_EH_PE_indirect=0x80
 };

enum { DWRF_TAG_compile_unit=0x11 };

enum { DWRF_children_no=0, DWRF_children_yes=1 };

enum { DWRF_AT_name=0x03, DWRF_AT_stmt_list=0x10, DWRF_AT_low_pc=0x11, DWRF_AT_high_pc=0x12 };

enum { DWRF_FORM_addr=0x01, DWRF_FORM_data4=0x06, DWRF_FORM_string=0x08 };

enum { DWRF_LNS_extended_op=0, DWRF_LNS_copy=1, DWRF_LNS_advance_pc=2, DWRF_LNS_advance_line=3 };

enum { DWRF_LNE_end_sequence=1, DWRF_LNE_set_address=2 };

enum {
#ifdef__x86_64__
/* Yes, the order is strange, but correct. */
DWRF_REG_AX,
DWRF_REG_DX,
DWRF_REG_CX,
DWRF_REG_BX,
DWRF_REG_SI,
DWRF_REG_DI,
DWRF_REG_BP,
DWRF_REG_SP,
DWRF_REG_8,
DWRF_REG_9,
DWRF_REG_10,
DWRF_REG_11,
DWRF_REG_12,
DWRF_REG_13,
DWRF_REG_14,
DWRF_REG_15,
DWRF_REG_RA,
#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__)
DWRF_REG_SP=31,
DWRF_REG_RA=30,
#else
# error "Unsupported target architecture"
#endif
};

typedefstructELFObjectContext
{
uint8_t*p; /* Pointer to next address in obj.space. */
uint8_t*startp; /* Pointer to start address in obj.space. */
uint8_t*eh_frame_p; /* Pointer to start address in obj.space. */
uint32_tcode_size; /* Size of machine code. */
} ELFObjectContext;

/* Append a null-terminated string. */
staticuint32_t
elfctx_append_string(ELFObjectContext*ctx, constchar*str)
{
uint8_t*p=ctx->p;
uint32_tofs= (uint32_t)(p-ctx->startp);
do {
*p++= (uint8_t)*str;
 } while (*str++);
ctx->p=p;
returnofs;
}

/* Append a SLEB128 value. */
staticvoid
elfctx_append_sleb128(ELFObjectContext*ctx, int32_tv)
{
uint8_t*p=ctx->p;
for (; (uint32_t)(v+0x40) >= 0x80; v >>= 7) {
*p++= (uint8_t)((v&0x7f) | 0x80);
 }
*p++= (uint8_t)(v&0x7f);
ctx->p=p;
}

/* Append a ULEB128 to buffer. */
staticvoid
elfctx_append_uleb128(ELFObjectContext*ctx, uint32_tv)
{
uint8_t*p=ctx->p;
for (; v >= 0x80; v >>= 7) {
*p++= (char)((v&0x7f) | 0x80);
 }
*p++= (char)v;
ctx->p=p;
}

/* Shortcuts to generate DWARF structures. */
#defineDWRF_U8(x) (*p++ = (x))
#defineDWRF_I8(x) (*(int8_t*)p = (x), p++)
#defineDWRF_U16(x) (*(uint16_t*)p = (x), p += 2)
#defineDWRF_U32(x) (*(uint32_t*)p = (x), p += 4)
#defineDWRF_ADDR(x) (*(uintptr_t*)p = (x), p += sizeof(uintptr_t))
#defineDWRF_UV(x) (ctx->p = p, elfctx_append_uleb128(ctx, (x)), p = ctx->p)
#defineDWRF_SV(x) (ctx->p = p, elfctx_append_sleb128(ctx, (x)), p = ctx->p)
#defineDWRF_STR(str) (ctx->p = p, elfctx_append_string(ctx, (str)), p = ctx->p)
#defineDWRF_ALIGNNOP(s) \
 while ((uintptr_t)p & ((s)-1)) { \
 *p++ = DWRF_CFA_nop; \
 }
#defineDWRF_SECTION(name, stmt) \
 { \
 uint32_t* szp_##name = (uint32_t*)p; \
 p += 4; \
 stmt; \
 *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4); \
 }

/* Initialize .eh_frame section. */
staticvoid
elf_init_ehframe(ELFObjectContext*ctx)
{
uint8_t*p=ctx->p;
uint8_t*framep=p;

/* Emit DWARF EH CIE. */
DWRF_SECTION(CIE, DWRF_U32(0); /* Offset to CIE itself. */
DWRF_U8(DWRF_CIE_VERSION);
DWRF_STR("zR"); /* Augmentation. */
DWRF_UV(1); /* Code alignment factor. */
DWRF_SV(-(int64_t)sizeof(uintptr_t)); /* Data alignment factor. */
DWRF_U8(DWRF_REG_RA); /* Return address register. */
DWRF_UV(1);
DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); /* Augmentation data. */
DWRF_U8(DWRF_CFA_def_cfa); DWRF_UV(DWRF_REG_SP); DWRF_UV(sizeof(uintptr_t));
DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); DWRF_UV(1);
DWRF_ALIGNNOP(sizeof(uintptr_t));
 )

ctx->eh_frame_p=p;

/* Emit DWARF EH FDE. */
DWRF_SECTION(FDE, DWRF_U32((uint32_t)(p-framep)); /* Offset to CIE. */
DWRF_U32(-0x30); /* Machine code offset relative to .text. */
DWRF_U32(ctx->code_size); /* Machine code length. */
DWRF_U8(0); /* Augmentation data. */
/* Registers saved in CFRAME. */
#ifdef__x86_64__
DWRF_U8(DWRF_CFA_advance_loc | 4);
DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16);
DWRF_U8(DWRF_CFA_advance_loc | 6);
DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(8);
/* Extra registers saved for JIT-compiled code. */
#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__)
DWRF_U8(DWRF_CFA_advance_loc | 1);
DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16);
DWRF_U8(DWRF_CFA_offset | 29); DWRF_UV(2);
DWRF_U8(DWRF_CFA_offset | 30); DWRF_UV(1);
DWRF_U8(DWRF_CFA_advance_loc | 3);
DWRF_U8(DWRF_CFA_offset | -(64-29));
DWRF_U8(DWRF_CFA_offset | -(64-30));
DWRF_U8(DWRF_CFA_def_cfa_offset);
DWRF_UV(0);
#else
# error "Unsupported target architecture"
#endif
DWRF_ALIGNNOP(sizeof(uintptr_t));)

ctx->p=p;
}

staticvoidperf_map_jit_write_entry(void*state, constvoid*code_addr,
unsigned intcode_size, PyCodeObject*co)
{

if (perf_jit_map_state.perf_map==NULL) {
void*ret=perf_map_jit_init();
if(ret==NULL){
return;
 }
 }

constchar*entry="";
if (co->co_qualname!=NULL) {
entry=PyUnicode_AsUTF8(co->co_qualname);
 }
constchar*filename="";
if (co->co_filename!=NULL) {
filename=PyUnicode_AsUTF8(co->co_filename);
 }


size_tperf_map_entry_size=snprintf(NULL, 0, "py::%s:%s", entry, filename) +1;
char*perf_map_entry= (char*) PyMem_RawMalloc(perf_map_entry_size);
if (perf_map_entry==NULL) {
return;
 }
snprintf(perf_map_entry, perf_map_entry_size, "py::%s:%s", entry, filename);

constsize_tname_length=strlen(perf_map_entry);
uwordbase= (uword)code_addr;
uwordsize=code_size;

// Write the code unwinding info event.

// Create unwinding information (eh frame)
ELFObjectContextctx;
charbuffer[1024];
ctx.code_size=code_size;
ctx.startp=ctx.p= (uint8_t*)buffer;
elf_init_ehframe(&ctx);
inteh_frame_size=ctx.p-ctx.startp;

// Populate the unwind info event for perf
CodeUnwindingInfoEventev2;
ev2.base.event=PerfUnwindingInfo;
ev2.base.time_stamp=get_current_monotonic_ticks();
ev2.unwind_data_size=sizeof(EhFrameHeader) +eh_frame_size;
// Ensure we have enough space between DSOs when perf maps them
assert(ev2.unwind_data_size <= PERF_JIT_CODE_PADDING);
ev2.eh_frame_hdr_size=sizeof(EhFrameHeader);
ev2.mapped_size=round_up(ev2.unwind_data_size, 16);
intcontent_size=sizeof(ev2) +sizeof(EhFrameHeader) +eh_frame_size;
intpadding_size=round_up(content_size, 8) -content_size;
ev2.base.size=content_size+padding_size;
perf_map_jit_write_fully(&ev2, sizeof(ev2));


// Populate the eh Frame header
EhFrameHeaderf;
f.version=1;
f.eh_frame_ptr_enc=DwarfSData4 | DwarfPcRel;
f.fde_count_enc=DwarfUData4;
f.table_enc=DwarfSData4 | DwarfDataRel;
f.eh_frame_ptr=-(eh_frame_size+4*sizeof(unsigned char));
f.eh_fde_count=1;
f.from=-(round_up(code_size, 8) +eh_frame_size);
intcie_size=ctx.eh_frame_p-ctx.startp;
f.to=-(eh_frame_size-cie_size);

perf_map_jit_write_fully(ctx.startp, eh_frame_size);
perf_map_jit_write_fully(&f, sizeof(f));

charpadding_bytes[] ="\0\0\0\0\0\0\0\0";
perf_map_jit_write_fully(&padding_bytes, padding_size);

// Write the code load event.
CodeLoadEventev;
ev.base.event=PerfLoad;
ev.base.size=sizeof(ev) + (name_length+1) +size;
ev.base.time_stamp=get_current_monotonic_ticks();
ev.process_id=getpid();
ev.thread_id=syscall(SYS_gettid);
ev.vma=base;
ev.code_address=base;
ev.code_size=size;
perf_jit_map_state.code_id+=1;
ev.code_id=perf_jit_map_state.code_id;

perf_map_jit_write_fully(&ev, sizeof(ev));
perf_map_jit_write_fully(perf_map_entry, name_length+1);
perf_map_jit_write_fully((void*)(base), size);
return;
}

staticintperf_map_jit_fini(void*state) {
if (perf_jit_map_state.perf_map!=NULL) {
// close the file
PyThread_acquire_lock(perf_jit_map_state.map_lock, 1);
fclose(perf_jit_map_state.perf_map);
PyThread_release_lock(perf_jit_map_state.map_lock);

// clean up the lock and state
PyThread_free_lock(perf_jit_map_state.map_lock);
perf_jit_map_state.perf_map=NULL;
 }
if (perf_jit_map_state.mapped_buffer!=NULL) {
munmap(perf_jit_map_state.mapped_buffer, perf_jit_map_state.mapped_size);
 }
trampoline_api.state=NULL;
return0;
}

_PyPerf_Callbacks_Py_perfmap_jit_callbacks= {
&perf_map_jit_init,
&perf_map_jit_write_entry,
&perf_map_jit_fini,
};

#endif