eBPF 用户态跟踪

1
2

g++ --std=c++20 -g -o aa aa.c

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81

dwarfdump aa

.debug_info

COMPILE_UNIT<header overall offset = 0x00000000>:
< 0><0x0000000c>  DW_TAG_compile_unit
                    DW_AT_producer              GNU C++20 13.3.0 -mtune=generic -march=x86-64
					-g -std=c++20 -fasynchronous-unwind-tables -fstack-protector-strong
					-fstack-clash-protection -fcf-protection
                    DW_AT_language              DW_LANG_C_plus_plus_14
                    DW_AT_name                  aa.c
                    DW_AT_comp_dir              /data/test/gcc
                    DW_AT_low_pc                0x00001129
                    DW_AT_high_pc               <offset-from-lowpc> 20 <highpc: 0x0000113d>
                    DW_AT_stmt_list             0x00000000

LOCAL_SYMBOLS:
< 1><0x0000002e>    DW_TAG_subprogram
                      DW_AT_external              yes(1)
                      DW_AT_name                  main
                      DW_AT_decl_file             0x00000001 /data/test/gcc/aa.c
                      DW_AT_decl_line             0x00000001
                      DW_AT_decl_column           0x00000005
                      DW_AT_type                  <0x0000005e>
                      DW_AT_low_pc                0x00001129
                      DW_AT_high_pc               <offset-from-lowpc> 20 <highpc: 0x0000113d>
                      DW_AT_frame_base            len 0x0001: 0x9c:
                          DW_OP_call_frame_cfa
                      DW_AT_call_all_calls        yes(1)
                      DW_AT_sibling               <0x0000005e>
< 2><0x00000050>      DW_TAG_variable
                        DW_AT_name                  x
                        DW_AT_decl_file             0x00000001 /data/test/gcc/aa.c
                        DW_AT_decl_line             0x00000002
                        DW_AT_decl_column           0x00000009
                        DW_AT_type                  <0x0000005e>
                        DW_AT_location              len 0x0002: 0x916c:
                            DW_OP_fbreg -20
< 1><0x0000005e>    DW_TAG_base_type
                      DW_AT_byte_size             0x00000004
                      DW_AT_encoding              DW_ATE_signed
                      DW_AT_name                  int

.debug_line: line number info for a single cu
Source lines (from CU-DIE at .debug_info offset 0x0000000c):

            NS new statement, BB new basic block, ET end of text sequence
            PE prologue end, EB epilogue begin
            IS=val ISA number, DI=val discriminator value
<pc>        [lno,col] NS BB ET PE EB IS= DI= uri: "filepath"
0x00001129  [   1,12] NS uri: "/data/test/gcc/aa.c"
0x00001131  [   2, 9] NS
0x00001138  [   3,12] NS
0x0000113b  [   4, 1] NS
0x0000113d  [   4, 1] NS ET

.debug_str
name at offset 0x00000000, length  154 is 'GNU C++20 13.3.0 -mtune=generic -march=x86-64 
-g -std=c++20 -fasynchronous-unwind-tables -fstack-protector-strong -fstack-clash-protection
 -fcf-protection'
name at offset 0x0000009b, length    4 is 'main'

.debug_aranges

COMPILE_UNIT<header overall offset = 0x00000000>:
< 0><0x0000000c>  DW_TAG_compile_unit
                    DW_AT_producer              GNU C++20 13.3.0 -mtune=generic -march=x86-64 
					-g -std=c++20 -fasynchronous-unwind-tables -fstack-protector-strong 
					-fstack-clash-protection -fcf-protection
                    DW_AT_language              DW_LANG_C_plus_plus_14
                    DW_AT_name                  aa.c
                    DW_AT_comp_dir              /data/test/gcc
                    DW_AT_low_pc                0x00001129
                    DW_AT_high_pc               <offset-from-lowpc> 20 <highpc: 0x0000113d>
                    DW_AT_stmt_list             0x00000000


arange starts at 0x00001129, length of 0x00000014, cu_die_offset = 0x0000000c
arange end

.debug_frame is not present

1
2
3
4
5
6
7
8
9

<1><0x0000002e>      // Level 1, offset 0x2e
DW_TAG_typedef        // Type definition
DW_AT_name            // Attribute: name
size_t                // Value: defined as "size_t"
DW_AT_decl_file       // Declaration file
0x00000003/usr/lib/gcc/x86_64-linux-gnu/13/include/stddef.h // File path
DW_AT_decl_line 0x0000000d  // Line 13 (0xd in hex)
DW_AT_decl_column 0x00000017 // Column 23 (0x17 hex)
DW_AT_type <0x0000003a> // Points to underlying type at offset 0x3a

1
2
3
4
5
6

# 查询符号表（RHEL8系统中 替换为/usr/lib64/libc.so.6）
readelf -Ws /usr/lib/x86_64-linux-gnu/libc.so.6

readelf -Ws /usr/lib64/libc.so.6 | grep -E 'malloc|free|calloc

readelf -Ws /usr/lib64/libc.so.6 | grep pthread_create

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

# 查询USDT信息（USDT信息位于ELF文件的notes段）
readelf -n /usr/lib/x86_64-linux-gnu/libc.so.6

  stapsdt              0x0000004c       NT_STAPSDT (SystemTap probe descriptors)
    Provider: libc
    Name: memory_mallopt_arena_max
    Location: 0x00000000000a8f57, Base: 0x00000000001d6720, Semaphore: 0x0000000000000000
    Arguments: 8@%rax 8@32+mp_(%rip)
  stapsdt              0x0000004d       NT_STAPSDT (SystemTap probe descriptors)
    Provider: libc
    Name: memory_mallopt_arena_test
    Location: 0x00000000000a8f67, Base: 0x00000000001d6720, Semaphore: 0x0000000000000000
    Arguments: 8@%rax 8@24+mp_(%rip)
  stapsdt              0x00000054       NT_STAPSDT (SystemTap probe descriptors)
    Provider: libc
    Name: memory_tunable_tcache_max_bytes
    Location: 0x00000000000a8f80, Base: 0x00000000001d6720, Semaphore: 0x0000000000000000
    Arguments: 8@%rax 8@112+mp_(%rip)

1
2
3

bpftrace -l 'uprobe:/usr/lib/x86_64-linux-gnu/libc.so.6:*' | grep fork

bpftrace -l 'usdt:/usr/lib/x86_64-linux-gnu/libc.so.6:*' | grep fork

1

bpftrace -e 'uprobe:/usr/lib/x86_64-linux-gnu/libc.so.6:malloc { @[comm] = count(); }'

1
2
3
4
5

@[bpftrace]: 3
@[gsd-power]: 6
。。。
@[sshd]: 27588
@[bash]: 149148

1
2
3
4
5
6
7
8

# Terminal 1: Trace __libc_fork
sudo bpftrace -e \
'uprobe:/usr/lib/x86_64-linux-gnu/libc.so.6:__libc_fork { 
    printf("[%d] %s called __libc_fork\n", pid, comm); 
}'

# Terminal 2: Trigger fork
sleep 1 &  # Creates background process

1
2
3
4
5

bpftrace -e ' \
usdt:/usr/lib/x86_64-linux-gnu/libc.so.6:libc:memory_* {
    printf("PID %d allocated %d bytes\n", pid, arg0);
    @allocations[comm] = count();
}'

1
2
3
4
5
6
7
8
9

// Compiled into binary​​: Probes become part of the ELF file (.note.stapsdt section).
// ​​BPF attaches dynamically​​: No need to modify running binaries.
#include <sys/sdt.h>  // USDT header

void process_payment(Payment *p) {
    DTRACE_PROBE3(myapp, payment_start,  // ← USDT probe
                  p->id, p->amount, p->currency);
    // ... processing logic ...
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

readelf -n /opt/myapp/bin/server | grep -A4 'stapsdt'

// 输出
Provider: myapp
Name: payment_start
Location: 0x4015a0
Arguments: -8@%rdi -4@%rsi -8@%rdx  # arg1=id, arg2=amount, arg3=currency


// BPF 程序
// Attach to USDT probe
SEC("usdt:/opt/myapp/bin/server:myapp:payment_start")
int trace_payment(struct pt_regs *ctx) {
    u64 id = BPF_CORE_READ(ctx, di);      // Read arg1 (rdi)
    u32 amount = BPF_CORE_READ(ctx, si);   // Read arg2 (rsi)
    char currency[8];
    bpf_probe_read_user_str(currency, sizeof(currency), 
                            (void*)BPF_CORE_READ(ctx, dx)); // arg3 (rdx)
    
    bpf_printk("Payment %llu: %d %s", id, amount, currency);
    return 0;
}

1
2
3
4
5
6
7
8

bpftrace -e \
'uprobe:/lib/x86_64-linux-gnu/libc.so.6:malloc { 
    printf("malloc(%d)\n", arg0);   
}'

bpftrace -e 'usdt:/lib/x86_64-linux-gnu/libc.so.6:memory_malloc_retry {
    printf("malloc(%d)\n", arg0);
}'

1

bpftrace -l 'uprobe:/usr/bin/bash:*'  | grep read | grep line

1
2
3
4

sudo bpftrace -e \
'uretprobe:/usr/bin/bash:readline { 
  printf("User %d executed \"%s\" command\n", uid, str(retval)); 
}'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

// 包含头文件
#include <uapi/linux/ptrace.h>

// 定义数据结构和性能事件映射
struct data_t {
    u32 uid;
    char command[64];
};
BPF_PERF_OUTPUT(events);

// 定义uretprobe处理函数
int bash_readline(struct pt_regs *ctx)
{
    // 查询uid
    struct data_t data = { };
    data.uid = bpf_get_current_uid_gid();

    // 从PT_REGS_RC(ctx)读取返回值
    bpf_probe_read_user(&data.command, sizeof(data.command), (void *)PT_REGS_RC(ctx));

    // 提交性能事件
    events.perf_submit(ctx, &data, sizeof(data));
    return 0;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

# 引入BCC库
from bcc import BPF
from time import strftime

# 加载eBPF 程序
b = BPF(src_file="bashreadline.c")

# 挂载uretprobe
b.attach_uretprobe(name="/usr/bin/bash", sym="readline", fn_name="bash_readline")

# 定义性能事件回调（输出时间、UID以及Bash中执行的命令）
def print_event(cpu, data, size):
    event = b["events"].event(data)
    print("%-9s %-6d %s" % (strftime("%H:%M:%S"), event.uid, event.command.decode("utf-8")))

# 打印头
print("%-9s %-6s %s" % ("TIME", "UID", "COMMAND"))

# 绑定性能事件映射和输出函数，并从映射中循环读取数据
b["events"].open_perf_buffer(print_event)
while 1:
    try:
        b.perf_buffer_poll()
    except KeyboardInterrupt:
        exit()

1

bpftrace -l '*:/usr/bin/python3:*'

1
2
3

// 三个参数分别是文件名、函数名和行号
function__entry(str filename, str funcname, int lineno)
function__return(str filename, str funcname, int lineno)

1
2
3

sudo bpftrace -e \
'usdt:/usr/bin/python3:function__entry { printf("%s:%d %s\n", 
str(arg0), arg2, str(arg1))}'

1
2
3
4
5

sudo bpftrace -e '  
uprobe:/usr/bin/python3:PyDict_New {  
    printf("Process %s (PID %d) created a new dictionary\n", comm, pid);  
}  
'  

1

apt-get install python3-dbg   

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

sudo bpftrace -e '
uprobe:/usr/bin/python3:PyDict_New {
    @calls[comm] = count();  // Count calls per process
}

uretprobe:/usr/bin/python3:PyDict_New {
    printf("PID %d created dict at 0x%p\n", pid, retval);
    @dicts = hist(retval);   // Track address distribution
}

END {
    printf("\n=== Dictionary Creation Summary ===\n");
    print(@calls);
}'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

// 头文件引用和数据结构定义...

int print_functions(struct pt_regs *ctx)
{
    uint64_t argptr;
    struct data_t data = { };

  // 参数1是文件名
    bpf_usdt_readarg(1, ctx, &argptr);
    bpf_probe_read_user(&data.filename, sizeof(data.filename),
                (void *)argptr);

  // 参数2是函数名
    bpf_usdt_readarg(2, ctx, &argptr);
    bpf_probe_read_user(&data.funcname, sizeof(data.funcname),
                (void *)argptr);

  // 参数3是行号
  bpf_usdt_readarg(3, ctx, &data.lineno);

    // 最后提交性能事件
    events.perf_submit(ctx, &data, sizeof(data));
    return 0;
};

1
2
3
4
5
6
7

from bcc import BPF, USDT

u = USDT(pid=pid)
u.enable_probe(probe="function__entry", fn_name="print_functions")
b = BPF(src_file="<ebpf-program>.c", usdt_contexts=[u])

# 其他处理逻辑

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

bpftrace -l '*:/data/soft/jdk-24.0.1/bin/java:*'
uprobe:/data/soft/jdk-24.0.1/bin/java:__do_global_dtors_aux
uprobe:/data/soft/jdk-24.0.1/bin/java:__libc_csu_fini
uprobe:/data/soft/jdk-24.0.1/bin/java:__libc_csu_init
uprobe:/data/soft/jdk-24.0.1/bin/java:_fini
uprobe:/data/soft/jdk-24.0.1/bin/java:_init
uprobe:/data/soft/jdk-24.0.1/bin/java:_start
uprobe:/data/soft/jdk-24.0.1/bin/java:call_gmon_start
uprobe:/data/soft/jdk-24.0.1/bin/java:deregister_tm_clones
uprobe:/data/soft/jdk-24.0.1/bin/java:frame_dummy
uprobe:/data/soft/jdk-24.0.1/bin/java:main
uprobe:/data/soft/jdk-24.0.1/bin/java:register_tm_clones

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

bpftrace -e '
uprobe:/data/soft/jdk-24.0.1/bin/java:main {
    printf("Java launched with %d arguments:\n", arg0);
    $argv = arg1;

    // Read first 3 arguments (adjust as needed)
    printf("argv[0] = %s\n", str(*(uint64*)($argv + 0)));
    printf("argv[1] = %s\n", str(*(uint64*)($argv + 8)));
    printf("argv[2] = %s\n", str(*(uint64*)($argv + 16)));
}'

1
2
3
4
5

Attaching 1 probe...
Java launched with 3 arguments:
argv[0] = java
argv[1] = -agentpath:./libperfmap.so
argv[2] = MyApp

1
2
3
4
5
6
7
8

# 很多 jvm 相关的跟踪点
bpftrace -l '*:/data/soft/jdk-24.0.1/lib/libjava.so:*'

# nio 相关的
bpftrace -l '*:/data/soft/jdk-24.0.1/lib/libnio.so:*'

# 包含了 39801 个跟踪点
bpftrace -l '*:/data/soft/jdk-24.0.1/lib/*.so:*' | wc -l