eBPF 总结 2

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

#!/usr/bin/env bpftrace

#include <linux/sched.h>

BEGIN
{
        printf("Tracing CPU scheduler... Hit Ctrl-C to end.\n");
}

tracepoint:sched:sched_wakeup,
tracepoint:sched:sched_wakeup_new
{
        @qtime[args->pid] = nsecs;
}

tracepoint:sched:sched_switch
{
        if (args->prev_state == TASK_RUNNING) {
                @qtime[args->prev_pid] = nsecs;
        }

        $ns = @qtime[args->next_pid];
        if ($ns) {
                @usecs = hist((nsecs - $ns) / 1000);
        }
        delete(@qtime[args->next_pid]);
}

END
{
        clear(@qtime);
}
root@node2:~#
root@node2:~# bpftrace -e '
#include <linux/sched.h>
tracepoint:sched:sched_wakeup,
tracepoint:sched:sched_wakeup_new
{
        @qtime[args->pid] = nsecs;
}

tracepoint:sched:sched_switch
{
        if (args->prev_state == TASK_RUNNING) {
                @qtime[args->prev_pid] = nsecs;
        }

        $ns = @qtime[args->next_pid];
        if ($ns) {
                @usecs = hist((nsecs - $ns) / 1000);
        }
        delete(@qtime[args->next_pid]);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

# 跟踪新进程，包括进程参数：
bpftrace -e 'tracepoint:syscalls:sys_enter_execve { join(args->argv); }'


# 输出哪个进程执行了哪个新进程：
bpftrace -e 'tracepoint:syscalls:sys_enter_execve { printf("%s -> %s\n", comm, str(args->filename)); }'


# 按进程统计系统调用的数量：
bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @[comm] = count(); }'


# 按系统调用的探针名字来统计系统调用的数量：
bpftrace -e 'tracepoint:syscalls:sys_enter_* { @[probe] = count(); }'


# 按系统调用的函数名来统计系统调用的数量：
bpftrace -e 'tracepoint:raw_syscalls:sys_enter {
    @[sym(*(kaddr("sys_call_table") + args->id * 8))] = count(); }'
	

# 以 99Hz 的频率采样正在运行的进程名：
bpftrace -e 'profile:hz:99 { @[comm] = count(); }'


# 以 49Hz 的频率采样进程 ID 为 189 的用户态调用栈信息：
bpftrace -e 'profile:hz:49 /pid == 189/ { @[ustack] = count(); }'	


# 采样所有的进程名和调用栈信息：
bpftrace -e 'profile:hz:49 { @[ustack, stack, comm] = count(); }'


# 按 99Hz 的频率采样正在运行的 CPU，并且以线性直方图输出：
bpftrace -e 'profile:hz:99 { @cpu = lhist(cpu, 0, 256, 1); }'


# 统计内核中以 "vfs_" 开头的函数调用频率：
bpftrace -e 'kprobe:vfs_* { @[func] = count(); }'


# 按名字和内核调用栈来统计 SMP 调用：
bpftrace -e 'kprobe:smp_call* { @[probe, kstack(5)] = count(); }'


# 按名字和内核调用栈来统计 Intel x2APIC 调用：
bpftrace -e 'kprobe:x2apic_send_IPI* { @[probe, kstack(5)] = count(); }'


# 跟踪通过 pthread_create () 创建的新线程：
bpftrace -e 'u:/lib/x86_64-linux-gnu/libpthread-2.27.so:pthread_create {
    printf("%s by %s (%d)\n", probe, comm, pid); }'

1

bpftrace -l 'usdt:/lib/x86_64-linux-gnu/libc.so.6:libc:*'s

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

#!/usr/local/bin/bpftrace

#include <linux/mm.h>

kprobe:handle_mm_fault
{
    $vma = (struct vm_area_struct *)arg0;
    $file = $vma->vm_file->f_path.dentry->d_name.name;
    @[str($file)] = count();
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

#!/usr/local/bin/bpftrace

tracepoint:vmscan:mm_shrink_slab_start { @start_ss[tid] = nsecs; }
tracepoint:vmscan:mm_shrink_slab_end /@start_ss[tid]/
{
    $dur_ss = nsecs - @start_ss[tid];
    @sum_ss = @sum_ss + $dur_ss;
    @shrink_slab_ns = hist($dur_ss);
    delete(@start_ss[tid]);
}

tracepoint:vmscan:mm_vmscan_direct_reclaim_begin { @start_dr[tid] = nsecs; }
tracepoint:vmscan:mm_vmscan_direct_reclaim_end /@start_dr[tid]/
{
    $dur_dr = nsecs - @start_dr[tid];
    @sum_dr = @sum_dr + $dur_dr;
    @direct_reclaim_ns = hist($dur_dr);
    delete(@start_dr[tid]);
}

tracepoint:vmscan:mm_vmscan_memcg_reclaim_begin { @start_mr[tid] = nsecs; }
tracepoint:vmscan:mm_vmscan_memcg_reclaim_end /@start_mr[tid]/
{
    $dur_mr = nsecs - @start_mr[tid];
    @sum_mr = @sum_mr + $dur_mr;
    @memcg_reclaim_ns = hist($dur_mr);
    delete(@start_mr[tid]);
}

tracepoint:vmscan:mm_vmscan_wakeup_kswapd { @count_wk++; }

tracepoint:vmscan:mm_vmscan_writepage { @count_wp++; }

BEGIN
{
    printf("%-10s %10s %12s %12s %6s %9s\n", "TIME",
        "S-SLABms", "D-RECLAIMms", "M-RECLAIMms", "KSWAPD", "WRITEPAGE");
}

interval:s:1
{
    time("%H:%M:%S");
    printf("    %10d %12d %12d %6d %9d\n",
        @sum_ss / 1000000, @sum_dr / 1000000, @sum_mr / 1000000,
        @count_wk, @count_wp);
    clear(@sum_ss);
    clear(@sum_dr);
    clear(@sum_mr);
    clear(@count_wk);
    clear(@count_wp);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

#!/usr/local/bin/bpftrace

kprobe:swap_readpage
{
        @[comm, pid] = count();
}

interval:s:1
{
        time();
        print(@);
        clear(@);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

#!/usr/local/bin/bpftrace

BEGIN
{
    printf("Tracing Huge Page faults per process... Hit Ctrl-C to end.\n");
}

kprobe:hugetlb_fault
{
    @[pid, comm] = count();
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

# 根据用户态调用栈信息统计进程堆内存扩展（brk ()）：
stackcount -U t:syscalls:sys_enter_brk

# 根据用户态调用栈信息统计缺页错误：
stackcount -U t:exceptions:page_fault_user

# 通过跟踪点来统计 vmscan 操作：
funccount 't:vmscan:*'

# 按进程展示 hugepage_madvise () 调用：
trace hugepage_madvise

# 统计页迁移事件：
funccount t:migrate:mm_migrate_pages

# 统计页压缩事件：
trace t:compaction:mm_compaction_begin

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

# 根据用户态调用栈信息统计进程堆内存扩展（brk ()）：bpftrace -e tracepoint:syscalls:sys_enter_brk { @[ustack, comm] = count(); }

# 按进程统计缺页错误：
bpftrace -e 'software:page-fault:1 { @[comm, pid] = count(); }'

# 根据用户态调用栈信息统计缺页错误：
bpftrace -e 'tracepoint:exceptions:page_fault_user { @[ustack, comm] = count(); }'

# 通过跟踪点来统计 vmscan 操作：
bpftrace -e 'tracepoint:vmscan:* { @[probe] = count(); }'

# 按进程展示 hugepage_madvise () 调用：
bpftrace -e 'kprobe:hugepage_madvise { printf("%s by PID %d\n", probe, pid); }'

# 统计页迁移事件：
bpftrace -e 'tracepoint:migrate:mm_migrate_pages { @ = count(); }'

# 统计页压缩事件：
bpftrace -e 't:compaction:mm_compaction_begin { time(); }'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

#!/usr/local/bin/bpftrace

#include <linux/mm.h>

kprobe:do_mmap
{
    $file = (struct file *)arg0;
    $name = $file->f_path.dentry;
    $dir1 = $name->d_parent;
    $dir2 = $dir1->d_parent;
    @[str($dir2->d_name.name), str($dir1->d_name.name),
        str($name->d_name.name)] = count();
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

#!/usr/local/bin/bpftrace

#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/fdtable.h>

tracepoint:syscalls:sys_enter_read
{
    $task = (struct task_struct *)curtask;
    $file = (struct file *)*($task->files->fdt->fd + args->fd);
    @filename[str($file->f_path.dentry->d_name.name)] = count();
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

#!/usr/local/bin/bpftrace

#include <linux/mm.h>

kprobe:filemap_fault
{
    $vf = (struct vm_fault *)arg0;
    $file = $vf->vma->vm_file->f_path.dentry->d_name.name;
    @[comm, str($file)] = count();
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

#!/usr/local/bin/bpftrace

#include <linux/fs.h>

kprobe:vfs_read,
kprobe:vfs_readv,
kprobe:vfs_write,
kprobe:vfs_writev
{
    @file[tid] = arg0;
}

kretprobe:vfs_read,
kretprobe:vfs_readv,
kretprobe:vfs_write,
kretprobe:vfs_writev
/@file[tid]/
{
    if (retval >= 0) {
        $file = (struct file *)@file[tid];
        $name = $file->f_path.dentry->d_name.name;
        if ((($file->f_inode->i_mode >> 12) & 15) == DT_FIFO) {
            @[comm, "FIFO"] = hist(retval);
        } else {
            @[comm, str($name)] = hist(retval);
        }
    }
    delete(@file[tid]);
}

END
{
    clear(@file);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

#!/usr/local/bin/bpftrace

#include <linux/fs.h>

BEGIN
{
    printf("Tracing VFS reads and writes... Hit Ctrl-C to end.\n");
}

kprobe:vfs_read,
kprobe:vfs_readv,
kprobe:vfs_write,
kprobe:vfs_writev
{
    @[str(((struct file *)arg0)->f_inode->i_sb->s_type->name), func] = count();
}

interval:s:1
{
    time(); print(@); clear(@);
}

END
{
    clear(@);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

#!/usr/local/bin/bpftrace

#include <linux/fs.h>

BEGIN
{
    // from uapi/linux/stat.h:
    @type[0xc000] = "socket";
    @type[0xa000] = "link";
    @type[0x8000] = "regular";
    @type[0x6000] = "block";
    @type[0x4000] = "directory";
    @type[0x2000] = "character";
    @type[0x1000] = "fifo";
    @type[0] = "other";
}

kprobe:vfs_read,
kprobe:vfs_readv,
kprobe:vfs_write,
kprobe:vfs_writev
{
    $file = (struct file *)arg0;
    $mode = $file->f_inode->i_mode;
    @[@type[$mode & 0xf000], func, comm] = count();
}

END
{
    clear(@type);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

#!/usr/local/bin/bpftrace

#include <linux/fs.h>
#include <asm-generic/fcntl.h>

BEGIN
{
    printf("Tracing VFS write sync flags... Hit Ctrl-C to end.\n");
}

kprobe:vfs_write,
kprobe:vfs_writev
{
    $file = (struct file *)arg0;
    $name = $file->f_path.dentry->d_name.name;
    if (((($file->f_inode->i_mode >> 12) & 15) == DT_REG)) {
        if ($file->f_flags & O_DSYNC) {
            @sync[comm, str($name)] = count();
        } else {
            @regular[comm, str($name)] = count();
        }
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

#!/usr/local/bin/bpftrace

BEGIN
{
    printf("Tracing writeback... Hit Ctrl-C to end.\n");
    printf("%-9s %-8s %-8s %-16s %s\n", "TIME", "DEVICE", "PAGES", "REASON", "ms");

    // see /sys/kernel/debug/tracing/events/writeback/writeback_start/format
    @reason[0] = "background";
    @reason[1] = "vmscan";
    @reason[2] = "sync";
    @reason[3] = "periodic";
    @reason[4] = "laptop_timer";
    @reason[5] = "free_more_memory";
    @reason[6] = "fs_free_space";
    @reason[7] = "forker_thread";
}

tracepoint:writeback:writeback_start
{
    @start[args->sb_dev] = nsecs;
    @pages[args->sb_dev] = args->nr_pages;
}

tracepoint:writeback:writeback_written
/@start[args->sb_dev]/
{
    $sb_dev = args->sb_dev;
    $s = @start[$sb_dev];
    $lat = $s ? (nsecs - $s) / 1000 : 0;
    $pages = @pages[args->sb_dev] - args->nr_pages;

    time("%H:%M:%S  ");
    printf("%-8s %-8d %-16s %d.%03d\n", args->name, $pages, @reason[args->reason], $lat / 1000, $lat % 1000);

    delete(@start[$sb_dev]);
    delete(@pages[$sb_dev]);
}

END
{
    clear(@reason);
    clear(@start);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

#!/usr/local/bin/bpftrace

BEGIN
{
    printf("Tracing dcache lookups... Hit Ctrl-C to end.\n");
    printf("%10s %10s %5s\n", "REFS", "MISSES", "HIT%");
}

kprobe:lookup_fast { @hits++; }

kretprobe:d_lookup /retval == 0/ { @misses++; }

interval:s:1
{
    $refs = @hits + @misses;
    $percent = $refs > 0 ? 100 * @hits / $refs : 0;
    printf("%10d %10d %4d%%\n", $refs, @misses, $percent);
    clear(@hits);
    clear(@misses);
}

END
{
    clear(@hits);
    clear(@misses);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

#!/usr/local/bin/bpftrace

#include <linux/fs.h>

kprobe:add_to_page_cache_lru
{
    $as = (struct address_space *)arg1;
    $mode = $as->host->i_mode;
    // match block mode, uapi/linux/stat.h:
    if ($mode & 0x6000) {
        @kb[comm] = sum(4);     // page size
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

#!/usr/local/bin/bpftrace

kprobe:__do_page_cache_readahead    { @in_readahead[tid] = 1; }
kretprobe:__do_page_cache_readahead { @in_readahead[tid] = 0; }

kretprobe:__page_cache_alloc
/@in_readahead[tid]/
{
    @birth[retval] = nsecs;
    @rapages++;
}

kprobe:mark_page_accessed
/@birth[arg0]/
{
    @age_ms = hist((nsecs - @birth[arg0]) / 1000000);
    delete(@birth[arg0]);
    @rapages--;
}

END
{
    printf("\nReadahead unused pages: %d\n", @rapages);
    printf("\nReadahead used page age (ms):\n");
    print(@age_ms); clear(@age_ms);
    clear(@birth); clear(@in_readahead); clear(@rapages);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

# 按进程名跟踪通过 creat (2) 创建的文件：
trace 't:syscalls:sys_enter_creat "%s", args->pathname'

# 按文件名统计 newstat (2) 调用：
argdist -C 't:syscalls:sys_enter_newstat():char*:args->filename'

# 按系统调用方式统计 read 系统调用：
funccount 't:syscalls:sys_enter_*read*'

# 按系统调用方式统计 write 系统调用：
funccount 't:syscalls:sys_enter_*write*'

#展示 read () 系统调用的请求大小分布：
argdist -H 't:syscalls:sys_enter_read():int:args->count'

# 展示 read () 系统调用的实际读取字节数（以及错误）：
argdist -H 't:syscalls:sys_exit_read():int:args->ret'

# 按错误代码统计 read () 系统调用的错误：
argdist -C 't:syscalls:sys_exit_read():int:args->ret:args->ret<0'

# 统计 VFS 调用：
funccount 'vfs_*'

# 统计 ext4 跟踪点：
funccount 't:ext4:*'

# 统计 XFS 跟踪点：
funccount 't:xfs:*'

# 按进程名和调用栈信息统计 ext4 文件读取操作：
stackcount ext4_file_read_iter

# 跟踪 ZFS spa_sync () 时间：
trace -T 'spa_sync "ZFS spa_sync()"'

# 按进程名和调用栈信息统计使用 read_pages 向存储设备进行的文件系统读取操作：
stackcount -P read_pages

# 按调用栈和进程名统计所有通过 ext4 向存储设备进行的读取操作：
stackcount -P ext4_readpages

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

#按进程名统计通过 open (2) 打开的文件：
bpftrace -e '
t:syscalls:sys_enter_open { 
printf("%s %s\n", comm, str(args->filename)); 
}'

#按进程名统计通过 creat (2) 创建的文件：
bpftrace -e '
t:syscalls:sys_enter_creat { 
    printf("%s %s\n", comm, str(args->pathname)); 
}'

# 按文件名统计 newstat (2) 调用：
bpftrace -e '
t:syscalls:sys_enter_newstat {
   @[str(args->filename)] = count(); 
}'

# 按系统调用方式统计 read 系统调用：
bpftrace -e '
  tracepoint:syscalls:sys_enter_*read* { @[probe] = count(); 
}'


# 按系统调用方式统计 write 系统调用：
bpftrace -e '
  tracepoint:syscalls:sys_enter_*write* { @[probe] = count(); 
}'

# 展示 read () 系统调用的请求大小分布：
bpftrace -e '
  tracepoint:syscalls:sys_enter_read { @ = hist(args->count); 
}'

# 展示 read () 系统调用的实际读取字节数（以及错误）：
bpftrace -e '
tracepoint:syscalls:sys_exit_read { 
  @ = hist(args->ret); 
}'

# 统计 VFS 调用：
bpftrace -e '
kprobe:vfs_* {
 @[probe] = count(); 
}'

# 统计 ext4 跟踪点：
bpftrace -e '
  tracepoint:ext4:* { @[probe] = count(); 
}'

# 统计 XFS 跟踪点：
bpftrace -e '
tracepoint:xfs:* {
 @[probe] = count(); 
}'

# 按进程名和调用栈信息统计 ext4 文件读取操作：
bpftrace -e '
kprobe:ext4_file_read_iter {
 @[comm] = count(); 
}'

# 按进程名和用户态调用栈信息统计 ext4 文件读取操作：
bpftrace -e '
kprobe:ext4_file_read_iter {
 @[ustack, comm] = count(); 
}'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

#!/usr/local/bin/bpftrace

BEGIN
{
    printf("Tracing block I/O requested seeks... Hit Ctrl-C to end.\n");
}

tracepoint:block:block_rq_issue
{
    if (@last[args->dev]) {
        // calculate requested seek distance
        $last = @last[args->dev];
        $dist = (args->sector - $last) > 0 ?
            args->sector - $last : $last - args->sector;

        // store details
        @sectors[args->comm] = hist($dist);
    }
    // save last requested position of disk head
    @last[args->dev] = args->sector + args->nr_sector;
}

END
{
    clear(@last);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

#!/usr/local/bin/bpftrace

BEGIN
{
    printf("%-8s %5s %5s %8s %10s\n", "TIME", "%RND", "%SEQ", "COUNT", "KBYTES");
}

tracepoint:block:block_rq_complete
{
    if (@lastsector[args->dev] == args->sector) {
        @sequential++;
    } else {
        @random++;
    }
    @bytes = @bytes + args->nr_sector * 512;
    @lastsector[args->dev] = args->sector + args->nr_sector;
}

interval:s:1
{
    $count = @random + @sequential;
    $div = $count;
    if ($div == 0) {
        $div = 1;
    }
    time("%H:%M:%S ");
    printf("%5d %5d %8d %10d\n", @random * 100 / $div,
        @sequential * 100 / $div, $count, @bytes / 1024);
    clear(@random); clear(@sequential); clear(@bytes);
}

END
{
    clear(@lastsector);
    clear(@random); clear(@sequential); clear(@bytes);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

#!/usr/local/bin/bpftrace

BEGIN
{
    printf("Tracing block I/O with init stacks. Hit Ctrl-C to end.\n");
}

kprobe:blk_account_io_start
{
    @reqstack[arg0] = kstack;
    @reqts[arg0] = nsecs;
}

kprobe:blk_start_request,
kprobe:blk_mq_start_request
/@reqts[arg0]/
{
    @usecs[@reqstack[arg0]] = hist(nsecs - @reqts[arg0]);
    delete(@reqstack[arg0]);
    delete(@reqts[arg0]);
}

END
{
    clear(@reqstack); clear(@reqts);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

#!/usr/local/bin/bpftrace

BEGIN
{
    printf("Tracing block I/O errors. Hit Ctrl-C to end.\n");
}

tracepoint:block:block_rq_complete
/args->error != 0/
{
    time("%H:%M:%S ");
    printf("device: %d,%d, sector: %d, bytes: %d, flags: %s, error: %d\n",
        args->dev >> 20, args->dev & ((1 << 20) - 1), args->sector,
        args->nr_sector * 512, args->rwbs, args->error);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

#!/usr/local/bin/bpftrace

#include <linux/blkdev.h>

BEGIN
{
    printf("Tracing block I/O schedulers. Hit Ctrl-C to end.\n");
}

kprobe:__elv_add_request
{
    @start[arg1] = nsecs;
}

kprobe:blk_start_request,
kprobe:blk_mq_start_request
/@start[arg0]/
{
    $r = (struct request *)arg0;
    @usecs[$r->q->elevator->type->elevator_name] = hist((nsecs - @start[arg0]) / 1000);
    delete(@start[arg0]);
}

END
{
    clear(@start);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

#!/usr/local/bin/bpftrace

#include <scsi/scsi_cmnd.h>

BEGIN
{
    printf("Tracing scsi latency. Hit Ctrl-C to end.\n");
    // SCSI opcodes from scsi/scsi_proto.h; add more mappings if desired:
    @opcode[0x00] = "TEST_UNIT_READY";
    @opcode[0x03] = "REQUEST_SENSE";
    @opcode[0x08] = "READ_6";
    @opcode[0x0a] = "WRITE_6";
    @opcode[0x0b] = "SEEK_6";
    @opcode[0x12] = "INQUIRY";
    @opcode[0x18] = "ERASE";
    @opcode[0x28] = "READ_10";
    @opcode[0x2a] = "WRITE_10";
    @opcode[0x2b] = "SEEK_10";
    @opcode[0x35] = "SYNCHRONIZE_CACHE";
}

kprobe:scsi_init_io
{
    @start[arg0] = nsecs;
}

kprobe:scsi_done,
kprobe:scsi_mq_done
/@start[arg0]/
{
    $cmnd = (struct scsi_cmnd *)arg0;
    $opcode = *$cmnd->req.cmd & 0xff;
    @usecs[$opcode, @opcode[$opcode]] = hist((nsecs - @start[arg0]) / 1000);
}

END
{
    clear(@start); clear(@opcode);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

#!/usr/local/bin/bpftrace

BEGIN
{
    printf("Tracing scsi command results. Hit Ctrl-C to end.\n");

    // host byte codes, from include/scsi/scsi.h:
    @host[0x00] = "DID_OK";
    @host[0x01] = "DID_NO_CONNECT";
    @host[0x02] = "DID_BUS_BUSY";
    @host[0x03] = "DID_TIME_OUT";
    @host[0x04] = "DID_BAD_TARGET";
    @host[0x05] = "DID_ABORT";
    @host[0x06] = "DID_PARITY";
    @host[0x07] = "DID_ERROR";
    @host[0x08] = "DID_RESET";
    @host[0x09] = "DID_BAD_INTR";
    @host[0x0a] = "DID_PASSTHROUGH";
    @host[0x0b] = "DID_SOFT_ERROR";
    @host[0x0c] = "DID_IMM_RETRY";
    @host[0x0d] = "DID_REQUEUE";
    @host[0x0e] = "DID_TRANSPORT_DISRUPTED";
    @host[0x0f] = "DID_TRANSPORT_FAILFAST";
    @host[0x10] = "DID_TARGET_FAILURE";
    @host[0x11] = "DID_NEXUS_FAILURE";
    @host[0x12] = "DID_ALLOC_FAILURE";
    @host[0x13] = "DID_MEDIUM_ERROR";

    // status byte codes, from include/scsi/scsi_proto.h:
    @status[0x00] = "SAM_STAT_GOOD";
    @status[0x02] = "SAM_STAT_CHECK_CONDITION";
    @status[0x04] = "SAM_STAT_CONDITION_MET";
    @status[0x08] = "SAM_STAT_BUSY";
    @status[0x10] = "SAM_STAT_INTERMEDIATE";
    @status[0x14] = "SAM_STAT_INTERMEDIATE_CONDITION_MET";
    @status[0x18] = "SAM_STAT_RESERVATION_CONFLICT";
    @status[0x22] = "SAM_STAT_COMMAND_TERMINATED";
    @status[0x28] = "SAM_STAT_TASK_SET_FULL";
    @status[0x30] = "SAM_STAT_ACA_ACTIVE";
    @status[0x40] = "SAM_STAT_TASK_ABORTED";
}

tracepoint:scsi:scsi_dispatch_cmd_done
{
    @[@host[(args->result >> 16) & 0xff], @status[args->result & 0xff]] = count();
}

END
{
    clear(@status);
    clear(@host);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

#!/usr/local/bin/bpftrace

#include <linux/blkdev.h>
#include <linux/nvme.h>

BEGIN
{
    printf("Tracing nvme command latency. Hit Ctrl-C to end.\n");
    // from linux/nvme.h:
    @iopcode[0x00] = "nvme_cmd_flush";
    @iopcode[0x01] = "nvme_cmd_write";
    @iopcode[0x02] = "nvme_cmd_read";
    @iopcode[0x04] = "nvme_cmd_write_uncor";
    @iopcode[0x05] = "nvme_cmd_compare";
    @iopcode[0x08] = "nvme_cmd_write_zeros";
    @iopcode[0x09] = "nvme_cmd_dsm";
    @iopcode[0x0d] = "nvme_cmd_resv_register";
    @iopcode[0x0e] = "nvme_cmd_resv_report";
    @iopcode[0x11] = "nvme_cmd_resv_acquire";
    @iopcode[0x15] = "nvme_cmd_resv_release";
}

kprobe:nvme_setup_cmd
{
    $req = (struct request *)arg1;
    if ($req->rq_disk) {
        @start[arg1] = nsecs;
        @cmd[arg1] = arg2;
    } else {
        @admin_commands = count();
    }
}

kprobe:nvme_complete_rq
/@start[arg0]/
{
    $req = (struct request *)arg0;
    $cmd = (struct nvme_command *)@cmd[arg0];
    $disk = $req->rq_disk;
    $opcode = $cmd->common.opcode & 0xff;
    @usecs[$disk->disk_name, @iopcode[$opcode]] = hist((nsecs - @start[arg0]) / 1000);
    delete(@start[tid]); delete(@cmd[tid]);
}

END
{
    clear(@iopcode); clear(@start); clear(@cmd);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

# 统计块 I/O 跟踪点：
bpftrace -e 'tracepoint:block:* { @[probe] = count(); }'

# 以直方图方式统计块 I/O 尺寸：
bpftrace -e 't:block:block_rq_issue { @bytes = hist(args->bytes); }'

# 统计块 I/O 请求的用户态调用栈：
bpftrace -e 't:block:block_rq_issue { @[ustack] = count(); }'

# 统计块 I/O 的类型标记：
bpftrace -e 't:block:block_rq_issue { @[args->rwbs] = count(); }'

# 按 I/O 类型统计总字节数：
bpftrace -e 't:block:block_rq_issue { @[args->rwbs] = sum(args->bytes); }'

# 按设备和 I/O 类型跟踪块 I/O 错误：
bpftrace -e 't:block:block_rq_complete /args->error/ { printf("dev %d type %s error %d\n", args->dev, args->rwbs, args->error); }'

# 以直方图方式统计块 I/O plug 时间：
bpftrace -e 'k:blk_start_plug { @ts[arg0] = nsecs; } k:blk_flush_plug_list /@ts[arg0]/ { @plug_ns = hist(nsecs - @ts[arg0]); delete(@ts[arg0]); }'

# 统计 SCSI opcode：
bpftrace -e 't:scsi:scsi_dispatch_cmd_start { @opcode[args->opcode] = count(); }'

# 统计 SCSI 结果代码（包括全部 4 字节）：
bpftrace -e 't:scsi:scsi_dispatch_cmd_done { @result[args->result] = count(); }'

# 统计 blk_mq 请求的 CPU 分布：
bpftrace -e 'k:blk_mq_start_request { @swqueues = lhist(cpu, 0, 100, 1); }'

# 统计 scsi 驱动程序函数：
bpftrace -e 'kprobe:scsi* { @[func] = count(); }'

# 统计 nvme 驱动程序函数：
funccount 'nvme*'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

# 按错误代码统计失败的 socket 调用
argdist -C 't:syscalls:sys_exit_connect():int:args->ret:args->ret<0'

# 按用户态调用栈统计套接字 connect (2) 调用：
stackcount -U t:syscalls:sys_enter_connect

# 以直方图形式统计 TCP 发送的字节数：
argdist -H 'p:tcp_sendmsg(void *sk, void *msg, int size):int:size'

# 以直方图形式统计 TCP 接收的字节数：
argdist -H 'r:tcp_recvmsg():int:$retval:$retval>0'

# 统计所有的 TCP 函数的调用频率（对所有的 TCP 操作都添加额外开销）：
funccount 'tcp_*'

# 以直方图形式统计 UDP 发送的字节数：
argdist -H 'p:udp_sendmsg(void *sk, void *msg, int size):int:size'

# 以直方图形式统计 UDP 接收的字节数：
argdist -H 'r:udp_recvmsg():int:$retval:$retval>0'

# 统计所有的 UDP 函数的调用频率（对所有的 UDP 操作都添加额外开销）：
funccount 'udp_*'

# 统计网络包发送的调用栈信息：
stackcount t:net:net_dev_xmit

# 统计 ieee80211 层的函数的调用频率（对所有网络包都添加额外开销）：
funccount 'ieee80211_*'

# 统计所有 ixgbevf 设备驱动函数的调用频率（对 ixgbevf 驱动添加额外开销）：
funccount 'ixgbevf_*'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

# 按 PID 和进程名统计套接字 accept (2) 调用：
bpftrace -e 't:syscalls:sys_enter_accept* { @[pid, comm] = count(); }'

# 按 PID 和进程名统计套接字 connect (2) 调用：
bpftrace -e 't:syscalls:sys_enter_connect { @[pid, comm] = count(); }'

# 按进程名和错误代码统计失败的 connect (2) 调用：
bpftrace -e 't:syscalls:sys_exit_connect /args->ret < 0/ { @[comm, - args->ret] = count(); }'

# 按用户态调用栈统计套接字 connect (2) 调用：
bpftrace -e 't:syscalls:sys_enter_connect { @[ustack] = count(); }'

# 按发送 / 接收、在 CPU 上运行的、进程名统计套接字发送和接收的次数：
bpftrace -e 'k:sock_sendmsg,k:sock_recvmsg { @[func, pid, comm] = count(); }'

# 按在 CPU 上运行的 PID 和进程名统计套接字发送和接收的字节数：
bpftrace -e 'kr:sock_sendmsg,kr:sock_recvmsg /(int32)retval > 0/ { @[pid, comm] = sum((int32)retval); }'

# 按在 CPU 上运行的 PID 和进程名统计 TCP connect 调用：
bpftrace -e 'k:tcp_v*_connect { @[pid, comm] = count(); }'

# 按在 CPU 上运行的 PID 和进程名统计 TCP accept 调用：
bpftrace -e 'k:inet_csk_accept { @[pid, comm] = count(); }'

# 统计 TCP 的发送和接收次数：
bpftrace -e 'k:tcp_sendmsg,k:tcp*recvmsg { @[func] = count(); }'

# 按在 CPU 上运行的 PID 和进程名统计 TCP 发送 / 接收的次数：
bpftrace -e 'k:tcp_sendmsg,k:tcp_recvmsg { @[func, pid, comm] = count(); }'

# 以直方图形式统计 TCP 发送的字节数：
bpftrace -e 'k:tcp_sendmsg { @send_bytes = hist(arg2); }'

# 以直方图形式统计 TCP 接收的字节数：
bpftrace -e 'kr:tcp_recvmsg /retval >= 0/ { @recv_bytes = hist(retval); }'

# 按类型与远端主机（仅支持 IPv4）统计 TCP 重传：
bpftrace -e 't:tcp:tcp_retransmit_* { @[probe, ntop(2, args->saddr)] = count(); }'

# 统计所有的 TCP 函数的调用频率（对所有的 TCP 函数添加额外开销）：
bpftrace -e 'k:tcp_* { @[func] = count(); }'

# 按在 CPU 上运行的 PID 和进程名统计 UDP 发送和接收的次数：
bpftrace -e 'k:udp*_sendmsg,k:udp*_recvmsg { @[func, pid, comm] = count(); }'

# 以直方图形式统计 UDP 发送的字节数：
bpftrace -e 'k:udp_sendmsg { @send_bytes = hist(arg2); }'

# 以直方图形式统计 UDP 接收的字节数：
bpftrace -e 'kr:udp_recvmsg /retval >= 0/ { @recv_bytes = hist(retval); }'

# 统计所有的 UDP 函数的调用频率（为所有 UDP 包增加额外开销）：
bpftrace -e 'k:udp_* { @[func] = count(); }'

# 统计发送数据包时的内核态调用栈：
bpftrace -e 't:net:net_dev_xmit { @[kstack] = count(); }'

# 按每个设备进行 CPU 直方图统计：
bpftrace -e 't:net:netif_receive_skb { @[str(args->name)] = lhist(cpu, 0, 128, 1); }'

# 统计所有 ieee80211 层函数的调用频率（对所有数据包增加额外开销）：
bpftrace -e 'k:ieee80211_* { @[func] = count(); }'

# 统计所有 ixgbevf 设备驱动函数的调用频率（对所有 ixgbevf 添加额外开销）：
bpftrace -e 'k:ixgbevf_* { @[func] = count(); }'

# 统计所有 iwl 设备驱动中的跟踪点的调用频率（对 iwl 添加额外开销）：
bpftrace -e 't:iwlwifi:*;t:iwlwifi_io:* { @[probe] = count(); }'

1
2
3
4
5
6
7
8

#为 PID 为 1234 的进程统计安全审计事件数：
funccount -p 1234 'security_*'

# 跟踪可插入身份验证模块（PAM）会话的开始：
trace 'pam:pam_start "%s: %s", arg1, arg2'

# 跟踪内核模块加载：
trace 't:module:module_load "load: %s", args->name'

1
2
3
4
5
6
7
8

#为 PID 为 1234 的进程统计安全审计事件数：
bpftrace -e 'k:security_* /pid == 1234 { @[func] = count(); }'

# 跟踪可插入身份验证模块（PAM）会话的开始：
bpftrace -e 'u:/lib/x86_64-linux-gnu/libpam.so.0:pam_start { printf("%s: %s\n", str(arg0), str(arg1)); }'

# 跟踪内核模块加载：
bpftrace -e 't:module:module_load { printf("load: %s\n", str(args->name)); }'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

#!/usr/local/bin/bpftrace

BEGIN
{
    printf("Tracing bash commands... Hit Ctrl-C to end.\n");
    printf("%-9s %-6s %s\n", "TIME", "PID", "COMMAND");
}

uretprobe:/bin/bash:readline
{
    time("%H:%M:%S ");
    printf("%-6d %s\n", pid, str(retval));
}