| Demonstrations of offcputime, the Linux eBPF/bcc version. | |
| This program shows stack traces that were blocked, and the total duration they | |
| were blocked. It works by tracing when threads block and when they return to | |
| CPU, measuring both the time they were blocked (aka the "off-CPU time") and the | |
| blocked stack trace and the task name. This data is summarized in kernel by | |
| summing the blocked time by unique stack trace and task name. | |
| Here is some example output. The -K option was used to only match kernel stacks. | |
| To explain what we are seeing: the very first stack trace looks like a page | |
| fault (do_page_fault() etc) from the "chmod" command, and in total was off-CPU | |
| for 13 microseconds. | |
| # ./offcputime -K | |
| Tracing off-CPU time (us) of all threads by kernel stack... Hit Ctrl-C to end. | |
| ^C | |
| schedule | |
| schedule_timeout | |
| io_schedule_timeout | |
| bit_wait_io | |
| __wait_on_bit | |
| wait_on_page_bit_killable | |
| __lock_page_or_retry | |
| filemap_fault | |
| __do_fault | |
| handle_mm_fault | |
| __do_page_fault | |
| do_page_fault | |
| page_fault | |
| chmod | |
| 13 | |
| schedule | |
| rcu_nocb_kthread | |
| kthread | |
| ret_from_fork | |
| ddebug_tables | |
| rcuos/0 | |
| 22 | |
| schedule | |
| schedule_timeout | |
| io_schedule_timeout | |
| bit_wait_io | |
| __wait_on_bit_lock | |
| __lock_page | |
| lock_page | |
| __do_fault | |
| handle_mm_fault | |
| __do_page_fault | |
| do_page_fault | |
| page_fault | |
| run | |
| 27 | |
| schedule | |
| schedule_timeout | |
| io_schedule_timeout | |
| bit_wait_io | |
| __wait_on_bit | |
| wait_on_page_bit_killable | |
| __lock_page_or_retry | |
| filemap_fault | |
| __do_fault | |
| handle_mm_fault | |
| __do_page_fault | |
| do_page_fault | |
| page_fault | |
| clear_user | |
| padzero | |
| load_elf_binary | |
| search_binary_handler | |
| load_script | |
| search_binary_handler | |
| do_execveat_common.isra.27 | |
| run | |
| 28 | |
| schedule | |
| schedule_timeout | |
| io_schedule_timeout | |
| bit_wait_io | |
| __wait_on_bit | |
| wait_on_page_bit_killable | |
| __lock_page_or_retry | |
| filemap_fault | |
| __do_fault | |
| handle_mm_fault | |
| __do_page_fault | |
| do_page_fault | |
| page_fault | |
| run | |
| 82 | |
| schedule | |
| pipe_wait | |
| pipe_read | |
| __vfs_read | |
| vfs_read | |
| sys_read | |
| entry_SYSCALL_64_fastpath | |
| bash | |
| 94 | |
| schedule | |
| rcu_gp_kthread | |
| kthread | |
| ret_from_fork | |
| ddebug_tables | |
| rcu_sched | |
| 104 | |
| schedule | |
| schedule_timeout | |
| io_schedule_timeout | |
| bit_wait_io | |
| __wait_on_bit | |
| out_of_line_wait_on_bit | |
| __wait_on_buffer | |
| jbd2_journal_commit_transaction | |
| kjournald2 | |
| kthread | |
| ret_from_fork | |
| mb_cache_list | |
| jbd2/xvda1-8 | |
| 986 | |
| schedule | |
| schedule_timeout | |
| io_schedule_timeout | |
| bit_wait_io | |
| __wait_on_bit | |
| out_of_line_wait_on_bit | |
| __wait_on_buffer | |
| jbd2_journal_commit_transaction | |
| kjournald2 | |
| kthread | |
| ret_from_fork | |
| mb_cache_list | |
| jbd2/xvda1-8 | |
| 6630 | |
| schedule | |
| schedule_timeout | |
| io_schedule_timeout | |
| bit_wait_io | |
| __wait_on_bit | |
| out_of_line_wait_on_bit | |
| do_get_write_access | |
| jbd2_journal_get_write_access | |
| __ext4_journal_get_write_access | |
| ext4_mb_mark_diskspace_used | |
| ext4_mb_new_blocks | |
| ext4_ext_map_blocks | |
| ext4_map_blocks | |
| ext4_writepages | |
| do_writepages | |
| __filemap_fdatawrite_range | |
| filemap_flush | |
| ext4_alloc_da_blocks | |
| ext4_rename | |
| ext4_rename2 | |
| supervise | |
| 6645 | |
| schedule | |
| schedule_timeout | |
| io_schedule_timeout | |
| bit_wait_io | |
| __wait_on_bit | |
| out_of_line_wait_on_bit | |
| do_get_write_access | |
| jbd2_journal_get_write_access | |
| __ext4_journal_get_write_access | |
| __ext4_new_inode | |
| ext4_create | |
| vfs_create | |
| path_openat | |
| do_filp_open | |
| do_sys_open | |
| sys_open | |
| entry_SYSCALL_64_fastpath | |
| supervise | |
| 12702 | |
| schedule | |
| rcu_nocb_kthread | |
| kthread | |
| ret_from_fork | |
| rcuos/2 | |
| 16036 | |
| schedule | |
| rcu_nocb_kthread | |
| kthread | |
| ret_from_fork | |
| rcuos/4 | |
| 24085 | |
| schedule | |
| do_wait | |
| sys_wait4 | |
| entry_SYSCALL_64_fastpath | |
| run | |
| 233055 | |
| schedule | |
| schedule_timeout | |
| io_schedule_timeout | |
| bit_wait_io | |
| __wait_on_bit | |
| wait_on_page_bit | |
| truncate_inode_pages_range | |
| truncate_inode_pages_final | |
| ext4_evict_inode | |
| evict | |
| iput | |
| __dentry_kill | |
| dput | |
| sys_rename | |
| entry_SYSCALL_64_fastpath | |
| supervise | |
| 297113 | |
| schedule | |
| schedule_timeout | |
| wait_woken | |
| n_tty_read | |
| tty_read | |
| __vfs_read | |
| vfs_read | |
| sys_read | |
| entry_SYSCALL_64_fastpath | |
| bash | |
| 1789866 | |
| schedule | |
| schedule_timeout | |
| io_schedule_timeout | |
| do_blockdev_direct_IO | |
| __blockdev_direct_IO | |
| blkdev_direct_IO | |
| generic_file_read_iter | |
| blkdev_read_iter | |
| __vfs_read | |
| vfs_read | |
| sys_read | |
| entry_SYSCALL_64_fastpath | |
| dd | |
| 3310763 | |
| schedule | |
| smpboot_thread_fn | |
| kthread | |
| ret_from_fork | |
| watchdog/1 | |
| 3999989 | |
| schedule | |
| smpboot_thread_fn | |
| kthread | |
| ret_from_fork | |
| watchdog/5 | |
| 3999995 | |
| schedule | |
| smpboot_thread_fn | |
| kthread | |
| ret_from_fork | |
| watchdog/4 | |
| 3999996 | |
| schedule | |
| smpboot_thread_fn | |
| kthread | |
| ret_from_fork | |
| watchdog/0 | |
| 3999996 | |
| schedule | |
| smpboot_thread_fn | |
| kthread | |
| ret_from_fork | |
| watchdog/3 | |
| 3999998 | |
| schedule | |
| smpboot_thread_fn | |
| kthread | |
| ret_from_fork | |
| watchdog/7 | |
| 3999999 | |
| schedule | |
| smpboot_thread_fn | |
| kthread | |
| ret_from_fork | |
| watchdog/2 | |
| 4000001 | |
| schedule | |
| smpboot_thread_fn | |
| kthread | |
| ret_from_fork | |
| watchdog/6 | |
| 4000001 | |
| schedule | |
| do_wait | |
| sys_wait4 | |
| entry_SYSCALL_64_fastpath | |
| bash | |
| 4039675 | |
| schedule | |
| do_nanosleep | |
| hrtimer_nanosleep | |
| sys_nanosleep | |
| entry_SYSCALL_64_fastpath | |
| svscan | |
| 5000112 | |
| schedule | |
| schedule_hrtimeout_range_clock | |
| schedule_hrtimeout_range | |
| poll_schedule_timeout | |
| do_select | |
| core_sys_select | |
| sys_select | |
| entry_SYSCALL_64_fastpath | |
| snmpd | |
| 5998761 | |
| schedule | |
| smpboot_thread_fn | |
| kthread | |
| ret_from_fork | |
| migration/3 | |
| 6149779 | |
| schedule | |
| schedule_hrtimeout_range_clock | |
| schedule_hrtimeout_range | |
| poll_schedule_timeout | |
| do_select | |
| core_sys_select | |
| sys_select | |
| entry_SYSCALL_64_fastpath | |
| ntpd | |
| 6999832 | |
| schedule | |
| worker_thread | |
| kthread | |
| ret_from_fork | |
| kworker/u16:2 | |
| 7131941 | |
| schedule | |
| worker_thread | |
| kthread | |
| ret_from_fork | |
| kworker/3:0 | |
| 7999844 | |
| schedule | |
| worker_thread | |
| kthread | |
| ret_from_fork | |
| kworker/1:1 | |
| 7999872 | |
| schedule | |
| worker_thread | |
| kthread | |
| ret_from_fork | |
| kworker/2:1 | |
| 7999889 | |
| schedule | |
| worker_thread | |
| kthread | |
| ret_from_fork | |
| kworker/5:1 | |
| 7999936 | |
| schedule | |
| worker_thread | |
| kthread | |
| ret_from_fork | |
| kworker/7:1 | |
| 7999938 | |
| schedule | |
| worker_thread | |
| kthread | |
| ret_from_fork | |
| kworker/6:1 | |
| 7999940 | |
| schedule | |
| do_nanosleep | |
| hrtimer_nanosleep | |
| sys_nanosleep | |
| entry_SYSCALL_64_fastpath | |
| tail | |
| 8000905 | |
| schedule | |
| smpboot_thread_fn | |
| kthread | |
| ret_from_fork | |
| migration/7 | |
| 8197046 | |
| schedule | |
| pipe_wait | |
| pipe_read | |
| __vfs_read | |
| vfs_read | |
| sys_read | |
| entry_SYSCALL_64_fastpath | |
| readproctitle | |
| 8197835 | |
| schedule | |
| smpboot_thread_fn | |
| kthread | |
| ret_from_fork | |
| migration/4 | |
| 8201851 | |
| schedule | |
| smpboot_thread_fn | |
| kthread | |
| ret_from_fork | |
| migration/2 | |
| 8203375 | |
| schedule | |
| smpboot_thread_fn | |
| kthread | |
| ret_from_fork | |
| migration/6 | |
| 8208664 | |
| schedule | |
| smpboot_thread_fn | |
| kthread | |
| ret_from_fork | |
| migration/5 | |
| 8209819 | |
| schedule | |
| smpboot_thread_fn | |
| kthread | |
| ret_from_fork | |
| ddebug_tables | |
| migration/0 | |
| 8211292 | |
| schedule | |
| smpboot_thread_fn | |
| kthread | |
| ret_from_fork | |
| migration/1 | |
| 8212100 | |
| schedule | |
| worker_thread | |
| kthread | |
| ret_from_fork | |
| kworker/0:2 | |
| 8270305 | |
| schedule | |
| rcu_nocb_kthread | |
| kthread | |
| ret_from_fork | |
| rcuos/3 | |
| 8349697 | |
| schedule | |
| rcu_nocb_kthread | |
| kthread | |
| ret_from_fork | |
| rcuos/2 | |
| 8363357 | |
| schedule | |
| rcu_nocb_kthread | |
| kthread | |
| ret_from_fork | |
| rcuos/1 | |
| 8365338 | |
| schedule | |
| schedule_timeout | |
| xfs_buf_terminate | |
| kthread | |
| ret_from_fork | |
| xfsaild/md0 | |
| 8371514 | |
| schedule | |
| rcu_nocb_kthread | |
| kthread | |
| ret_from_fork | |
| rcuos/4 | |
| 8384013 | |
| schedule | |
| rcu_nocb_kthread | |
| kthread | |
| ret_from_fork | |
| rcuos/5 | |
| 8390016 | |
| schedule | |
| rcu_nocb_kthread | |
| kthread | |
| ret_from_fork | |
| ddebug_tables | |
| rcuos/0 | |
| 8405428 | |
| schedule | |
| schedule_timeout | |
| rcu_gp_kthread | |
| kthread | |
| ret_from_fork | |
| ddebug_tables | |
| rcu_sched | |
| 8406930 | |
| schedule | |
| rcu_nocb_kthread | |
| kthread | |
| ret_from_fork | |
| rcuos/7 | |
| 8409575 | |
| schedule | |
| rcu_nocb_kthread | |
| kthread | |
| ret_from_fork | |
| rcuos/6 | |
| 8415062 | |
| schedule | |
| schedule_hrtimeout_range_clock | |
| schedule_hrtimeout_range | |
| poll_schedule_timeout | |
| do_select | |
| core_sys_select | |
| sys_select | |
| entry_SYSCALL_64_fastpath | |
| offcputime | |
| 8421478 | |
| schedule | |
| worker_thread | |
| kthread | |
| ret_from_fork | |
| kworker/4:0 | |
| 8421492 | |
| schedule | |
| schedule_hrtimeout_range_clock | |
| schedule_hrtimeout_range | |
| poll_schedule_timeout | |
| do_select | |
| core_sys_select | |
| sys_select | |
| entry_SYSCALL_64_fastpath | |
| sshd | |
| 14249005 | |
| schedule | |
| schedule_hrtimeout_range_clock | |
| schedule_hrtimeout_range | |
| poll_schedule_timeout | |
| do_sys_poll | |
| sys_poll | |
| entry_SYSCALL_64_fastpath | |
| supervise | |
| 81670888 | |
| The last few stack traces aren't very interesting, since they are threads that | |
| are often blocked off-CPU waiting for work. | |
| Do be somewhat careful with overhead: this is tracing scheduler functions, which | |
| can be called very frequently. While this uses in-kernel summaries for | |
| efficiency, the rate of scheduler functions can be very high (> 1,000,000/sec), | |
| and this is performing stack walks when threads return to CPU. At some point | |
| the overhead will be measurable. | |
| A -p option can be used to filter (in-kernel) on a single process ID. For | |
| example, only matching PID 26651, which is a running "dd" command: | |
| # ./offcputime -K -p 26651 | |
| Tracing off-CPU time (us) of all threads by kernel stack... Hit Ctrl-C to end. | |
| ^C | |
| schedule | |
| schedule_timeout | |
| io_schedule_timeout | |
| do_blockdev_direct_IO | |
| __blockdev_direct_IO | |
| blkdev_direct_IO | |
| generic_file_read_iter | |
| blkdev_read_iter | |
| __vfs_read | |
| vfs_read | |
| sys_read | |
| entry_SYSCALL_64_fastpath | |
| dd | |
| 2405710 | |
| The stack trace shows "dd" is blocked waiting on disk I/O, as expected, for a | |
| total of 2.4 seconds during tracing. | |
| A duration can be added, for example, tracing for 5 seconds only: | |
| # ./offcputime -K -p 26651 5 | |
| Tracing off-CPU time (us) of all threads by kernel stack for 5 secs. | |
| schedule | |
| schedule_timeout | |
| io_schedule_timeout | |
| do_blockdev_direct_IO | |
| __blockdev_direct_IO | |
| blkdev_direct_IO | |
| generic_file_read_iter | |
| blkdev_read_iter | |
| __vfs_read | |
| vfs_read | |
| sys_read | |
| entry_SYSCALL_64_fastpath | |
| dd | |
| 4413909 | |
| Here, dd was blocked for 4.4 seconds out of 5. Or put differently, likely | |
| on-CPU for about 12% of the time. Which matches the ratio seen by time(1): | |
| # time dd if=/dev/md0 iflag=direct of=/dev/null bs=1k | |
| ^C108115+0 records in | |
| 108114+0 records out | |
| 110708736 bytes (111 MB) copied, 13.7565 s, 8.0 MB/s | |
| real 0m13.760s | |
| user 0m0.000s | |
| sys 0m1.739s | |
| A -f option will emit output using the "folded stacks" format, which can be | |
| read directly by flamegraph.pl from the FlameGraph open source software | |
| (https://github.com/brendangregg/FlameGraph). Eg: | |
| # ./offcputime -K -f 5 | |
| bash;entry_SYSCALL_64_fastpath;sys_read;vfs_read;__vfs_read;tty_read;n_tty_read;call_rwsem_down_read_failed;rwsem_down_read_failed;schedule 8 | |
| yes;entry_SYSCALL_64_fastpath;sys_write;vfs_write;__vfs_write;tty_write;n_tty_write;call_rwsem_down_read_failed;rwsem_down_read_failed;schedule 14 | |
| run;page_fault;do_page_fault;__do_page_fault;handle_mm_fault;__do_fault;filemap_fault;__lock_page_or_retry;wait_on_page_bit_killable;__wait_on_bit;bit_wait_io;io_schedule_timeout;schedule_timeout;schedule 33 | |
| rcuos/4;ret_from_fork;kthread;rcu_nocb_kthread;schedule 45 | |
| bash;entry_SYSCALL_64_fastpath;sys_read;vfs_read;__vfs_read;pipe_read;pipe_wait;schedule 88 | |
| run;page_fault;do_page_fault;__do_page_fault;handle_mm_fault;__do_fault;filemap_fault;__lock_page_or_retry;wait_on_page_bit_killable;__wait_on_bit;bit_wait_io;io_schedule_timeout;schedule_timeout;schedule 108 | |
| jbd2/xvda1-8;mb_cache_list;ret_from_fork;kthread;kjournald2;jbd2_journal_commit_transaction;__wait_on_buffer;out_of_line_wait_on_bit;__wait_on_bit;bit_wait_io;io_schedule_timeout;schedule_timeout;schedule 828 | |
| jbd2/xvda1-8;mb_cache_list;ret_from_fork;kthread;kjournald2;jbd2_journal_commit_transaction;__wait_on_buffer;out_of_line_wait_on_bit;__wait_on_bit;bit_wait_io;io_schedule_timeout;schedule_timeout;schedule 6201 | |
| supervise;entry_SYSCALL_64_fastpath;sys_rename;dput;__dentry_kill;iput;evict;ext4_evict_inode;truncate_inode_pages_final;truncate_inode_pages_range;wait_on_page_bit;__wait_on_bit;bit_wait_io;io_schedule_timeout;schedule_timeout;schedule 41049 | |
| run;entry_SYSCALL_64_fastpath;sys_wait4;do_wait;schedule 120709 | |
| bash;entry_SYSCALL_64_fastpath;sys_wait4;do_wait;schedule 699320 | |
| ksoftirqd/0;ret_from_fork;kthread;smpboot_thread_fn;schedule 1077529 | |
| bash;entry_SYSCALL_64_fastpath;sys_read;vfs_read;__vfs_read;tty_read;n_tty_read;wait_woken;schedule_timeout;schedule 1362045 | |
| sshd;entry_SYSCALL_64_fastpath;sys_select;core_sys_select;do_select;poll_schedule_timeout;schedule_hrtimeout_range;schedule_hrtimeout_range_clock;schedule 1377627 | |
| migration/0;ddebug_tables;ret_from_fork;kthread;smpboot_thread_fn;schedule 2040753 | |
| snmpd;entry_SYSCALL_64_fastpath;sys_select;core_sys_select;do_select;poll_schedule_timeout;schedule_hrtimeout_range;schedule_hrtimeout_range_clock;schedule 2197568 | |
| migration/5;ret_from_fork;kthread;smpboot_thread_fn;schedule 3079426 | |
| migration/7;ret_from_fork;kthread;smpboot_thread_fn;schedule 3084746 | |
| kworker/6:2;ret_from_fork;kthread;worker_thread;schedule 3940583 | |
| kworker/5:1;ret_from_fork;kthread;worker_thread;schedule 3944892 | |
| kworker/1:2;ret_from_fork;kthread;worker_thread;schedule 3999646 | |
| ntpd;entry_SYSCALL_64_fastpath;sys_select;core_sys_select;do_select;poll_schedule_timeout;schedule_hrtimeout_range;schedule_hrtimeout_range_clock;schedule 3999904 | |
| kworker/u16:0;ret_from_fork;kthread;worker_thread;schedule 3999967 | |
| kworker/7:0;ret_from_fork;kthread;worker_thread;schedule 3999987 | |
| tail;entry_SYSCALL_64_fastpath;sys_nanosleep;hrtimer_nanosleep;do_nanosleep;schedule 4000473 | |
| migration/1;ret_from_fork;kthread;smpboot_thread_fn;schedule 4091150 | |
| migration/4;ret_from_fork;kthread;smpboot_thread_fn;schedule 4095217 | |
| readproctitle;entry_SYSCALL_64_fastpath;sys_read;vfs_read;__vfs_read;pipe_read;pipe_wait;schedule 4108470 | |
| migration/3;ret_from_fork;kthread;smpboot_thread_fn;schedule 4109264 | |
| migration/2;ret_from_fork;kthread;smpboot_thread_fn;schedule 4109280 | |
| migration/6;ret_from_fork;kthread;smpboot_thread_fn;schedule 4111143 | |
| kworker/4:0;ret_from_fork;kthread;worker_thread;schedule 4402350 | |
| kworker/3:0;ret_from_fork;kthread;worker_thread;schedule 4433988 | |
| kworker/2:1;ret_from_fork;kthread;worker_thread;schedule 4636142 | |
| kworker/0:2;ret_from_fork;kthread;worker_thread;schedule 4832023 | |
| rcuos/1;ret_from_fork;kthread;rcu_nocb_kthread;schedule 4974186 | |
| rcuos/5;ret_from_fork;kthread;rcu_nocb_kthread;schedule 4977137 | |
| rcuos/6;ret_from_fork;kthread;rcu_nocb_kthread;schedule 4987769 | |
| rcuos/3;ret_from_fork;kthread;rcu_nocb_kthread;schedule 4992282 | |
| rcuos/4;ret_from_fork;kthread;rcu_nocb_kthread;schedule 4992364 | |
| rcuos/2;ret_from_fork;kthread;rcu_nocb_kthread;schedule 4992714 | |
| rcuos/0;ddebug_tables;ret_from_fork;kthread;rcu_nocb_kthread;schedule 4996504 | |
| rcuos/7;ret_from_fork;kthread;rcu_nocb_kthread;schedule 4998497 | |
| rcu_sched;ddebug_tables;ret_from_fork;kthread;rcu_gp_kthread;schedule_timeout;schedule 5000686 | |
| offcputime;entry_SYSCALL_64_fastpath;sys_select;core_sys_select;do_select;poll_schedule_timeout;schedule_hrtimeout_range;schedule_hrtimeout_range_clock;schedule 5005063 | |
| dd;entry_SYSCALL_64_fastpath;sys_read;vfs_read;__vfs_read;blkdev_read_iter;generic_file_read_iter;blkdev_direct_IO;__blockdev_direct_IO;do_blockdev_direct_IO;io_schedule_timeout;schedule_timeout;schedule 8025599 | |
| supervise;entry_SYSCALL_64_fastpath;sys_poll;do_sys_poll;poll_schedule_timeout;schedule_hrtimeout_range;schedule_hrtimeout_range_clock;schedule 40835611 | |
| The stack traces are shown as single lines, with functions separated by | |
| semicolons. The first entry is the task name. The 2nd column is the total | |
| off-CPU time. | |
| I'd save this output to a file, then move it to the system where you'll be | |
| creating your "off-CPU time flame graphs". | |
| USAGE message: | |
| # ./offcputime.py -h | |
| usage: offcputime.py [-h] [-p PID | -t TID | -u | -k] [-U | -K] [-d] [-f] | |
| [--stack-storage-size STACK_STORAGE_SIZE] | |
| [-m MIN_BLOCK_TIME] [-M MAX_BLOCK_TIME] [--state STATE] | |
| [duration] | |
| Summarize off-CPU time by stack trace | |
| positional arguments: | |
| duration duration of trace, in seconds | |
| optional arguments: | |
| -h, --help show this help message and exit | |
| -p PID, --pid PID trace this PID only | |
| -t TID, --tid TID trace this TID only | |
| -u, --user-threads-only | |
| user threads only (no kernel threads) | |
| -k, --kernel-threads-only | |
| kernel threads only (no user threads) | |
| -U, --user-stacks-only | |
| show stacks from user space only (no kernel space | |
| stacks) | |
| -K, --kernel-stacks-only | |
| show stacks from kernel space only (no user space | |
| stacks) | |
| -d, --delimited insert delimiter between kernel/user stacks | |
| -f, --folded output folded format | |
| --stack-storage-size STACK_STORAGE_SIZE | |
| the number of unique stack traces that can be stored | |
| and displayed (default 1024) | |
| -m MIN_BLOCK_TIME, --min-block-time MIN_BLOCK_TIME | |
| the amount of time in microseconds over which we store | |
| traces (default 1) | |
| -M MAX_BLOCK_TIME, --max-block-time MAX_BLOCK_TIME | |
| the amount of time in microseconds under which we | |
| store traces (default U64_MAX) | |
| --state STATE filter on this thread state bitmask (eg, 2 == | |
| TASK_UNINTERRUPTIBLE) see include/linux/sched.h | |
| examples: | |
| ./offcputime # trace off-CPU stack time until Ctrl-C | |
| ./offcputime 5 # trace for 5 seconds only | |
| ./offcputime -f 5 # 5 seconds, and output in folded format | |
| ./offcputime -m 1000 # trace only events that last more than 1000 usec | |
| ./offcputime -M 10000 # trace only events that last less than 10000 usec | |
| ./offcputime -p 185 # only trace threads for PID 185 | |
| ./offcputime -t 188 # only trace thread 188 | |
| ./offcputime -u # only trace user threads (no kernel) | |
| ./offcputime -k # only trace kernel threads (no user) | |
| ./offcputime -U # only show user space stacks (no kernel) | |
| ./offcputime -K # only show kernel space stacks (no user) |