????

??? DTrace ? SDT ?????????????????SDT ?????? FBT ?????? PID ??????????DTrace ??????????????SDT ?????????????????DTrace ???????????????

SDT ???????

SDT(Statically Defined Tracing) ??????????????? DTrace ???????????????????????SDT ????????? DTrace ??????????????????????????????????? FBT ?????????????????????????????????????????? DTrace ???????????????????????????? DTrace ?????????????????????????????????????????????????? ????? DTrace ?????????????????????????????????????????????????? ?????????????????????????????????????????????????? ???????????SDT ?????????????????????????????????????????????????? ?????????????????????????????????????????????????? ????????????Statically Defined Tracing ???????????????????????????

SDT ?????????????????????????????????????????????????? ????????????????????????????????Solaris ????????? SDT ?????????????????????????????? IP ????? ? SDT ?????????????? Sched ?????? IO ?????? SDT ??????????????????????

SDT ?????????

SDT ?????????????? dtrace ????? -lP sdt ???????????????????????Solaris 10 10/09 ?? IO, Sched, Proc ?????? SDT ?????????????????????????????????????????????????? ??????????????

*** SDT ???????????????? *** # dtrace -lP sdt,io,sched,proc ID PROVIDER MODULE FUNCTION NAME 2679 sdt unix cpupm_utilization_event cpupm-ti-ungoverned 2688 sdt unix page_get_replacement_page page-get 2689 sdt unix page_get_cachelist page-get 2690 sdt unix page_get_freelist page-get 2701 sdt unix cpu_set_curr_clock cpu-change-speed 2702 sdt unix cpupm_utilization_event cpupm-lower-req 2703 sdt unix ecc_page_zero page_zero_result 2704 sdt unix cpupm_utilization_event cpupm-tw-governed 2705 sdt unix cpupm_utilization_event cpupm-ti-governed ...?????????????????????????????????????? 864 ?? SDT ???????????????????????? OS ???????????????????????????????????????????

*** SDT ????????????? 864 ? *** # dtrace -lP sdt,io,sched,proc | wc -l 865 tq_threadlock, RW_READER); 1503 start = gethrtime(); 1504 DTRACE_PROBE2(taskq__exec__start, taskq_t *, tq, tqent_func(tqe->tqent_arg); 1507 DTRACE_PROBE2(taskq__exec__end, taskq_t *, tq, 1508 taskq_ent_t *, tqe); 1509 end = gethrtime(); 1510 rw_exit(&tq->tq_threadlock);?????? taskq-exec-start ????????????????????????????????????dtrace ????????? 10 ????kmem_cache_reap() ??? 385 ??callout_execute() ??? 515 ????????????????taskq-exec-start ????????????? arg1 ??? taskq_ent_t * ?????????????????????????????????????????????????? ??????????????????? DTrace ? sym() ?????????????????????arg1 ?????????????????????????????????????????? DTrace ?????????????????SDT ????????????????????????????????????

*** SDT ????????? 2 *** # dtrace -qn 'sdt:::taskq-exec-start {@agg[sym((uintptr_t)((taskq_ent_t *)arg1) ->tqent_func)]=count()} tick-10sec{printa(@agg);exit(0)}' rpcmod`endpnt_reclaim 1 genunix`kmem_reap_done 1 genunix`kmem_reap_start 1 genunix`kmem_cache_reap 385 genunix`callout_execute 515 ~~~~~~~~~~~~~~~ ?????? ~~~ ????????? SDT ??????????????? ZFS ? ARC ??????????????????????????????????????? / ??????????????????????????SDT ?????????????????????????????????????arc-hit, arc-miss ???????????? /usr/src/uts/common/fs/zfs/arc.c ??????

*** SDT ????????? 3 *** # dtrace -qn 'BEGIN{printf("hit\tmiss\n")} sdt:::arc-hit{hit++} sdt:::arc-miss{miss++} tick-1sec{printf("%d\t%d\n", hit,miss);hit = miss =0} tick-5sec{exit(0)}' hit miss 2035 60 0 0 1436 10 886 99 1061 164Sched ??????????????????? Sched ?????? preempt ?????????????????????????? CPU ??????????preempt ???????????? /usr/src/uts/common/disp/disp.c#687 ??????

*** Sched ????????? *** # dtrace -qn 'BEGIN{printf("\tcpu\t\tnum\n")} sched:::preempt{@agg[cpu]=count()} tick-1sec{printa(@agg); clear(@agg)} tick-5sec{exit(0)}' cpu num 3 1 2 4 1 28 0 55 3 0 2 1 1 27 0 46 3 0 2 1 1 12 0 19 3 0 2 1 0 25 1 36 2 0 3 1 0 29 1 36 SDT ?????????

????? SDT ??????????????? arc-miss ?????????????????????

*** arc-miss ???? *** # dtrace -ln 'sdt:zfs:arc_read_nolock:arc-miss' ID PROVIDER MODULE FUNCTION NAME 5628 sdt zfs arc_read_nolock arc-miss DTRACE_PROBE ???

SDT ???????????????????? SDT ?????????????????????????????????????????????????? ????? /usr/src/uts/common/sys/sdt.h ?????????????????????????????????????????????????? ? DTRACE_PROBE ????????????? DTRACE_PROBE1 ????????? 5 ??? DTRACE_PROBE5 ???6 ?????????????????????????????????????????SDT ????????????????

*** DTRACE_PROBE ??? *** 75 #else /* _KERNEL */ 76 77 #define DTRACE_PROBE(name) { \ 78 extern void __dtrace_probe_##name(void); \ 79 __dtrace_probe_##name(); \ 80 } 81 82 #define DTRACE_PROBE1(name, type1, arg1) { \ 83 extern void __dtrace_probe_##name(uintptr_t); \ 84 __dtrace_probe_##name((uintptr_t)(arg1)); \ 85 } ...????????????????????????????arc-miss ???????????? /usr/src/uts/common/fs/zfs/arc.c#2797 ??????"DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp, uint64_t, size, zbookmark_t *, zb);" ???????????????????? "arc__miss" ????????????"__" ? DTrace ???? "-" ????????????????????????????????????????? "-" ? "__" ????????????

*** arc-miss ??????????? *** 2794 mutex_exit(hash_lock); 2795 2796 ASSERT3U(hdr->b_size, ==, size); 2797 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp, b_flags & ARC_PREFETCH), 2801 demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 2802 data, metadata, misses);DTRACE_PROBE4 ????????? 4 ?????????????????????????????????DTRACE_PROBE4 ??????? 9 ?????????????????????????????????????"arc__miss" ??????? "arc-miss" ????????????????2 ?????????????????????????????????????????????????? ?????????????????????????????????????????????????? ??????????????? DTrace ???? argN ????????????????? DTRACE_PROBE4 ????? 2 ??? "arc_buf_hdr_t *" ?? 3 ??? "hdr" ????arc-miss ??????????????? "arc_buf_hdr_t *" ?? "hdr" ??????????? arc-miss ???????????? arg0 ??????????????

?????? 4 ??? "blkptr_t *" ?? 5 ??? "bp" ????arc-miss ????? arg1 ??? "blkptr_t *" ?? "bp" ??????????????? arg2 ?? "uint64_t" ?? "size" ??????arg3 ?? "zbookmark_t *" ?? "zb" ???????????????????? arc-miss ????? arg2 ?????????????????????? "uint64_t" ?? "size" ??????????????????

*** ??????????????? argN *** # dtrace -qn 'sdt:::arc-miss{printf("%u\n", arg2)}' 2890529992504 2890529992504 2890529992504 2890529992504 2890529992504 __dtrace_probe_[name] ????

DTRACE_PROBE ??????????????????????? __dtrace_probe_[name] ??????????????????????[name] ? DTRACE_PROBE ????? 1 ???????????????????????????????????????????

arc-miss ???????? __dtrace_probe_arc__miss ??????????????????"arc__miss" ????????????????????????????????nm ????? __dtrace_probe_arc__miss ????????? UNDEF ????????????????????????????????????

*** __dtrace_probe_arc__miss ? UNDEF *** # nm /kernel/drv/amd64/zfs | grep __dtrace_probe_arc__miss [2672] | 0| 0|NOTY |GLOB |0 |UNDEF |__dtrace_probe_arc__miss ???????

__dtrace_probe_[name] ?????????????????????????????????????????????????? ??????? krtld ???????????????????????? x86_64 Solaris ???? /usr/src/uts/intel/amd64/krtld/kobj_reloc.c ??????????instr ???????????????????????????????? 1 ?????????? 5 ???? nop ????????????

*** ????????? nop x5 ????????? *** 86 #define SDT_NOP 0x90 87 #define SDT_NOPS 5 88 89 static int 90 sdt_reloc_resolve(struct module *mp, char *symname, uint8_t *instr) 91 { ... 116 for (i = 0; i < SDT_NOPS; i++) 117 instr[i - 1] = SDT_NOP; sdp_patchpoint = sdp->sdp_patchval; sdp_next; 310 }'sdt:zfs:arc_read_nolock:arc-miss' ???????????????????????????

*** arc-miss ???????? *** # dtrace -n 'sdt:zfs:arc_read_nolock:arc-miss' dtrace: description 'sdt:zfs:arc_read_nolock:arc-miss' matched 1 probe?? mdb ?????????????? nop ???? lock nop ?????????????????????????????????????????????????? ???????????

*** arc_read_nolock+0x3e3 ? lock nop ??????? *** # mdb -k Loading modules: [ unix krtld genunix specfs dtrace cpu.generic cpu_ms.AuthenticAMD.15 uppc pcplusmp ufs mpt ip hook neti sctp arp usba fcp fctl nca lofs zfs md cpc random crypto fcip logindmux ptm sppp nfs ] > arc_read_nolock+0x3e3::dis arc_read_nolock+0x3b9: call +0x35ed7 arc_read_nolock+0x3be: testl %eax,%eax arc_read_nolock+0x3c0: je +0x442 arc_read_nolock+0x3c6: movq $0x0,-0x60(%rbp) arc_read_nolock+0x3ce: movq -0x30(%rbp),%rdi arc_read_nolock+0x3d2: call +0xc1ee4ce arc_read_nolock+0x3d7: movq 0x20(%rbp),%rdx arc_read_nolock+0x3db: movq -0x58(%rbp),%rsi arc_read_nolock+0x3df: movq %r12,%rdi arc_read_nolock+0x3e2: nop arc_read_nolock+0x3e3: lock nop dtih_func(addr, stack, eax)) != 0) 52 return (rval); 53 } 54 55 return (0); 56 }??? DTrace ???????????? CPU ?????????????????????????????????????????????????? ?????????????????????????????????????????????????? ????? nop ????????????????????????????????????? nop ?????????????????????????????????????????? DTrace ????????????????????

???
  • ??????? DTRACE_PROBE ????????? SDT ????????????????
  • DTRACE_PROBE ?????????????? __dtrace_probe_[name] ??????????????????????
  • ?????????????__dtrace_probe_[name] ???????????????????? nop ? 5 ??????????
  • DTrace ???????????? 2 ??? nop ? lock ???????????
  • lock nop ? #UD Exception ??????????????? DTrace ????????????
  • ??? DTrace ???????????????? x86_64 ???? SDT ???????????
SDT ?????????????

DTrace ?????????????????????????????????????????????????? SDT ????????????????????????????SDT ??????? nop ??????????????????????????????????????????????? 2 ??????? disable ???????????????????? disabled probe effect ?????????

????nop ?????????????????????????????????????????????????? ?????????????????????????????????????? Solaris ??????? SDT ?????????????????????????????????????????????????? ?????????????

????????????????????????? DTrace ??????????????????????????????????????????????????

????

???SDT ???????????????????????????????SDT ??????? DTrace ???????????????????????????????????????????

????????? SDT ?????????????????SDT ?????????????????????????????????????????? SDT ????????????????????????????????

????
??????

More...