I notice that ext3 and jbd tracepoints aren't available for current systemtap: # stap -p4 -e 'probe kernel.trace("*") {}' -w --vp 04 Pass 2: getting a tracepoint query for 1 headers: /lib/modules/2.6.32-195.el6.ppc64/build/include/trace/events/ext3.h Running make -C /lib/modules/2.6.32-195.el6.ppc64/build M=/tmp/stapf7FP1Y/tracequery_kmod_3 modules ARCH=powerpc V=1 make: Entering directory `/usr/src/kernels/2.6.32-195.el6.ppc64' test -e include/linux/autoconf.h -a -e include/config/auto.conf || ( \ echo; \ echo " ERROR: Kernel configuration is invalid."; \ echo " include/linux/autoconf.h or include/config/auto.conf are missing."; \ echo " Run 'make oldconfig && make prepare' on kernel src to fix it."; \ echo; \ /bin/false) mkdir -p /tmp/stapf7FP1Y/tracequery_kmod_3/.tmp_versions ; rm -f /tmp/stapf7FP1Y/tracequery_kmod_3/.tmp_versions/* make -f scripts/Makefile.build obj=/tmp/stapf7FP1Y/tracequery_kmod_3 gcc -m64 -Wp,-MD,/tmp/stapf7FP1Y/tracequery_kmod_3/.tracequery_kmod_3.o.d -nostdinc -isystem /usr/lib/gcc/ppc64-redhat-linux/4.4.6/include -Iinclude -I/usr/src/kernels/2.6.32-195.el6.ppc64/arch/powerpc/include -include include/linux/autoconf.h -D__KERNEL__ -Iarch/powerpc -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Wno-format-security -fno-delete-null-pointer-checks -O2 -msoft-float -pipe -Iarch/powerpc -mminimal-toc -mtraceback=none -mcall-aixdesc -mcpu=power4 -mno-altivec -mno-spe -mspe=no -funit-at-a-time -mno-string -Wa,-maltivec -Wframe-larger-than=2048 -fno-stack-protector -Wno-unused-but-set-variable -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -fno-strict-overflow -fno-dwarf2-cfi-asm -fconserve-stack -g -Wno-implicit-function-declaration -Werror -I/usr/src/debug////////kernel-2.6.32-195.el6/linux-2.6.32-195.el6.ppc64 -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(tracequery_kmod_3)" -D"KBUILD_MODNAME=KBUILD_STR(tracequery_kmod_3)" -D"DEBUG_HASH=26" -D"DEBUG_HASH2=39" -c -o /tmp/stapf7FP1Y/tracequery_kmod_3/.tmp_tracequery_kmod_3.o /tmp/stapf7FP1Y/tracequery_kmod_3/tracequery_kmod_3.c cc1: warnings being treated as errors In file included from /tmp/stapf7FP1Y/tracequery_kmod_3/tracequery_kmod_3.c:29: include/trace/events/ext3.h:450: error: ‘struct ext3_reserve_window_node’ declared inside parameter list include/trace/events/ext3.h:450: error: its scope is only this definition or declaration, which is probably not what you want include/trace/events/ext3.h:473: error: ‘struct ext3_reserve_window_node’ declared inside parameter list include/trace/events/ext3.h:519: error: ‘struct ext3_reserve_window_node’ declared inside parameter list make[1]: *** [/tmp/stapf7FP1Y/tracequery_kmod_3/tracequery_kmod_3.o] Error 1 make: *** [_module_/tmp/stapf7FP1Y/tracequery_kmod_3] Error 2 make: Leaving directory `/usr/src/kernels/2.6.32-195.el6.ppc64' Spawn waitpid result (0x200): 2 Warning: make exited with status: 2 and Pass 2: getting a tracepoint query for 1 headers: /lib/modules/2.6.32-195.el6.ppc64/build/include/trace/events/jbd.h Running make -C /lib/modules/2.6.32-195.el6.ppc64/build M=/tmp/stapf7FP1Y/tracequery_kmod_7 modules ARCH=powerpc V=1 make: Entering directory `/usr/src/kernels/2.6.32-195.el6.ppc64' test -e include/linux/autoconf.h -a -e include/config/auto.conf || ( \ echo; \ echo " ERROR: Kernel configuration is invalid."; \ echo " include/linux/autoconf.h or include/config/auto.conf are missing."; \ echo " Run 'make oldconfig && make prepare' on kernel src to fix it."; \ echo; \ /bin/false) mkdir -p /tmp/stapf7FP1Y/tracequery_kmod_7/.tmp_versions ; rm -f /tmp/stapf7FP1Y/tracequery_kmod_7/.tmp_versions/* make -f scripts/Makefile.build obj=/tmp/stapf7FP1Y/tracequery_kmod_7 gcc -m64 -Wp,-MD,/tmp/stapf7FP1Y/tracequery_kmod_7/.tracequery_kmod_7.o.d -nostdinc -isystem /usr/lib/gcc/ppc64-redhat-linux/4.4.6/include -Iinclude -I/usr/src/kernels/2.6.32-195.el6.ppc64/arch/powerpc/include -include include/linux/autoconf.h -D__KERNEL__ -Iarch/powerpc -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Wno-format-security -fno-delete-null-pointer-checks -O2 -msoft-float -pipe -Iarch/powerpc -mminimal-toc -mtraceback=none -mcall-aixdesc -mcpu=power4 -mno-altivec -mno-spe -mspe=no -funit-at-a-time -mno-string -Wa,-maltivec -Wframe-larger-than=2048 -fno-stack-protector -Wno-unused-but-set-variable -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -fno-strict-overflow -fno-dwarf2-cfi-asm -fconserve-stack -g -Wno-implicit-function-declaration -Werror -I/usr/src/debug////////kernel-2.6.32-195.el6/linux-2.6.32-195.el6.ppc64 -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(tracequery_kmod_7)" -D"KBUILD_MODNAME=KBUILD_STR(tracequery_kmod_7)" -D"DEBUG_HASH=34" -D"DEBUG_HASH2=39" -c -o /tmp/stapf7FP1Y/tracequery_kmod_7/.tmp_tracequery_kmod_7.o /tmp/stapf7FP1Y/tracequery_kmod_7/tracequery_kmod_7.c In file included from include/trace/events/jbd.h:7, from /tmp/stapf7FP1Y/tracequery_kmod_7/tracequery_kmod_7.c:29: include/linux/jbd.h:102: error: redefinition of typedef ‘handle_t’ include/linux/jbd2.h:96: note: previous declaration of ‘handle_t’ was here include/linux/jbd.h:116: error: redefinition of typedef ‘journal_t’ include/linux/jbd2.h:110: note: previous declaration of ‘journal_t’ was here include/linux/jbd.h:143: error: redefinition of ‘struct journal_header_s’ include/linux/jbd.h:147: error: conflicting types for ‘journal_header_t’ include/linux/jbd2.h:141: note: previous declaration of ‘journal_header_t’ was here include/linux/jbd.h:154: error: redefinition of ‘struct journal_block_tag_s’ include/linux/jbd.h:157: error: conflicting types for ‘journal_block_tag_t’ include/linux/jbd2.h:179: note: previous declaration of ‘journal_block_tag_t’ was here include/linux/jbd.h:181: error: redefinition of ‘struct journal_superblock_s’ include/linux/jbd.h:223: error: conflicting types for ‘journal_superblock_t’ include/linux/jbd2.h:248: note: previous declaration of ‘journal_superblock_t’ was here In file included from include/trace/events/jbd.h:7, from /tmp/stapf7FP1Y/tracequery_kmod_7/tracequery_kmod_7.c:29: include/linux/jbd.h:268:1: error: "__journal_expect" redefined In file included from /usr/src/debug////////kernel-2.6.32-195.el6/linux-2.6.32-195.el6.ppc64/fs/ext4/ext4.h:22, from /tmp/stapf7FP1Y/tracequery_kmod_7/tracequery_kmod_7.c:26: include/linux/jbd2.h:299:1: error: this is the location of the previous definition include/linux/jbd.h:283: error: nested redefinition of ‘enum jbd_state_bits’ include/linux/jbd.h:283: error: redeclaration of ‘enum jbd_state_bits’ include/linux/jbd.h:285: error: redeclaration of enumerator ‘BH_JBD’ include/linux/jbd2.h:317: note: previous definition of ‘BH_JBD’ was here include/linux/jbd.h:286: error: redeclaration of enumerator ‘BH_JWrite’ include/linux/jbd2.h:318: note: previous definition of ‘BH_JWrite’ was here include/linux/jbd.h:287: error: redeclaration of enumerator ‘BH_Freed’ include/linux/jbd2.h:319: note: previous definition of ‘BH_Freed’ was here include/linux/jbd.h:288: error: redeclaration of enumerator ‘BH_Revoked’ include/linux/jbd2.h:320: note: previous definition of ‘BH_Revoked’ was here include/linux/jbd.h:289: error: redeclaration of enumerator ‘BH_RevokeValid’ include/linux/jbd2.h:321: note: previous definition of ‘BH_RevokeValid’ was here include/linux/jbd.h:290: error: redeclaration of enumerator ‘BH_JBDDirty’ include/linux/jbd2.h:322: note: previous definition of ‘BH_JBDDirty’ was here include/linux/jbd.h:291: error: redeclaration of enumerator ‘BH_State’ include/linux/jbd2.h:323: note: previous definition of ‘BH_State’ was here include/linux/jbd.h:292: error: redeclaration of enumerator ‘BH_JournalHead’ include/linux/jbd2.h:324: note: previous definition of ‘BH_JournalHead’ was here include/linux/jbd.h:293: error: redeclaration of enumerator ‘BH_Unshadow’ include/linux/jbd2.h:325: note: previous definition of ‘BH_Unshadow’ was here include/linux/jbd.h:296: error: redefinition of ‘set_buffer_jbd’ include/linux/jbd2.h:329: note: previous definition of ‘set_buffer_jbd’ was here include/linux/jbd.h:296: error: redefinition of ‘clear_buffer_jbd’ include/linux/jbd2.h:329: note: previous definition of ‘clear_buffer_jbd’ was here include/linux/jbd.h:296: error: redefinition of ‘buffer_jbd’ include/linux/jbd2.h:329: note: previous definition of ‘buffer_jbd’ was here include/linux/jbd.h:297: error: redefinition of ‘set_buffer_jwrite’ include/linux/jbd2.h:330: note: previous definition of ‘set_buffer_jwrite’ was here include/linux/jbd.h:297: error: redefinition of ‘clear_buffer_jwrite’ ... .... include/linux/jbd2.h:1286:1: error: this is the location of the previous definition In file included from include/trace/events/jbd.h:7, from /tmp/stapf7FP1Y/tracequery_kmod_7/tracequery_kmod_7.c:29: include/linux/jbd.h:1091:1: error: "BJ_Types" redefined In file included from /usr/src/debug////////kernel-2.6.32-195.el6/linux-2.6.32-195.el6.ppc64/fs/ext4/ext4.h:22, from /tmp/stapf7FP1Y/tracequery_kmod_7/tracequery_kmod_7.c:26: include/linux/jbd2.h:1287:1: error: this is the location of the previous definition make[1]: *** [/tmp/stapf7FP1Y/tracequery_kmod_7/tracequery_kmod_7.o] Error 1 make: *** [_module_/tmp/stapf7FP1Y/tracequery_kmod_7] Error 2 make: Leaving directory `/usr/src/kernels/2.6.32-195.el6.ppc64' Spawn waitpid result (0x200): 2 Warning: make exited with status: 2 Looks like the familiar problem of bug 9993.
Created attachment 5920 [details] patch v1 With this patch, I can get ext3 and jbd tracepoints with 'statp -l 'kernel.trace("*")''.
I cannot replicate this on x86_64. Is this ppc64 specific?
(In reply to comment #2) > I cannot replicate this on x86_64. Is this ppc64 specific? This bug is occuring on all archs. I can replicate it on x86_64: Pass 2: getting a tracepoint query for 1 headers: /Warning: make exited with status: 2 Copying /tmp/stapobl4bG/tracequery_kmod_2/tracequery_kmod_2.ko to /root/.systemtap/cache/a2/tracequery_a2a924989dc36ad014e5b9a5eb45172e_648.ko Pass 2: getting a tracepoint query for 1 headers: /lib/modules/2.6.32-195.el6.x86_64/build/include/trace/events/ext3.h Running make -C /lib/modules/2.6.32-195.el6.x86_64/build M=/tmp/stapobl4bG/tracequery_kmod_3 modules ARCH=x86_64 V=1 make: Entering directory `/usr/src/kernels/2.6.32-195.el6.x86_64' test -e include/linux/autoconf.h -a -e include/config/auto.conf || ( \ echo; \ echo " ERROR: Kernel configuration is invalid."; \ echo " include/linux/autoconf.h or include/config/auto.conf are missing."; \ echo " Run 'make oldconfig && make prepare' on kernel src to fix it."; \ echo; \ /bin/false) mkdir -p /tmp/stapobl4bG/tracequery_kmod_3/.tmp_versions ; rm -f /tmp/stapobl4bG/tracequery_kmod_3/.tmp_versions/* make -f scripts/Makefile.build obj=/tmp/stapobl4bG/tracequery_kmod_3 gcc -Wp,-MD,/tmp/stapobl4bG/tracequery_kmod_3/.tracequery_kmod_3.o.d -nostdinc -isystem /usr/lib/gcc/x86_64-redhat-linux/4.4.6/include -Iinclude -I/usr/src/kernels/2.6.32-195.el6.x86_64/arch/x86/include -include include/linux/autoconf.h -D__KERNEL__ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Wno-format-security -fno-delete-null-pointer-checks -O2 -m64 -mtune=generic -mno-red-zone -mcmodel=kernel -funit-at-a-time -maccumulate-outgoing-args -fstack-protector -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -Wframe-larger-than=2048 -Wno-unused-but-set-variable -fno-omit-frame-pointer -fno-optimize-sibling-calls -g -pg -Wdeclaration-after-statement -Wno-pointer-sign -fno-strict-overflow -fno-dwarf2-cfi-asm -fconserve-stack -g -Wno-implicit-function-declaration -Werror -I/usr/src/debug////////kernel-2.6.32-195.el6/linux-2.6.32-195.el6.x86_64 -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(tracequery_kmod_3)" -D"KBUILD_MODNAME=KBUILD_STR(tracequery_kmod_3)" -D"DEBUG_HASH=23" -D"DEBUG_HASH2=46" -c -o /tmp/stapobl4bG/tracequery_kmod_3/.tmp_tracequery_kmod_3.o /tmp/stapobl4bG/tracequery_kmod_3/tracequery_kmod_3.c cc1: warnings being treated as errors In file included from /tmp/stapobl4bG/tracequery_kmod_3/tracequery_kmod_3.c:53: include/trace/events/ext3.h:450: error: ‘struct ext3_reserve_window_node’ declared inside parameter list include/trace/events/ext3.h:450: error: its scope is only this definition or declaration, which is probably not what you want include/trace/events/ext3.h:473: error: ‘struct ext3_reserve_window_node’ declared inside parameter list include/trace/events/ext3.h:519: error: ‘struct ext3_reserve_window_node’ declared inside parameter list make[1]: *** [/tmp/stapobl4bG/tracequery_kmod_3/tracequery_kmod_3.o] Error 1 make: *** [_module_/tmp/stapobl4bG/tracequery_kmod_3] Error 2 make: Leaving directory `/usr/src/kernels/2.6.32-195.el6.x86_64' Spawn waitpid result (0x200): 2 Warning: make exited with status: 2
I am seeing the following error: In file included from /tmp/stapPG5S1M/tracequery_kmod_1/tracequery_kmod_1.c:146: 0: include/trace/events/napi.h:12:1: error: redefinition of ‘stapprobe_napi_poll’ include/trace/events/napi.h:12:1: note: previous definition of ‘stapprobe_napi_poll’ was here make[1]: *** [/tmp/stapPG5S1M/tracequery_kmod_1/tracequery_kmod_1.o] Error 1 But no others with systemtap git against 2.6.35.14-96.fc14.x86_64 Although I am only seeing jdb2 and ext4 probes, I assume ext3 and jdb probes are different. Will try against another kernel. Could you show the stap -l output expected so I can compare more easily?
Against 2.6.32-131.12.1.el6.x86_64 I get the same results as mentioned in comment #4.
Hi Mark, Please see https://bugzilla.redhat.com/show_bug.cgi?id=696332 https://bugzilla.redhat.com/show_bug.cgi?id=696332#c8 lists all ext3 tracepoints and https://bugzilla.redhat.com/show_bug.cgi?id=696332#c9 lists all jbd tracepoints. Those new tracepoints were introduced in kernel-2.6.32-182.el6. Sorry for the obscurness.
Unfortunately attachment #1 [details] is enough only to make the stap -l work. Several tracepoint headers still conflict, which makes scripts like probe kernel.trace("*") {} break during pass 4 due to inter-header conflicts.
If we accept that the kernel will continue creating issues like this, then we'll need to start segregating the tracepoint registration and callbacks into separate compile units. Those that come from the same tracepoint header could still share, as an optimization, but they don't necessarily need to. Our current generated code looks like this for trace("sys_enter"): > #include <trace/events/syscalls.h> > #ifdef DECLARE_TRACE_NOARGS > #define STAP_TP_DATA , NULL > #define STAP_TP_PROTO void *cb_data __attribute__ ((unused)), > #else > #define STAP_TP_DATA > #define STAP_TP_PROTO > #endif > static void enter_tracepoint_probe_1( > STAP_TP_PROTO > struct pt_regs* __tracepoint_arg_regs, > long int __tracepoint_arg_id > ) > { [... common entry boilerplate] > c->marker_name = "sys_enter"; > c->probe_locals.probe_2904.__tracepoint_arg_regs = (int64_t)(intptr_t)__tracepoint_arg_regs; > c->probe_locals.probe_2904.__tracepoint_arg_id = (int64_t)__tracepoint_arg_id; > (*probe->ph) (c); [... common exit boilerplate] > } > static int register_tracepoint_probe_1(void) { > return register_trace_sys_enter(enter_tracepoint_probe_1 STAP_TP_DATA); > } > static void unregister_tracepoint_probe_1(void) { > (void) unregister_trace_sys_enter(enter_tracepoint_probe_1 STAP_TP_DATA); > } The registration can easily be moved to a separate file, but enter_tracepoint is a bit more complicated for mixing the tracepoint's typed arguments and systemtap's context setup. We can probably just split it so the arguments are cast to int64_t ahead of time, something like: In stap_HASH.c: > static void stp_enter_tracepoint_probe_1( > int64_t __tracepoint_arg_regs, > int64_t __tracepoint_arg_id > ) > { [... common entry boilerplate] > c->marker_name = "sys_enter"; > c->probe_locals.probe_2904.__tracepoint_arg_regs = __tracepoint_arg_regs; > c->probe_locals.probe_2904.__tracepoint_arg_id = __tracepoint_arg_id; > (*probe->ph) (c); [... common exit boilerplate] > } In stap_HASH_tracepoint_probe_1.c: > static void enter_tracepoint_probe_1( > STAP_TP_PROTO > struct pt_regs* __tracepoint_arg_regs, > long int __tracepoint_arg_id > ) > { > stp_enter_tracepoint_probe_1((int64_t)(intptr_t)__tracepoint_arg_regs, > (int64_t)__tracepoint_arg_id); > }
Created attachment 5975 [details] tracepoint.exp systemtap.log Finally got a ppc box to test against. But recent commits seem to have made things broken in different ways. It does now show up as a FAIL in the testsuite for tracepoints.exp. See attached log. It seems to run fine on x86_64 for me, but with an older kernel.
commit a4b9c3b3