]>
Commit | Line | Data |
---|---|---|
cc52276b WC |
1 | /* -*- linux-c -*- |
2 | * uprobe Functions | |
ef36f781 | 3 | * Copyright (C) 2014 Red Hat Inc. |
cc52276b WC |
4 | * |
5 | * This file is part of systemtap, and is free software. You can | |
6 | * redistribute it and/or modify it under the terms of the GNU General | |
7 | * Public License (GPL); either version 2, or (at your option) any | |
8 | * later version. | |
9 | */ | |
10 | ||
11 | #ifndef _UPROBE_COMMON_C_ | |
12 | #define _UPROBE_COMMON_C_ | |
13 | ||
14 | /* NB: Because these utrace callbacks only occur before / after | |
15 | userspace instructions run, there is no concurrency control issue | |
16 | between active uprobe callbacks and these registration / | |
17 | unregistration pieces. | |
18 | ||
19 | We protect the stap_uprobe->spec_index (which also serves as a | |
20 | free/busy flag) value with the outer protective stap_probes_lock | |
21 | spinlock, to protect it against concurrent registration / | |
22 | unregistration. | |
23 | */ | |
24 | ||
25 | static int stap_uprobe_change_plus (struct task_struct *tsk, unsigned long relocation, unsigned long length, const struct stap_uprobe_tf *stf, unsigned long offset, unsigned long vm_flags) { | |
26 | int tfi = (stf - stap_uprobe_finders); | |
27 | int spec_index; | |
28 | /* iterate over stap_uprobe_spec[] that use this same stap_uprobe_tf */ | |
29 | for (spec_index=0; spec_index<sizeof(stap_uprobe_specs)/sizeof(stap_uprobe_specs[0]); spec_index++) { | |
30 | int handled_p = 0; | |
31 | int slotted_p = 0; | |
32 | const struct stap_uprobe_spec *sups = &stap_uprobe_specs [spec_index]; | |
33 | struct stap_uprobe *sup; | |
34 | pid_t sdt_sem_pid; | |
35 | int rc = 0; | |
36 | int i; | |
4fa83377 SC |
37 | int pci; |
38 | ||
cc52276b WC |
39 | if (likely(sups->tfi != tfi)) continue; |
40 | /* skip probes with an address beyond this map event; should not | |
41 | happen unless a shlib/exec got mmapped in weirdly piecemeal */ | |
77694f53 | 42 | if (likely((vm_flags & VM_EXEC) && sups->address >= length)) continue; |
cc52276b WC |
43 | |
44 | /* Found a uprobe_spec for this stap_uprobe_tf. Need to lock the | |
45 | stap_uprobes[] array to allocate a free spot, but then we can | |
46 | unlock and do the register_*probe subsequently. */ | |
47 | ||
48 | mutex_lock (& stap_uprobes_lock); | |
49 | for (i=0; i<MAXUPROBES; i++) { /* XXX: slow linear search */ | |
50 | sup = & stap_uprobes[i]; | |
51 | ||
52 | /* register new uprobe | |
53 | We make two passes for semaphores; | |
86229a55 | 54 | see stap_uprobe_change_semaphore_plus */ |
cc52276b WC |
55 | |
56 | if (sup->spec_index < 0 || (sups->sdt_sem_offset && vm_flags & VM_WRITE && sup->spec_index == spec_index)) { | |
57 | #if (UPROBES_API_VERSION < 2) | |
58 | /* See PR6829 comment. */ | |
59 | if (sup->spec_index == -1 && sup->up.kdata != NULL) continue; | |
60 | else if (sup->spec_index == -2 && sup->urp.u.kdata != NULL) continue; | |
61 | #endif | |
62 | sup->spec_index = spec_index; | |
63 | slotted_p = 1; | |
64 | break; | |
65 | } | |
66 | } | |
67 | mutex_unlock (& stap_uprobes_lock); | |
68 | #ifdef DEBUG_UPROBES | |
26e63673 | 69 | _stp_dbug(__FUNCTION__,__LINE__, "+uprobe spec %d idx %d process %s[%d] addr %p pp %s\n", spec_index, (slotted_p ? i : -1), tsk->comm, tsk->tgid, (void*)(relocation+sups->address), sups->probe->pp); |
cc52276b | 70 | #endif |
77c59419 FCE |
71 | |
72 | /* NB: check for user-module build-id only if we have a pathname | |
73 | at all; for a process(PID#).* probe, we may not. If at some | |
74 | point we map process(PID#) to process("/proc/PID#/exe"), we'll | |
75 | get a pathname. */ | |
76 | if (stf->pathname) | |
77 | if ((rc = _stp_usermodule_check(tsk, stf->pathname, relocation))) | |
78 | return rc; | |
cc52276b WC |
79 | |
80 | /* Here, slotted_p implies that `i' points to the single | |
81 | stap_uprobes[] element that has been slotted in for registration | |
82 | or unregistration processing. !slotted_p implies that the table | |
83 | was full (registration; MAXUPROBES) or that no matching entry was | |
84 | found (unregistration; should not happen). */ | |
85 | ||
86 | sdt_sem_pid = (sups->return_p ? sup->urp.u.pid : sup->up.pid); | |
87 | if (sups->sdt_sem_offset && (sdt_sem_pid != tsk->tgid || sup->sdt_sem_address == 0)) { | |
529c7eae JS |
88 | /* If the probe is in an ET_EXEC binary, then the sdt_sem_offset already |
89 | * is a real address. But stap_uprobe_process_found calls us in this | |
90 | * case with relocation=offset=0, so we don't have to worry about it. */ | |
91 | sup->sdt_sem_address = (relocation - offset) + sups->sdt_sem_offset; | |
cc52276b | 92 | } /* sdt_sem_offset */ |
4fa83377 SC |
93 | |
94 | for (pci=0; pci < sups->perf_counters_dim; pci++) { | |
0d049a1d FCE |
95 | if ((sups->perf_counters)[pci] > -1) |
96 | _stp_perf_read_init ((sups->perf_counters)[pci], tsk); | |
4fa83377 SC |
97 | } |
98 | ||
cc52276b WC |
99 | if (slotted_p) { |
100 | struct stap_uprobe *sup = & stap_uprobes[i]; | |
101 | if (sups->return_p) { | |
102 | sup->urp.u.pid = tsk->tgid; | |
103 | sup->urp.u.vaddr = relocation + sups->address; | |
104 | sup->urp.handler = &enter_uretprobe_probe; | |
105 | rc = register_uretprobe (& sup->urp); | |
106 | } else { | |
107 | sup->up.pid = tsk->tgid; | |
108 | sup->up.vaddr = relocation + sups->address; | |
109 | sup->up.handler = &enter_uprobe_probe; | |
110 | rc = register_uprobe (& sup->up); | |
111 | } | |
86229a55 DS |
112 | |
113 | /* The u*probe failed to register. However, if we got EEXIST, | |
114 | * that means that the u*probe is already there, so just ignore | |
115 | * the error. This could happen if CLONE_THREAD or CLONE_VM was | |
116 | * used. */ | |
117 | if (rc != 0 && rc != -EEXIST) { | |
26e63673 | 118 | _stp_warn ("u*probe failed %s[%d] '%s' addr %p rc %d\n", tsk->comm, tsk->tgid, sups->probe->pp, (void*)(relocation + sups->address), rc); |
cc52276b WC |
119 | /* NB: we need to release this slot, |
120 | so we need to borrow the mutex temporarily. */ | |
121 | mutex_lock (& stap_uprobes_lock); | |
122 | sup->spec_index = -1; | |
7554a13f | 123 | sup->sdt_sem_address = 0; |
cc52276b WC |
124 | mutex_unlock (& stap_uprobes_lock); |
125 | } else { | |
126 | handled_p = 1; | |
127 | } | |
128 | } | |
129 | /* NB: handled_p implies slotted_p */ | |
130 | if (unlikely (! handled_p)) { | |
131 | #ifdef STP_TIMING | |
4e1434f6 | 132 | atomic_inc (skipped_count_uprobe_reg()); |
cc52276b WC |
133 | #endif |
134 | /* NB: duplicates common_entryfn_epilogue, | |
135 | but then this is not a probe entry fn epilogue. */ | |
7acb3e34 | 136 | #ifndef STAP_SUPPRESS_HANDLER_ERRORS |
228f84ae MF |
137 | if (unlikely (atomic_inc_return (skipped_count()) > MAXSKIPPED)) { |
138 | if (unlikely (pseudo_atomic_cmpxchg(session_state(), STAP_SESSION_RUNNING, STAP_SESSION_ERROR) == STAP_SESSION_RUNNING)) | |
7acb3e34 | 139 | _stp_error ("Skipped too many probes, check MAXSKIPPED or try again with stap -t for more details."); |
cc52276b | 140 | } |
228f84ae | 141 | #endif |
cc52276b WC |
142 | } |
143 | } /* close iteration over stap_uprobe_spec[] */ | |
144 | return 0; /* XXX: or rc? */ | |
145 | } | |
146 | ||
147 | static int stap_uprobe_change_semaphore_plus (struct task_struct *tsk, unsigned long relocation, unsigned long length, const struct stap_uprobe_tf *stf) { | |
148 | int tfi = (stf - stap_uprobe_finders); | |
149 | int spec_index; | |
150 | int rc = 0; | |
151 | struct stap_uprobe *sup; | |
152 | int i; | |
153 | ||
154 | /* We make two passes for semaphores. | |
155 | The first pass, stap_uprobe_change_plus, calculates the address of the | |
156 | semaphore. If the probe is in a .so, we calculate the | |
157 | address when the initial mmap maps the entire solib, e.g. | |
158 | 7f089885a000-7f089885b000 rw-p- libtcl.so | |
100a540e | 159 | A subsequent mmap maps in the writable segment where the |
cc52276b WC |
160 | semaphore control variable lives, e.g. |
161 | 7f089850d000-7f0898647000 r-xp- libtcl.so | |
162 | 7f0898647000-7f0898846000 ---p libtcl.so | |
163 | 7f0898846000-7f089885b000 rw-p- libtcl.so | |
164 | The second pass, stap_uprobe_change_semaphore_plus, sets the semaphore. | |
100a540e | 165 | If the probe is in a .so this will be when the writable segment of the .so |
cc52276b WC |
166 | is mapped in. If the task changes, then recalculate the address. |
167 | */ | |
168 | ||
169 | for (i=0; i<MAXUPROBES; i++) { /* XXX: slow linear search */ | |
170 | sup = & stap_uprobes[i]; | |
171 | if (sup->spec_index == -1) continue; | |
172 | if (sup->sdt_sem_address != 0 && !(sup->up.pid == tsk->tgid && sup->sdt_sem_address >= relocation && sup->sdt_sem_address < relocation+length)) continue; | |
173 | if (sup->sdt_sem_address) { | |
174 | unsigned short sdt_semaphore = 0; /* NB: fixed size */ | |
175 | if ((rc = get_user (sdt_semaphore, (unsigned short __user*) sup->sdt_sem_address)) == 0) { | |
176 | sdt_semaphore ++; | |
177 | #ifdef DEBUG_UPROBES | |
178 | { | |
179 | const struct stap_uprobe_spec *sups = &stap_uprobe_specs [sup->spec_index]; | |
180 | _stp_dbug(__FUNCTION__,__LINE__, "+semaphore %#x @ %#lx spec %d idx %d task %d\n", sdt_semaphore, sup->sdt_sem_address, sup->spec_index, i, tsk->tgid); | |
181 | } | |
182 | #endif | |
77694f53 | 183 | rc = put_user (sdt_semaphore, (unsigned short __user*) sup->sdt_sem_address); |
cc52276b WC |
184 | /* XXX: need to analyze possibility of race condition */ |
185 | } | |
186 | } | |
187 | } | |
188 | return rc; | |
189 | } | |
190 | ||
191 | /* Removing/unmapping a uprobe is simpler than adding one (in the | |
192 | _plus function above). We need not care about stap_uprobe_finders | |
193 | or anything, we just scan through stap_uprobes[] for a live probe | |
194 | within the given address range, and kill it. */ | |
195 | static int stap_uprobe_change_minus (struct task_struct *tsk, unsigned long relocation, unsigned long length, const struct stap_uprobe_tf *stf) { | |
196 | int i; | |
197 | ||
198 | /* NB: it's not an error for us not to find a live uprobe within the | |
199 | given range. We might have received a callback for a part of a | |
200 | shlib that was unmapped and unprobed. */ | |
201 | ||
202 | for (i=0; i<MAXUPROBES; i++) { /* XXX: slow linear search */ | |
203 | struct stap_uprobe *sup = & stap_uprobes[i]; | |
204 | struct stap_uprobe_spec *sups; | |
205 | if (sup->spec_index < 0) continue; /* skip free uprobes slot */ | |
206 | sups = (struct stap_uprobe_spec*) & stap_uprobe_specs[sup->spec_index]; | |
207 | mutex_lock (& stap_uprobes_lock); | |
208 | ||
209 | /* PR6829, PR9940: | |
210 | Here we're unregistering for one of two reasons: | |
211 | 1. the process image is going away (or gone) due to exit or exec; or | |
212 | 2. the vma containing the probepoint has been unmapped. | |
213 | In case 1, it's sort of a nop, because uprobes will notice the event | |
214 | and dispose of the probes eventually, if it hasn't already. But by | |
215 | calling unmap_u[ret]probe() ourselves, we free up sup right away. | |
216 | ||
217 | In both cases, we must use unmap_u[ret]probe instead of | |
218 | unregister_u[ret]probe, so uprobes knows not to try to restore the | |
219 | original opcode. | |
220 | */ | |
221 | ||
222 | /* URETPROBE */ | |
223 | if (sups->return_p && sup->urp.u.pid == tsk->tgid && sup->urp.u.vaddr >= relocation && sup->urp.u.vaddr < relocation+length) { /* in range */ | |
224 | ||
225 | #ifdef DEBUG_UPROBES | |
26e63673 | 226 | _stp_dbug (__FUNCTION__,__LINE__, "-uretprobe spec %d idx %d process %s[%d] addr %p pp %s\n", sup->spec_index, i, tsk->comm, tsk->tgid, (void*) sup->urp.u.vaddr, sups->probe->pp); |
cc52276b WC |
227 | #endif |
228 | #if (UPROBES_API_VERSION >= 2) | |
229 | unmap_uretprobe (& sup->urp); | |
230 | sup->spec_index = -1; | |
7554a13f | 231 | sup->sdt_sem_address = 0; |
cc52276b WC |
232 | #else |
233 | /* Uprobes lacks unmap_uretprobe. Before reusing sup, we must wait | |
234 | until uprobes turns loose of the uretprobe on its own, as indicated | |
235 | by uretprobe.kdata = NULL. */ | |
236 | sup->spec_index = -2; | |
237 | #endif | |
238 | /* UPROBE */ | |
239 | } else if (!sups->return_p && sup->up.pid == tsk->tgid && sup->up.vaddr >= relocation && sup->up.vaddr < relocation+length) { /* in range */ | |
240 | ||
241 | #ifdef DEBUG_UPROBES | |
26e63673 | 242 | _stp_dbug (__FUNCTION__,__LINE__, "-uprobe spec %d idx %d process %s[%d] reloc %p pp %s\n", sup->spec_index, i, tsk->comm, tsk->tgid, (void*) sup->up.vaddr, sups->probe->pp); |
cc52276b WC |
243 | #endif |
244 | #if (UPROBES_API_VERSION >= 2) | |
245 | unmap_uprobe (& sup->up); | |
246 | sup->spec_index = -1; | |
7554a13f | 247 | sup->sdt_sem_address = 0; |
cc52276b WC |
248 | #else |
249 | /* Uprobes lacks unmap_uprobe. Before reusing sup, we must wait | |
250 | until uprobes turns loose of the uprobe on its own, as indicated | |
251 | by uprobe.kdata = NULL. */ | |
252 | sup->spec_index = -1; | |
7554a13f | 253 | sup->sdt_sem_address = 0; |
cc52276b WC |
254 | #endif |
255 | /* PR10655: we don't need to fidget with the ENABLED semaphore either, | |
256 | as the process is gone, buh-bye, toodaloo, au revoir, see ya later! */ | |
257 | } | |
258 | mutex_unlock (& stap_uprobes_lock); | |
259 | } /* close iteration over stap_uprobes[] */ | |
260 | return 0; /* XXX: or !handled_p */ | |
261 | } | |
262 | ||
263 | /* The task_finder_callback we use for ET_EXEC targets. | |
264 | We used to perform uprobe insertion/removal here, but not any more. | |
265 | (PR10524) */ | |
266 | static int stap_uprobe_process_found (struct stap_task_finder_target *tgt, struct task_struct *tsk, int register_p, int process_p) { | |
267 | const struct stap_uprobe_tf *stf = container_of(tgt, struct stap_uprobe_tf, finder); | |
268 | if (! process_p) return 0; /* ignore threads */ | |
a2b0b5c8 | 269 | dbug_task_vma(1, "%cproc pid %d stf %p %p path %s\n", register_p?'+':'-', tsk->tgid, tgt, stf, stf->pathname); |
cc52276b WC |
270 | /* ET_EXEC events are like shlib events, but with 0 relocation bases */ |
271 | if (register_p) { | |
272 | int rc = stap_uprobe_change_plus (tsk, 0, TASK_SIZE, stf, 0, 0); | |
273 | stap_uprobe_change_semaphore_plus (tsk, 0, TASK_SIZE, stf); | |
274 | return rc; | |
275 | } else | |
276 | return stap_uprobe_change_minus (tsk, 0, TASK_SIZE, stf); | |
277 | } | |
278 | ||
279 | /* The task_finder_mmap_callback */ | |
7b9215b2 JS |
280 | static int |
281 | stap_uprobe_mmap_found (struct stap_task_finder_target *tgt, | |
282 | struct task_struct *tsk, char *path, | |
283 | struct dentry *dentry, unsigned long addr, | |
284 | unsigned long length, unsigned long offset, | |
285 | unsigned long vm_flags) | |
286 | { | |
287 | int rc = 0; | |
cc52276b WC |
288 | const struct stap_uprobe_tf *stf = container_of(tgt, struct stap_uprobe_tf, finder); |
289 | /* 1 - shared libraries' executable segments load from offset 0 | |
7b9215b2 JS |
290 | * - ld.so convention offset != 0 is now allowed |
291 | * so stap_uprobe_change_plus can set a semaphore, | |
292 | * i.e. a static extern, in a shared object | |
293 | * 2 - the shared library we're interested in | |
100a540e | 294 | * 3 - mapping should be executable or writable (for semaphore in .so) |
7b9215b2 JS |
295 | * NB: or both, on kernels that lack noexec mapping |
296 | */ | |
297 | if (path == NULL || strcmp (path, stf->pathname)) | |
298 | return 0; | |
299 | ||
d2696bba DS |
300 | /* Check non-writable, executable sections for probes. */ |
301 | if ((vm_flags & VM_EXEC) && !(vm_flags & VM_WRITE)) { | |
a2b0b5c8 | 302 | dbug_task_vma (1, |
7b9215b2 JS |
303 | "+mmap X pid %d path %s addr %p length %u offset %p stf %p %p path %s\n", |
304 | tsk->tgid, path, (void *) addr, (unsigned)length, (void*) offset, | |
305 | tgt, stf, stf->pathname); | |
7b9215b2 JS |
306 | rc = stap_uprobe_change_plus (tsk, addr, length, stf, offset, vm_flags); |
307 | } | |
308 | ||
100a540e | 309 | /* Check writable sections for semaphores. |
7b9215b2 JS |
310 | * NB: They may have also been executable for the check above, if we're |
311 | * running a kernel that lacks noexec mappings. So long as there's | |
312 | * no error (rc == 0), we need to look for semaphores too. | |
313 | */ | |
314 | if ((rc == 0) && (vm_flags & VM_WRITE)) { | |
a2b0b5c8 | 315 | dbug_task_vma (1, |
7b9215b2 JS |
316 | "+mmap W pid %d path %s addr %p length %u offset %p stf %p %p path %s\n", |
317 | tsk->tgid, path, (void *) addr, (unsigned)length, (void*) offset, | |
318 | tgt, stf, stf->pathname); | |
7b9215b2 JS |
319 | rc = stap_uprobe_change_semaphore_plus (tsk, addr, length, stf); |
320 | } | |
321 | ||
322 | return rc; | |
cc52276b WC |
323 | } |
324 | ||
325 | /* The task_finder_munmap_callback */ | |
326 | static int stap_uprobe_munmap_found (struct stap_task_finder_target *tgt, struct task_struct *tsk, unsigned long addr, unsigned long length) { | |
327 | const struct stap_uprobe_tf *stf = container_of(tgt, struct stap_uprobe_tf, finder); | |
a2b0b5c8 | 328 | dbug_task_vma (1, "-mmap pid %d addr %p length %lu stf %p %p path %s\n", tsk->tgid, (void *) addr, length, tgt, stf, stf->pathname); |
cc52276b WC |
329 | return stap_uprobe_change_minus (tsk, addr, length, stf); |
330 | } | |
331 | ||
19d91f6c JS |
332 | /* The task_finder_callback we use for ET_DYN targets. |
333 | This just forces an unmap of everything as the process exits. | |
334 | (PR11151) */ | |
335 | static int stap_uprobe_process_munmap (struct stap_task_finder_target *tgt, struct task_struct *tsk, int register_p, int process_p) { | |
336 | const struct stap_uprobe_tf *stf = container_of(tgt, struct stap_uprobe_tf, finder); | |
337 | if (! process_p) return 0; /* ignore threads */ | |
a2b0b5c8 | 338 | dbug_task_vma (1, "%cproc pid %d stf %p %p path %s\n", register_p?'+':'-', tsk->tgid, tgt, stf, stf->pathname); |
19d91f6c JS |
339 | /* Covering 0->TASK_SIZE means "unmap everything" */ |
340 | if (!register_p) | |
341 | return stap_uprobe_change_minus (tsk, 0, TASK_SIZE, stf); | |
342 | return 0; | |
343 | } | |
344 | ||
cc52276b | 345 | #endif /* _UPROBE_COMMON_C_ */ |