Line data Source code
1 : #include <linux/kdebug.h>
2 : #include <linux/kprobes.h>
3 : #include <linux/export.h>
4 : #include <linux/notifier.h>
5 : #include <linux/rcupdate.h>
6 : #include <linux/vmalloc.h>
7 : #include <linux/reboot.h>
8 :
9 : /*
10 : * Notifier list for kernel code which wants to be called
11 : * at shutdown. This is used to stop any idling DMA operations
12 : * and the like.
13 : */
14 : BLOCKING_NOTIFIER_HEAD(reboot_notifier_list);
15 :
16 : /*
17 : * Notifier chain core routines. The exported routines below
18 : * are layered on top of these, with appropriate locking added.
19 : */
20 :
21 : static int notifier_chain_register(struct notifier_block **nl,
22 : struct notifier_block *n)
23 : {
24 65 : while ((*nl) != NULL) {
25 50 : if (n->priority > (*nl)->priority)
26 : break;
27 41 : nl = &((*nl)->next);
28 : }
29 24 : n->next = *nl;
30 24 : rcu_assign_pointer(*nl, n);
31 : return 0;
32 : }
33 :
34 : static int notifier_chain_cond_register(struct notifier_block **nl,
35 : struct notifier_block *n)
36 : {
37 1 : while ((*nl) != NULL) {
38 0 : if ((*nl) == n)
39 : return 0;
40 0 : if (n->priority > (*nl)->priority)
41 : break;
42 0 : nl = &((*nl)->next);
43 : }
44 1 : n->next = *nl;
45 1 : rcu_assign_pointer(*nl, n);
46 : return 0;
47 : }
48 :
49 : static int notifier_chain_unregister(struct notifier_block **nl,
50 : struct notifier_block *n)
51 : {
52 0 : while ((*nl) != NULL) {
53 0 : if ((*nl) == n) {
54 0 : rcu_assign_pointer(*nl, n->next);
55 : return 0;
56 : }
57 0 : nl = &((*nl)->next);
58 : }
59 : return -ENOENT;
60 : }
61 :
62 : /**
63 : * notifier_call_chain - Informs the registered notifiers about an event.
64 : * @nl: Pointer to head of the blocking notifier chain
65 : * @val: Value passed unmodified to notifier function
66 : * @v: Pointer passed unmodified to notifier function
67 : * @nr_to_call: Number of notifier functions to be called. Don't care
68 : * value of this parameter is -1.
69 : * @nr_calls: Records the number of notifications sent. Don't care
70 : * value of this field is NULL.
71 : * @returns: notifier_call_chain returns the value returned by the
72 : * last notifier function called.
73 : */
74 1523700 : static int notifier_call_chain(struct notifier_block **nl,
75 : unsigned long val, void *v,
76 : int nr_to_call, int *nr_calls)
77 : {
78 : int ret = NOTIFY_DONE;
79 : struct notifier_block *nb, *next_nb;
80 :
81 1523700 : nb = rcu_dereference_raw(*nl);
82 :
83 4311777 : while (nb && nr_to_call) {
84 1264382 : next_nb = rcu_dereference_raw(nb->next);
85 :
86 : #ifdef CONFIG_DEBUG_NOTIFIERS
87 : if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
88 : WARN(1, "Invalid notifier called!");
89 : nb = next_nb;
90 : continue;
91 : }
92 : #endif
93 1264382 : ret = nb->notifier_call(nb, val, v);
94 :
95 1264382 : if (nr_calls)
96 0 : (*nr_calls)++;
97 :
98 1264382 : if ((ret & NOTIFY_STOP_MASK) == NOTIFY_STOP_MASK)
99 : break;
100 : nb = next_nb;
101 1264377 : nr_to_call--;
102 : }
103 1523700 : return ret;
104 : }
105 : NOKPROBE_SYMBOL(notifier_call_chain);
106 :
107 : /*
108 : * Atomic notifier chain routines. Registration and unregistration
109 : * use a spinlock, and call_chain is synchronized by RCU (no locks).
110 : */
111 :
112 : /**
113 : * atomic_notifier_chain_register - Add notifier to an atomic notifier chain
114 : * @nh: Pointer to head of the atomic notifier chain
115 : * @n: New entry in notifier chain
116 : *
117 : * Adds a notifier to an atomic notifier chain.
118 : *
119 : * Currently always returns zero.
120 : */
121 5 : int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
122 : struct notifier_block *n)
123 : {
124 : unsigned long flags;
125 : int ret;
126 :
127 5 : spin_lock_irqsave(&nh->lock, flags);
128 5 : ret = notifier_chain_register(&nh->head, n);
129 : spin_unlock_irqrestore(&nh->lock, flags);
130 5 : return ret;
131 : }
132 : EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
133 :
134 : /**
135 : * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
136 : * @nh: Pointer to head of the atomic notifier chain
137 : * @n: Entry to remove from notifier chain
138 : *
139 : * Removes a notifier from an atomic notifier chain.
140 : *
141 : * Returns zero on success or %-ENOENT on failure.
142 : */
143 0 : int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
144 : struct notifier_block *n)
145 : {
146 : unsigned long flags;
147 : int ret;
148 :
149 0 : spin_lock_irqsave(&nh->lock, flags);
150 0 : ret = notifier_chain_unregister(&nh->head, n);
151 : spin_unlock_irqrestore(&nh->lock, flags);
152 0 : synchronize_rcu();
153 0 : return ret;
154 : }
155 : EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
156 :
157 : /**
158 : * __atomic_notifier_call_chain - Call functions in an atomic notifier chain
159 : * @nh: Pointer to head of the atomic notifier chain
160 : * @val: Value passed unmodified to notifier function
161 : * @v: Pointer passed unmodified to notifier function
162 : * @nr_to_call: See the comment for notifier_call_chain.
163 : * @nr_calls: See the comment for notifier_call_chain.
164 : *
165 : * Calls each function in a notifier chain in turn. The functions
166 : * run in an atomic context, so they must not block.
167 : * This routine uses RCU to synchronize with changes to the chain.
168 : *
169 : * If the return value of the notifier can be and'ed
170 : * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
171 : * will return immediately, with the return value of
172 : * the notifier function which halted execution.
173 : * Otherwise the return value is the return value
174 : * of the last notifier function called.
175 : */
176 0 : int __atomic_notifier_call_chain(struct atomic_notifier_head *nh,
177 : unsigned long val, void *v,
178 : int nr_to_call, int *nr_calls)
179 : {
180 : int ret;
181 :
182 : rcu_read_lock();
183 1271592 : ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
184 : rcu_read_unlock();
185 0 : return ret;
186 : }
187 : EXPORT_SYMBOL_GPL(__atomic_notifier_call_chain);
188 : NOKPROBE_SYMBOL(__atomic_notifier_call_chain);
189 :
190 1271592 : int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
191 : unsigned long val, void *v)
192 : {
193 1271592 : return __atomic_notifier_call_chain(nh, val, v, -1, NULL);
194 : }
195 : EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
196 : NOKPROBE_SYMBOL(atomic_notifier_call_chain);
197 :
198 : /*
199 : * Blocking notifier chain routines. All access to the chain is
200 : * synchronized by an rwsem.
201 : */
202 :
203 : /**
204 : * blocking_notifier_chain_register - Add notifier to a blocking notifier chain
205 : * @nh: Pointer to head of the blocking notifier chain
206 : * @n: New entry in notifier chain
207 : *
208 : * Adds a notifier to a blocking notifier chain.
209 : * Must be called in process context.
210 : *
211 : * Currently always returns zero.
212 : */
213 9 : int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
214 : struct notifier_block *n)
215 : {
216 : int ret;
217 :
218 : /*
219 : * This code gets used during boot-up, when task switching is
220 : * not yet working and interrupts must remain disabled. At
221 : * such times we must not call down_write().
222 : */
223 9 : if (unlikely(system_state == SYSTEM_BOOTING))
224 18 : return notifier_chain_register(&nh->head, n);
225 :
226 0 : down_write(&nh->rwsem);
227 0 : ret = notifier_chain_register(&nh->head, n);
228 0 : up_write(&nh->rwsem);
229 0 : return ret;
230 : }
231 : EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
232 :
233 : /**
234 : * blocking_notifier_chain_cond_register - Cond add notifier to a blocking notifier chain
235 : * @nh: Pointer to head of the blocking notifier chain
236 : * @n: New entry in notifier chain
237 : *
238 : * Adds a notifier to a blocking notifier chain, only if not already
239 : * present in the chain.
240 : * Must be called in process context.
241 : *
242 : * Currently always returns zero.
243 : */
244 1 : int blocking_notifier_chain_cond_register(struct blocking_notifier_head *nh,
245 : struct notifier_block *n)
246 : {
247 : int ret;
248 :
249 1 : down_write(&nh->rwsem);
250 1 : ret = notifier_chain_cond_register(&nh->head, n);
251 1 : up_write(&nh->rwsem);
252 1 : return ret;
253 : }
254 : EXPORT_SYMBOL_GPL(blocking_notifier_chain_cond_register);
255 :
256 : /**
257 : * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
258 : * @nh: Pointer to head of the blocking notifier chain
259 : * @n: Entry to remove from notifier chain
260 : *
261 : * Removes a notifier from a blocking notifier chain.
262 : * Must be called from process context.
263 : *
264 : * Returns zero on success or %-ENOENT on failure.
265 : */
266 0 : int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
267 : struct notifier_block *n)
268 : {
269 : int ret;
270 :
271 : /*
272 : * This code gets used during boot-up, when task switching is
273 : * not yet working and interrupts must remain disabled. At
274 : * such times we must not call down_write().
275 : */
276 0 : if (unlikely(system_state == SYSTEM_BOOTING))
277 0 : return notifier_chain_unregister(&nh->head, n);
278 :
279 0 : down_write(&nh->rwsem);
280 0 : ret = notifier_chain_unregister(&nh->head, n);
281 0 : up_write(&nh->rwsem);
282 0 : return ret;
283 : }
284 : EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
285 :
286 : /**
287 : * __blocking_notifier_call_chain - Call functions in a blocking notifier chain
288 : * @nh: Pointer to head of the blocking notifier chain
289 : * @val: Value passed unmodified to notifier function
290 : * @v: Pointer passed unmodified to notifier function
291 : * @nr_to_call: See comment for notifier_call_chain.
292 : * @nr_calls: See comment for notifier_call_chain.
293 : *
294 : * Calls each function in a notifier chain in turn. The functions
295 : * run in a process context, so they are allowed to block.
296 : *
297 : * If the return value of the notifier can be and'ed
298 : * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
299 : * will return immediately, with the return value of
300 : * the notifier function which halted execution.
301 : * Otherwise the return value is the return value
302 : * of the last notifier function called.
303 : */
304 207 : int __blocking_notifier_call_chain(struct blocking_notifier_head *nh,
305 : unsigned long val, void *v,
306 : int nr_to_call, int *nr_calls)
307 : {
308 : int ret = NOTIFY_DONE;
309 :
310 : /*
311 : * We check the head outside the lock, but if this access is
312 : * racy then it does not matter what the result of the test
313 : * is, we re-check the list after having taken the lock anyway:
314 : */
315 207 : if (rcu_access_pointer(nh->head)) {
316 100 : down_read(&nh->rwsem);
317 100 : ret = notifier_call_chain(&nh->head, val, v, nr_to_call,
318 : nr_calls);
319 100 : up_read(&nh->rwsem);
320 : }
321 207 : return ret;
322 : }
323 : EXPORT_SYMBOL_GPL(__blocking_notifier_call_chain);
324 :
325 207 : int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
326 : unsigned long val, void *v)
327 : {
328 207 : return __blocking_notifier_call_chain(nh, val, v, -1, NULL);
329 : }
330 : EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
331 :
332 : /*
333 : * Raw notifier chain routines. There is no protection;
334 : * the caller must provide it. Use at your own risk!
335 : */
336 :
337 : /**
338 : * raw_notifier_chain_register - Add notifier to a raw notifier chain
339 : * @nh: Pointer to head of the raw notifier chain
340 : * @n: New entry in notifier chain
341 : *
342 : * Adds a notifier to a raw notifier chain.
343 : * All locking must be provided by the caller.
344 : *
345 : * Currently always returns zero.
346 : */
347 10 : int raw_notifier_chain_register(struct raw_notifier_head *nh,
348 : struct notifier_block *n)
349 : {
350 20 : return notifier_chain_register(&nh->head, n);
351 : }
352 : EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
353 :
354 : /**
355 : * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
356 : * @nh: Pointer to head of the raw notifier chain
357 : * @n: Entry to remove from notifier chain
358 : *
359 : * Removes a notifier from a raw notifier chain.
360 : * All locking must be provided by the caller.
361 : *
362 : * Returns zero on success or %-ENOENT on failure.
363 : */
364 0 : int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
365 : struct notifier_block *n)
366 : {
367 0 : return notifier_chain_unregister(&nh->head, n);
368 : }
369 : EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
370 :
371 : /**
372 : * __raw_notifier_call_chain - Call functions in a raw notifier chain
373 : * @nh: Pointer to head of the raw notifier chain
374 : * @val: Value passed unmodified to notifier function
375 : * @v: Pointer passed unmodified to notifier function
376 : * @nr_to_call: See comment for notifier_call_chain.
377 : * @nr_calls: See comment for notifier_call_chain
378 : *
379 : * Calls each function in a notifier chain in turn. The functions
380 : * run in an undefined context.
381 : * All locking must be provided by the caller.
382 : *
383 : * If the return value of the notifier can be and'ed
384 : * with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
385 : * will return immediately, with the return value of
386 : * the notifier function which halted execution.
387 : * Otherwise the return value is the return value
388 : * of the last notifier function called.
389 : */
390 0 : int __raw_notifier_call_chain(struct raw_notifier_head *nh,
391 : unsigned long val, void *v,
392 : int nr_to_call, int *nr_calls)
393 : {
394 251985 : return notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
395 : }
396 : EXPORT_SYMBOL_GPL(__raw_notifier_call_chain);
397 :
398 251985 : int raw_notifier_call_chain(struct raw_notifier_head *nh,
399 : unsigned long val, void *v)
400 : {
401 251985 : return __raw_notifier_call_chain(nh, val, v, -1, NULL);
402 : }
403 : EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
404 :
405 : /*
406 : * SRCU notifier chain routines. Registration and unregistration
407 : * use a mutex, and call_chain is synchronized by SRCU (no locks).
408 : */
409 :
410 : /**
411 : * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
412 : * @nh: Pointer to head of the SRCU notifier chain
413 : * @n: New entry in notifier chain
414 : *
415 : * Adds a notifier to an SRCU notifier chain.
416 : * Must be called in process context.
417 : *
418 : * Currently always returns zero.
419 : */
420 0 : int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
421 : struct notifier_block *n)
422 : {
423 : int ret;
424 :
425 : /*
426 : * This code gets used during boot-up, when task switching is
427 : * not yet working and interrupts must remain disabled. At
428 : * such times we must not call mutex_lock().
429 : */
430 0 : if (unlikely(system_state == SYSTEM_BOOTING))
431 0 : return notifier_chain_register(&nh->head, n);
432 :
433 0 : mutex_lock(&nh->mutex);
434 0 : ret = notifier_chain_register(&nh->head, n);
435 0 : mutex_unlock(&nh->mutex);
436 0 : return ret;
437 : }
438 : EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
439 :
440 : /**
441 : * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
442 : * @nh: Pointer to head of the SRCU notifier chain
443 : * @n: Entry to remove from notifier chain
444 : *
445 : * Removes a notifier from an SRCU notifier chain.
446 : * Must be called from process context.
447 : *
448 : * Returns zero on success or %-ENOENT on failure.
449 : */
450 0 : int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
451 : struct notifier_block *n)
452 : {
453 : int ret;
454 :
455 : /*
456 : * This code gets used during boot-up, when task switching is
457 : * not yet working and interrupts must remain disabled. At
458 : * such times we must not call mutex_lock().
459 : */
460 0 : if (unlikely(system_state == SYSTEM_BOOTING))
461 0 : return notifier_chain_unregister(&nh->head, n);
462 :
463 0 : mutex_lock(&nh->mutex);
464 0 : ret = notifier_chain_unregister(&nh->head, n);
465 0 : mutex_unlock(&nh->mutex);
466 0 : synchronize_srcu(&nh->srcu);
467 0 : return ret;
468 : }
469 : EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
470 :
471 : /**
472 : * __srcu_notifier_call_chain - Call functions in an SRCU notifier chain
473 : * @nh: Pointer to head of the SRCU notifier chain
474 : * @val: Value passed unmodified to notifier function
475 : * @v: Pointer passed unmodified to notifier function
476 : * @nr_to_call: See comment for notifier_call_chain.
477 : * @nr_calls: See comment for notifier_call_chain
478 : *
479 : * Calls each function in a notifier chain in turn. The functions
480 : * run in a process context, so they are allowed to block.
481 : *
482 : * If the return value of the notifier can be and'ed
483 : * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
484 : * will return immediately, with the return value of
485 : * the notifier function which halted execution.
486 : * Otherwise the return value is the return value
487 : * of the last notifier function called.
488 : */
489 22 : int __srcu_notifier_call_chain(struct srcu_notifier_head *nh,
490 : unsigned long val, void *v,
491 : int nr_to_call, int *nr_calls)
492 : {
493 : int ret;
494 : int idx;
495 :
496 22 : idx = srcu_read_lock(&nh->srcu);
497 22 : ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
498 : srcu_read_unlock(&nh->srcu, idx);
499 22 : return ret;
500 : }
501 : EXPORT_SYMBOL_GPL(__srcu_notifier_call_chain);
502 :
503 22 : int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
504 : unsigned long val, void *v)
505 : {
506 22 : return __srcu_notifier_call_chain(nh, val, v, -1, NULL);
507 : }
508 : EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
509 :
510 : /**
511 : * srcu_init_notifier_head - Initialize an SRCU notifier head
512 : * @nh: Pointer to head of the srcu notifier chain
513 : *
514 : * Unlike other sorts of notifier heads, SRCU notifier heads require
515 : * dynamic initialization. Be sure to call this routine before
516 : * calling any of the other SRCU notifier routines for this head.
517 : *
518 : * If an SRCU notifier head is deallocated, it must first be cleaned
519 : * up by calling srcu_cleanup_notifier_head(). Otherwise the head's
520 : * per-cpu data (used by the SRCU mechanism) will leak.
521 : */
522 1 : void srcu_init_notifier_head(struct srcu_notifier_head *nh)
523 : {
524 1 : mutex_init(&nh->mutex);
525 1 : if (init_srcu_struct(&nh->srcu) < 0)
526 0 : BUG();
527 1 : nh->head = NULL;
528 1 : }
529 : EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
530 :
531 : static ATOMIC_NOTIFIER_HEAD(die_chain);
532 :
533 0 : int notrace notify_die(enum die_val val, const char *str,
534 : struct pt_regs *regs, long err, int trap, int sig)
535 : {
536 0 : struct die_args args = {
537 : .regs = regs,
538 : .str = str,
539 : .err = err,
540 : .trapnr = trap,
541 : .signr = sig,
542 :
543 : };
544 0 : return atomic_notifier_call_chain(&die_chain, val, &args);
545 : }
546 : NOKPROBE_SYMBOL(notify_die);
547 :
548 1 : int register_die_notifier(struct notifier_block *nb)
549 : {
550 1 : vmalloc_sync_all();
551 1 : return atomic_notifier_chain_register(&die_chain, nb);
552 : }
553 : EXPORT_SYMBOL_GPL(register_die_notifier);
554 :
555 0 : int unregister_die_notifier(struct notifier_block *nb)
556 : {
557 0 : return atomic_notifier_chain_unregister(&die_chain, nb);
558 : }
559 : EXPORT_SYMBOL_GPL(unregister_die_notifier);
|
