Line data Source code
1 : /*
2 : * jump label support
3 : *
4 : * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
5 : * Copyright (C) 2011 Peter Zijlstra <pzijlstr@redhat.com>
6 : *
7 : */
8 : #include <linux/memory.h>
9 : #include <linux/uaccess.h>
10 : #include <linux/module.h>
11 : #include <linux/list.h>
12 : #include <linux/slab.h>
13 : #include <linux/sort.h>
14 : #include <linux/err.h>
15 : #include <linux/static_key.h>
16 : #include <linux/jump_label_ratelimit.h>
17 :
18 : #ifdef HAVE_JUMP_LABEL
19 :
20 : /* mutex to protect coming/going of the the jump_label table */
21 : static DEFINE_MUTEX(jump_label_mutex);
22 :
23 0 : void jump_label_lock(void)
24 : {
25 34 : mutex_lock(&jump_label_mutex);
26 0 : }
27 :
28 0 : void jump_label_unlock(void)
29 : {
30 38 : mutex_unlock(&jump_label_mutex);
31 0 : }
32 :
33 521 : static int jump_label_cmp(const void *a, const void *b)
34 : {
35 : const struct jump_entry *jea = a;
36 : const struct jump_entry *jeb = b;
37 :
38 521 : if (jea->key < jeb->key)
39 : return -1;
40 :
41 172 : if (jea->key > jeb->key)
42 : return 1;
43 :
44 59 : return 0;
45 : }
46 :
47 : static void
48 : jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
49 : {
50 : unsigned long size;
51 :
52 1 : size = (((unsigned long)stop - (unsigned long)start)
53 : / sizeof(struct jump_entry));
54 1 : sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
55 : }
56 :
57 : static void jump_label_update(struct static_key *key, int enable);
58 :
59 5 : void static_key_slow_inc(struct static_key *key)
60 : {
61 : STATIC_KEY_CHECK_USE();
62 8 : if (atomic_inc_not_zero(&key->enabled))
63 4 : return;
64 :
65 : jump_label_lock();
66 1 : if (atomic_read(&key->enabled) == 0) {
67 1 : if (!jump_label_get_branch_default(key))
68 1 : jump_label_update(key, JUMP_LABEL_ENABLE);
69 : else
70 0 : jump_label_update(key, JUMP_LABEL_DISABLE);
71 : }
72 : atomic_inc(&key->enabled);
73 : jump_label_unlock();
74 : }
75 : EXPORT_SYMBOL_GPL(static_key_slow_inc);
76 :
77 8 : static void __static_key_slow_dec(struct static_key *key,
78 : unsigned long rate_limit, struct delayed_work *work)
79 : {
80 4 : if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {
81 0 : WARN(atomic_read(&key->enabled) < 0,
82 : "jump label: negative count!\n");
83 4 : return;
84 : }
85 :
86 4 : if (rate_limit) {
87 : atomic_inc(&key->enabled);
88 : schedule_delayed_work(work, rate_limit);
89 : } else {
90 4 : if (!jump_label_get_branch_default(key))
91 0 : jump_label_update(key, JUMP_LABEL_DISABLE);
92 : else
93 4 : jump_label_update(key, JUMP_LABEL_ENABLE);
94 : }
95 : jump_label_unlock();
96 : }
97 :
98 0 : static void jump_label_update_timeout(struct work_struct *work)
99 : {
100 : struct static_key_deferred *key =
101 : container_of(work, struct static_key_deferred, work.work);
102 0 : __static_key_slow_dec(&key->key, 0, NULL);
103 0 : }
104 :
105 4 : void static_key_slow_dec(struct static_key *key)
106 : {
107 : STATIC_KEY_CHECK_USE();
108 4 : __static_key_slow_dec(key, 0, NULL);
109 4 : }
110 : EXPORT_SYMBOL_GPL(static_key_slow_dec);
111 :
112 0 : void static_key_slow_dec_deferred(struct static_key_deferred *key)
113 : {
114 : STATIC_KEY_CHECK_USE();
115 0 : __static_key_slow_dec(&key->key, key->timeout, &key->work);
116 0 : }
117 : EXPORT_SYMBOL_GPL(static_key_slow_dec_deferred);
118 :
119 1 : void jump_label_rate_limit(struct static_key_deferred *key,
120 : unsigned long rl)
121 : {
122 : STATIC_KEY_CHECK_USE();
123 1 : key->timeout = rl;
124 2 : INIT_DELAYED_WORK(&key->work, jump_label_update_timeout);
125 1 : }
126 : EXPORT_SYMBOL_GPL(jump_label_rate_limit);
127 :
128 : static int addr_conflict(struct jump_entry *entry, void *start, void *end)
129 : {
130 0 : if (entry->code <= (unsigned long)end &&
131 0 : entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start)
132 : return 1;
133 :
134 : return 0;
135 : }
136 :
137 : static int __jump_label_text_reserved(struct jump_entry *iter_start,
138 : struct jump_entry *iter_stop, void *start, void *end)
139 : {
140 0 : struct jump_entry *iter;
141 :
142 : iter = iter_start;
143 0 : while (iter < iter_stop) {
144 0 : if (addr_conflict(iter, start, end))
145 : return 1;
146 0 : iter++;
147 : }
148 :
149 : return 0;
150 : }
151 :
152 : /*
153 : * Update code which is definitely not currently executing.
154 : * Architectures which need heavyweight synchronization to modify
155 : * running code can override this to make the non-live update case
156 : * cheaper.
157 : */
158 0 : void __weak __init_or_module arch_jump_label_transform_static(struct jump_entry *entry,
159 : enum jump_label_type type)
160 : {
161 0 : arch_jump_label_transform(entry, type);
162 0 : }
163 :
164 5 : static void __jump_label_update(struct static_key *key,
165 : struct jump_entry *entry,
166 : struct jump_entry *stop, int enable)
167 : {
168 39 : for (; (entry < stop) &&
169 17 : (entry->key == (jump_label_t)(unsigned long)key);
170 12 : entry++) {
171 : /*
172 : * entry->code set to 0 invalidates module init text sections
173 : * kernel_text_address() verifies we are not in core kernel
174 : * init code, see jump_label_invalidate_module_init().
175 : */
176 12 : if (entry->code && kernel_text_address(entry->code))
177 12 : arch_jump_label_transform(entry, enable);
178 : }
179 5 : }
180 :
181 59 : static enum jump_label_type jump_label_type(struct static_key *key)
182 : {
183 : bool true_branch = jump_label_get_branch_default(key);
184 : bool state = static_key_enabled(key);
185 :
186 59 : if ((!true_branch && state) || (true_branch && !state))
187 : return JUMP_LABEL_ENABLE;
188 :
189 : return JUMP_LABEL_DISABLE;
190 : }
191 :
192 1 : void __init jump_label_init(void)
193 : {
194 : struct jump_entry *iter_start = __start___jump_table;
195 : struct jump_entry *iter_stop = __stop___jump_table;
196 : struct static_key *key = NULL;
197 : struct jump_entry *iter;
198 :
199 : jump_label_lock();
200 : jump_label_sort_entries(iter_start, iter_stop);
201 :
202 60 : for (iter = iter_start; iter < iter_stop; iter++) {
203 : struct static_key *iterk;
204 :
205 59 : iterk = (struct static_key *)(unsigned long)iter->key;
206 59 : arch_jump_label_transform_static(iter, jump_label_type(iterk));
207 59 : if (iterk == key)
208 28 : continue;
209 :
210 : key = iterk;
211 : /*
212 : * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH.
213 : */
214 31 : *((unsigned long *)&key->entries) += (unsigned long)iter;
215 : #ifdef CONFIG_MODULES
216 31 : key->next = NULL;
217 : #endif
218 : }
219 1 : static_key_initialized = true;
220 : jump_label_unlock();
221 1 : }
222 :
223 : #ifdef CONFIG_MODULES
224 :
225 : struct static_key_mod {
226 : struct static_key_mod *next;
227 : struct jump_entry *entries;
228 : struct module *mod;
229 : };
230 :
231 0 : static int __jump_label_mod_text_reserved(void *start, void *end)
232 : {
233 : struct module *mod;
234 :
235 0 : mod = __module_text_address((unsigned long)start);
236 0 : if (!mod)
237 : return 0;
238 :
239 0 : WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
240 :
241 0 : return __jump_label_text_reserved(mod->jump_entries,
242 0 : mod->jump_entries + mod->num_jump_entries,
243 : start, end);
244 : }
245 :
246 5 : static void __jump_label_mod_update(struct static_key *key, int enable)
247 : {
248 5 : struct static_key_mod *mod = key->next;
249 :
250 10 : while (mod) {
251 0 : struct module *m = mod->mod;
252 :
253 0 : __jump_label_update(key, mod->entries,
254 0 : m->jump_entries + m->num_jump_entries,
255 : enable);
256 0 : mod = mod->next;
257 : }
258 5 : }
259 :
260 : /***
261 : * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
262 : * @mod: module to patch
263 : *
264 : * Allow for run-time selection of the optimal nops. Before the module
265 : * loads patch these with arch_get_jump_label_nop(), which is specified by
266 : * the arch specific jump label code.
267 : */
268 0 : void jump_label_apply_nops(struct module *mod)
269 : {
270 0 : struct jump_entry *iter_start = mod->jump_entries;
271 0 : struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
272 : struct jump_entry *iter;
273 :
274 : /* if the module doesn't have jump label entries, just return */
275 0 : if (iter_start == iter_stop)
276 0 : return;
277 :
278 0 : for (iter = iter_start; iter < iter_stop; iter++) {
279 0 : arch_jump_label_transform_static(iter, JUMP_LABEL_DISABLE);
280 : }
281 : }
282 :
283 16 : static int jump_label_add_module(struct module *mod)
284 : {
285 16 : struct jump_entry *iter_start = mod->jump_entries;
286 16 : struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
287 : struct jump_entry *iter;
288 : struct static_key *key = NULL;
289 : struct static_key_mod *jlm;
290 :
291 : /* if the module doesn't have jump label entries, just return */
292 16 : if (iter_start == iter_stop)
293 : return 0;
294 :
295 : jump_label_sort_entries(iter_start, iter_stop);
296 :
297 0 : for (iter = iter_start; iter < iter_stop; iter++) {
298 : struct static_key *iterk;
299 :
300 0 : iterk = (struct static_key *)(unsigned long)iter->key;
301 0 : if (iterk == key)
302 0 : continue;
303 :
304 : key = iterk;
305 0 : if (__module_address(iter->key) == mod) {
306 : /*
307 : * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH.
308 : */
309 0 : *((unsigned long *)&key->entries) += (unsigned long)iter;
310 0 : key->next = NULL;
311 0 : continue;
312 : }
313 : jlm = kzalloc(sizeof(struct static_key_mod), GFP_KERNEL);
314 0 : if (!jlm)
315 : return -ENOMEM;
316 0 : jlm->mod = mod;
317 0 : jlm->entries = iter;
318 0 : jlm->next = key->next;
319 0 : key->next = jlm;
320 :
321 0 : if (jump_label_type(key) == JUMP_LABEL_ENABLE)
322 0 : __jump_label_update(key, iter, iter_stop, JUMP_LABEL_ENABLE);
323 : }
324 :
325 : return 0;
326 : }
327 :
328 0 : static void jump_label_del_module(struct module *mod)
329 : {
330 0 : struct jump_entry *iter_start = mod->jump_entries;
331 0 : struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
332 : struct jump_entry *iter;
333 : struct static_key *key = NULL;
334 : struct static_key_mod *jlm, **prev;
335 :
336 0 : for (iter = iter_start; iter < iter_stop; iter++) {
337 0 : if (iter->key == (jump_label_t)(unsigned long)key)
338 0 : continue;
339 :
340 0 : key = (struct static_key *)(unsigned long)iter->key;
341 :
342 0 : if (__module_address(iter->key) == mod)
343 0 : continue;
344 :
345 0 : prev = &key->next;
346 0 : jlm = key->next;
347 :
348 0 : while (jlm && jlm->mod != mod) {
349 0 : prev = &jlm->next;
350 0 : jlm = jlm->next;
351 : }
352 :
353 0 : if (jlm) {
354 0 : *prev = jlm->next;
355 0 : kfree(jlm);
356 : }
357 : }
358 0 : }
359 :
360 16 : static void jump_label_invalidate_module_init(struct module *mod)
361 : {
362 16 : struct jump_entry *iter_start = mod->jump_entries;
363 16 : struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
364 : struct jump_entry *iter;
365 :
366 16 : for (iter = iter_start; iter < iter_stop; iter++) {
367 0 : if (within_module_init(iter->code, mod))
368 0 : iter->code = 0;
369 : }
370 16 : }
371 :
372 : static int
373 32 : jump_label_module_notify(struct notifier_block *self, unsigned long val,
374 : void *data)
375 : {
376 : struct module *mod = data;
377 : int ret = 0;
378 :
379 32 : switch (val) {
380 : case MODULE_STATE_COMING:
381 : jump_label_lock();
382 16 : ret = jump_label_add_module(mod);
383 16 : if (ret)
384 0 : jump_label_del_module(mod);
385 : jump_label_unlock();
386 : break;
387 : case MODULE_STATE_GOING:
388 : jump_label_lock();
389 0 : jump_label_del_module(mod);
390 : jump_label_unlock();
391 : break;
392 : case MODULE_STATE_LIVE:
393 : jump_label_lock();
394 16 : jump_label_invalidate_module_init(mod);
395 : jump_label_unlock();
396 : break;
397 : }
398 :
399 32 : return notifier_from_errno(ret);
400 : }
401 :
402 : struct notifier_block jump_label_module_nb = {
403 : .notifier_call = jump_label_module_notify,
404 : .priority = 1, /* higher than tracepoints */
405 : };
406 :
407 1 : static __init int jump_label_init_module(void)
408 : {
409 1 : return register_module_notifier(&jump_label_module_nb);
410 : }
411 : early_initcall(jump_label_init_module);
412 :
413 : #endif /* CONFIG_MODULES */
414 :
415 : /***
416 : * jump_label_text_reserved - check if addr range is reserved
417 : * @start: start text addr
418 : * @end: end text addr
419 : *
420 : * checks if the text addr located between @start and @end
421 : * overlaps with any of the jump label patch addresses. Code
422 : * that wants to modify kernel text should first verify that
423 : * it does not overlap with any of the jump label addresses.
424 : * Caller must hold jump_label_mutex.
425 : *
426 : * returns 1 if there is an overlap, 0 otherwise
427 : */
428 0 : int jump_label_text_reserved(void *start, void *end)
429 : {
430 : int ret = __jump_label_text_reserved(__start___jump_table,
431 : __stop___jump_table, start, end);
432 :
433 0 : if (ret)
434 : return ret;
435 :
436 : #ifdef CONFIG_MODULES
437 0 : ret = __jump_label_mod_text_reserved(start, end);
438 : #endif
439 0 : return ret;
440 : }
441 :
442 5 : static void jump_label_update(struct static_key *key, int enable)
443 : {
444 : struct jump_entry *stop = __stop___jump_table;
445 : struct jump_entry *entry = jump_label_get_entries(key);
446 :
447 : #ifdef CONFIG_MODULES
448 5 : struct module *mod = __module_address((unsigned long)key);
449 :
450 5 : __jump_label_mod_update(key, enable);
451 :
452 5 : if (mod)
453 0 : stop = mod->jump_entries + mod->num_jump_entries;
454 : #endif
455 : /* if there are no users, entry can be NULL */
456 5 : if (entry)
457 5 : __jump_label_update(key, entry, stop, enable);
458 5 : }
459 :
460 : #endif
|
