rcupdate.h 33.9 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
/*
2
 * Read-Copy Update mechanism for mutual exclusion
Linus Torvalds's avatar
Linus Torvalds committed
3 4 5 6 7 8 9 10 11 12 13 14
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
15 16
 * along with this program; if not, you can access it online at
 * http://www.gnu.org/licenses/gpl-2.0.html.
Linus Torvalds's avatar
Linus Torvalds committed
17
 *
18
 * Copyright IBM Corporation, 2001
Linus Torvalds's avatar
Linus Torvalds committed
19 20
 *
 * Author: Dipankar Sarma <dipankar@in.ibm.com>
21
 *
22
 * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
Linus Torvalds's avatar
Linus Torvalds committed
23 24 25 26 27 28
 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
 * Papers:
 * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
 * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
 *
 * For detailed explanation of Read-Copy Update mechanism see -
29
 *		http://lse.sourceforge.net/locking/rcupdate.html
Linus Torvalds's avatar
Linus Torvalds committed
30 31 32 33 34 35
 *
 */

#ifndef __LINUX_RCUPDATE_H
#define __LINUX_RCUPDATE_H

36
#include <linux/types.h>
37
#include <linux/compiler.h>
38
#include <linux/atomic.h>
39
#include <linux/irqflags.h>
40 41 42 43 44
#include <linux/preempt.h>
#include <linux/bottom_half.h>
#include <linux/lockdep.h>
#include <asm/processor.h>
#include <linux/cpumask.h>
45

46 47
#define ULONG_CMP_GE(a, b)	(ULONG_MAX / 2 >= (a) - (b))
#define ULONG_CMP_LT(a, b)	(ULONG_MAX / 2 < (a) - (b))
48
#define ulong2long(a)		(*(long *)(&(a)))
49

50
/* Exported common interfaces */
51
void call_rcu(struct rcu_head *head, rcu_callback_t func);
52
void rcu_barrier_tasks(void);
53
void synchronize_rcu(void);
54

55 56
#ifdef CONFIG_PREEMPT_RCU

57 58
void __rcu_read_lock(void);
void __rcu_read_unlock(void);
59

60 61 62 63 64 65 66 67
/*
 * Defined as a macro as it is a very low level header included from
 * areas that don't even know about current.  This gives the rcu_read_lock()
 * nesting depth, but makes sense only if CONFIG_PREEMPT_RCU -- in other
 * types of kernel builds, the rcu_read_lock() nesting depth is unknowable.
 */
#define rcu_preempt_depth() (current->rcu_read_lock_nesting)

68 69 70 71
#else /* #ifdef CONFIG_PREEMPT_RCU */

static inline void __rcu_read_lock(void)
{
72 73
	if (IS_ENABLED(CONFIG_PREEMPT_COUNT))
		preempt_disable();
74 75 76 77
}

static inline void __rcu_read_unlock(void)
{
78 79
	if (IS_ENABLED(CONFIG_PREEMPT_COUNT))
		preempt_enable();
80 81 82 83 84 85 86 87 88 89
}

static inline int rcu_preempt_depth(void)
{
	return 0;
}

#endif /* #else #ifdef CONFIG_PREEMPT_RCU */

/* Internal to kernel */
90
void rcu_init(void);
91
extern int rcu_scheduler_active __read_mostly;
92
void rcu_check_callbacks(int user);
93
void rcu_report_dead(unsigned int cpu);
94
void rcutree_migrate_callbacks(int cpu);
95

96 97 98 99
#ifdef CONFIG_RCU_STALL_COMMON
void rcu_sysrq_start(void);
void rcu_sysrq_end(void);
#else /* #ifdef CONFIG_RCU_STALL_COMMON */
100 101
static inline void rcu_sysrq_start(void) { }
static inline void rcu_sysrq_end(void) { }
102 103
#endif /* #else #ifdef CONFIG_RCU_STALL_COMMON */

104
#ifdef CONFIG_NO_HZ_FULL
105 106
void rcu_user_enter(void);
void rcu_user_exit(void);
107 108 109
#else
static inline void rcu_user_enter(void) { }
static inline void rcu_user_exit(void) { }
110
#endif /* CONFIG_NO_HZ_FULL */
111

112 113 114
#ifdef CONFIG_RCU_NOCB_CPU
void rcu_init_nohz(void);
#else /* #ifdef CONFIG_RCU_NOCB_CPU */
115
static inline void rcu_init_nohz(void) { }
116 117
#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */

118 119 120 121
/**
 * RCU_NONIDLE - Indicate idle-loop code that needs RCU readers
 * @a: Code that RCU needs to pay attention to.
 *
122 123 124 125
 * RCU read-side critical sections are forbidden in the inner idle loop,
 * that is, between the rcu_idle_enter() and the rcu_idle_exit() -- RCU
 * will happily ignore any such read-side critical sections.  However,
 * things like powertop need tracepoints in the inner idle loop.
126 127
 *
 * This macro provides the way out:  RCU_NONIDLE(do_something_with_RCU())
128 129
 * will tell RCU that it needs to pay attention, invoke its argument
 * (in this example, calling the do_something_with_RCU() function),
130
 * and then tell RCU to go back to ignoring this CPU.  It is permissible
131 132 133 134
 * to nest RCU_NONIDLE() wrappers, but not indefinitely (but the limit is
 * on the order of a million or so, even on 32-bit systems).  It is
 * not legal to block within RCU_NONIDLE(), nor is it permissible to
 * transfer control either into or out of RCU_NONIDLE()'s statement.
135 136 137
 */
#define RCU_NONIDLE(a) \
	do { \
138
		rcu_irq_enter_irqson(); \
139
		do { a; } while (0); \
140
		rcu_irq_exit_irqson(); \
141 142
	} while (0)

143
/*
144 145
 * Note a quasi-voluntary context switch for RCU-tasks's benefit.
 * This is a macro rather than an inline function to avoid #include hell.
146 147
 */
#ifdef CONFIG_TASKS_RCU
148
#define rcu_tasks_qs(t) \
149
	do { \
150 151
		if (READ_ONCE((t)->rcu_tasks_holdout)) \
			WRITE_ONCE((t)->rcu_tasks_holdout, false); \
152
	} while (0)
153
#define rcu_note_voluntary_context_switch(t) rcu_tasks_qs(t)
154 155
void call_rcu_tasks(struct rcu_head *head, rcu_callback_t func);
void synchronize_rcu_tasks(void);
156 157
void exit_tasks_rcu_start(void);
void exit_tasks_rcu_finish(void);
158
#else /* #ifdef CONFIG_TASKS_RCU */
159
#define rcu_tasks_qs(t)	do { } while (0)
160
#define rcu_note_voluntary_context_switch(t) do { } while (0)
161
#define call_rcu_tasks call_rcu
162
#define synchronize_rcu_tasks synchronize_rcu
163 164
static inline void exit_tasks_rcu_start(void) { }
static inline void exit_tasks_rcu_finish(void) { }
165 166
#endif /* #else #ifdef CONFIG_TASKS_RCU */

167
/**
168
 * cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU
169 170 171 172 173
 *
 * This macro resembles cond_resched(), except that it is defined to
 * report potential quiescent states to RCU-tasks even if the cond_resched()
 * machinery were to be shut off, as some advocate for PREEMPT kernels.
 */
174
#define cond_resched_tasks_rcu_qs() \
175
do { \
176
	rcu_tasks_qs(current); \
177
	cond_resched(); \
178 179
} while (0)

180 181 182 183 184
/*
 * Infrastructure to implement the synchronize_() primitives in
 * TREE_RCU and rcu_barrier_() primitives in TINY_RCU.
 */

185
#if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU)
186
#include <linux/rcutree.h>
187
#elif defined(CONFIG_TINY_RCU)
188
#include <linux/rcutiny.h>
189 190
#else
#error "Unknown RCU implementation specified to kernel configuration"
191
#endif
192

193
/*
194 195 196 197 198 199
 * The init_rcu_head_on_stack() and destroy_rcu_head_on_stack() calls
 * are needed for dynamic initialization and destruction of rcu_head
 * on the stack, and init_rcu_head()/destroy_rcu_head() are needed for
 * dynamic initialization and destruction of statically allocated rcu_head
 * structures.  However, rcu_head structures allocated dynamically in the
 * heap don't need any initialization.
200 201
 */
#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
202 203
void init_rcu_head(struct rcu_head *head);
void destroy_rcu_head(struct rcu_head *head);
204 205
void init_rcu_head_on_stack(struct rcu_head *head);
void destroy_rcu_head_on_stack(struct rcu_head *head);
206
#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
207 208 209 210
static inline void init_rcu_head(struct rcu_head *head) { }
static inline void destroy_rcu_head(struct rcu_head *head) { }
static inline void init_rcu_head_on_stack(struct rcu_head *head) { }
static inline void destroy_rcu_head_on_stack(struct rcu_head *head) { }
211
#endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
212

213 214 215
#if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU)
bool rcu_lockdep_current_cpu_online(void);
#else /* #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
216
static inline bool rcu_lockdep_current_cpu_online(void) { return true; }
217 218
#endif /* #else #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */

219
#ifdef CONFIG_DEBUG_LOCK_ALLOC
220

221 222
static inline void rcu_lock_acquire(struct lockdep_map *map)
{
223
	lock_acquire(map, 0, 0, 2, 0, NULL, _THIS_IP_);
224 225 226 227 228 229 230
}

static inline void rcu_lock_release(struct lockdep_map *map)
{
	lock_release(map, 1, _THIS_IP_);
}

231
extern struct lockdep_map rcu_lock_map;
232 233
extern struct lockdep_map rcu_bh_lock_map;
extern struct lockdep_map rcu_sched_lock_map;
234
extern struct lockdep_map rcu_callback_map;
235
int debug_lockdep_rcu_enabled(void);
236
int rcu_read_lock_held(void);
237
int rcu_read_lock_bh_held(void);
238
int rcu_read_lock_sched_held(void);
239 240 241

#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */

242 243
# define rcu_lock_acquire(a)		do { } while (0)
# define rcu_lock_release(a)		do { } while (0)
244 245 246 247 248 249 250 251 252 253 254 255 256

static inline int rcu_read_lock_held(void)
{
	return 1;
}

static inline int rcu_read_lock_bh_held(void)
{
	return 1;
}

static inline int rcu_read_lock_sched_held(void)
{
257
	return !preemptible();
258 259 260 261 262
}
#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */

#ifdef CONFIG_PROVE_RCU

263 264 265 266 267 268 269 270 271 272 273 274 275 276
/**
 * RCU_LOCKDEP_WARN - emit lockdep splat if specified condition is met
 * @c: condition to check
 * @s: informative message
 */
#define RCU_LOCKDEP_WARN(c, s)						\
	do {								\
		static bool __section(.data.unlikely) __warned;		\
		if (debug_lockdep_rcu_enabled() && !__warned && (c)) {	\
			__warned = true;				\
			lockdep_rcu_suspicious(__FILE__, __LINE__, s);	\
		}							\
	} while (0)

277 278 279
#if defined(CONFIG_PROVE_RCU) && !defined(CONFIG_PREEMPT_RCU)
static inline void rcu_preempt_sleep_check(void)
{
280 281
	RCU_LOCKDEP_WARN(lock_is_held(&rcu_lock_map),
			 "Illegal context switch in RCU read-side critical section");
282 283
}
#else /* #ifdef CONFIG_PROVE_RCU */
284
static inline void rcu_preempt_sleep_check(void) { }
285 286
#endif /* #else #ifdef CONFIG_PROVE_RCU */

287 288
#define rcu_sleep_check()						\
	do {								\
289
		rcu_preempt_sleep_check();				\
290 291 292 293
		RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map),	\
				 "Illegal context switch in RCU-bh read-side critical section"); \
		RCU_LOCKDEP_WARN(lock_is_held(&rcu_sched_lock_map),	\
				 "Illegal context switch in RCU-sched read-side critical section"); \
294 295
	} while (0)

296 297
#else /* #ifdef CONFIG_PROVE_RCU */

298
#define RCU_LOCKDEP_WARN(c, s) do { } while (0)
299
#define rcu_sleep_check() do { } while (0)
300 301 302 303 304 305 306

#endif /* #else #ifdef CONFIG_PROVE_RCU */

/*
 * Helper functions for rcu_dereference_check(), rcu_dereference_protected()
 * and rcu_assign_pointer().  Some of these could be folded into their
 * callers, but they are left separate in order to ease introduction of
307 308
 * multiple pointers markings to match different RCU implementations
 * (e.g., __srcu), should this make sense in the future.
309
 */
310 311 312 313 314 315 316 317

#ifdef __CHECKER__
#define rcu_dereference_sparse(p, space) \
	((void)(((typeof(*p) space *)p) == p))
#else /* #ifdef __CHECKER__ */
#define rcu_dereference_sparse(p, space)
#endif /* #else #ifdef __CHECKER__ */

318
#define __rcu_access_pointer(p, space) \
319
({ \
320
	typeof(*p) *_________p1 = (typeof(*p) *__force)READ_ONCE(p); \
321 322 323
	rcu_dereference_sparse(p, space); \
	((typeof(*p) __force __kernel *)(_________p1)); \
})
324
#define __rcu_dereference_check(p, c, space) \
325
({ \
326
	/* Dependency order vs. p above. */ \
327
	typeof(*p) *________p1 = (typeof(*p) *__force)READ_ONCE(p); \
328
	RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_check() usage"); \
329
	rcu_dereference_sparse(p, space); \
330
	((typeof(*p) __force __kernel *)(________p1)); \
331
})
332
#define __rcu_dereference_protected(p, c, space) \
333
({ \
334
	RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_protected() usage"); \
335 336 337
	rcu_dereference_sparse(p, space); \
	((typeof(*p) __force __kernel *)(p)); \
})
338 339 340
#define rcu_dereference_raw(p) \
({ \
	/* Dependency order vs. p above. */ \
341
	typeof(p) ________p1 = READ_ONCE(p); \
342 343
	((typeof(*p) __force __kernel *)(________p1)); \
})
344

345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381
/**
 * RCU_INITIALIZER() - statically initialize an RCU-protected global variable
 * @v: The value to statically initialize with.
 */
#define RCU_INITIALIZER(v) (typeof(*(v)) __force __rcu *)(v)

/**
 * rcu_assign_pointer() - assign to RCU-protected pointer
 * @p: pointer to assign to
 * @v: value to assign (publish)
 *
 * Assigns the specified value to the specified RCU-protected
 * pointer, ensuring that any concurrent RCU readers will see
 * any prior initialization.
 *
 * Inserts memory barriers on architectures that require them
 * (which is most of them), and also prevents the compiler from
 * reordering the code that initializes the structure after the pointer
 * assignment.  More importantly, this call documents which pointers
 * will be dereferenced by RCU read-side code.
 *
 * In some special cases, you may use RCU_INIT_POINTER() instead
 * of rcu_assign_pointer().  RCU_INIT_POINTER() is a bit faster due
 * to the fact that it does not constrain either the CPU or the compiler.
 * That said, using RCU_INIT_POINTER() when you should have used
 * rcu_assign_pointer() is a very bad thing that results in
 * impossible-to-diagnose memory corruption.  So please be careful.
 * See the RCU_INIT_POINTER() comment header for details.
 *
 * Note that rcu_assign_pointer() evaluates each of its arguments only
 * once, appearances notwithstanding.  One of the "extra" evaluations
 * is in typeof() and the other visible only to sparse (__CHECKER__),
 * neither of which actually execute the argument.  As with most cpp
 * macros, this execute-arguments-only-once property is important, so
 * please be careful when making changes to rcu_assign_pointer() and the
 * other macros that it invokes.
 */
382 383 384 385 386 387 388 389 390 391
#define rcu_assign_pointer(p, v)					      \
({									      \
	uintptr_t _r_a_p__v = (uintptr_t)(v);				      \
									      \
	if (__builtin_constant_p(v) && (_r_a_p__v) == (uintptr_t)NULL)	      \
		WRITE_ONCE((p), (typeof(p))(_r_a_p__v));		      \
	else								      \
		smp_store_release(&p, RCU_INITIALIZER((typeof(p))_r_a_p__v)); \
	_r_a_p__v;							      \
})
392

393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408
/**
 * rcu_swap_protected() - swap an RCU and a regular pointer
 * @rcu_ptr: RCU pointer
 * @ptr: regular pointer
 * @c: the conditions under which the dereference will take place
 *
 * Perform swap(@rcu_ptr, @ptr) where @rcu_ptr is an RCU-annotated pointer and
 * @c is the argument that is passed to the rcu_dereference_protected() call
 * used to read that pointer.
 */
#define rcu_swap_protected(rcu_ptr, ptr, c) do {			\
	typeof(ptr) __tmp = rcu_dereference_protected((rcu_ptr), (c));	\
	rcu_assign_pointer((rcu_ptr), (ptr));				\
	(ptr) = __tmp;							\
} while (0)

409 410 411 412 413
/**
 * rcu_access_pointer() - fetch RCU pointer with no dereferencing
 * @p: The pointer to read
 *
 * Return the value of the specified RCU-protected pointer, but omit the
414 415 416 417 418 419
 * lockdep checks for being in an RCU read-side critical section.  This is
 * useful when the value of this pointer is accessed, but the pointer is
 * not dereferenced, for example, when testing an RCU-protected pointer
 * against NULL.  Although rcu_access_pointer() may also be used in cases
 * where update-side locks prevent the value of the pointer from changing,
 * you should instead use rcu_dereference_protected() for this use case.
420 421 422 423 424 425 426
 *
 * It is also permissible to use rcu_access_pointer() when read-side
 * access to the pointer was removed at least one grace period ago, as
 * is the case in the context of the RCU callback that is freeing up
 * the data, or after a synchronize_rcu() returns.  This can be useful
 * when tearing down multi-linked structures after a grace period
 * has elapsed.
427 428 429
 */
#define rcu_access_pointer(p) __rcu_access_pointer((p), __rcu)

430
/**
431
 * rcu_dereference_check() - rcu_dereference with debug checking
432 433
 * @p: The pointer to read, prior to dereferencing
 * @c: The conditions under which the dereference will take place
434
 *
435
 * Do an rcu_dereference(), but check that the conditions under which the
436 437 438 439 440
 * dereference will take place are correct.  Typically the conditions
 * indicate the various locking conditions that should be held at that
 * point.  The check should return true if the conditions are satisfied.
 * An implicit check for being in an RCU read-side critical section
 * (rcu_read_lock()) is included.
441 442 443
 *
 * For example:
 *
444
 *	bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock));
445 446
 *
 * could be used to indicate to lockdep that foo->bar may only be dereferenced
447
 * if either rcu_read_lock() is held, or that the lock required to replace
448 449 450 451 452 453
 * the bar struct at foo->bar is held.
 *
 * Note that the list of conditions may also include indications of when a lock
 * need not be held, for example during initialisation or destruction of the
 * target struct:
 *
454
 *	bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock) ||
455
 *					      atomic_read(&foo->usage) == 0);
456 457 458 459 460 461
 *
 * Inserts memory barriers on architectures that require them
 * (currently only the Alpha), prevents the compiler from refetching
 * (and from merging fetches), and, more importantly, documents exactly
 * which pointers are protected by RCU and checks that the pointer is
 * annotated as __rcu.
462 463
 */
#define rcu_dereference_check(p, c) \
464
	__rcu_dereference_check((p), (c) || rcu_read_lock_held(), __rcu)
465 466 467 468 469 470 471 472 473

/**
 * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking
 * @p: The pointer to read, prior to dereferencing
 * @c: The conditions under which the dereference will take place
 *
 * This is the RCU-bh counterpart to rcu_dereference_check().
 */
#define rcu_dereference_bh_check(p, c) \
474
	__rcu_dereference_check((p), (c) || rcu_read_lock_bh_held(), __rcu)
475

476
/**
477 478 479 480 481 482 483
 * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking
 * @p: The pointer to read, prior to dereferencing
 * @c: The conditions under which the dereference will take place
 *
 * This is the RCU-sched counterpart to rcu_dereference_check().
 */
#define rcu_dereference_sched_check(p, c) \
484
	__rcu_dereference_check((p), (c) || rcu_read_lock_sched_held(), \
485 486
				__rcu)

487 488 489 490
/*
 * The tracing infrastructure traces RCU (we want that), but unfortunately
 * some of the RCU checks causes tracing to lock up the system.
 *
491
 * The no-tracing version of rcu_dereference_raw() must not call
492 493 494 495
 * rcu_read_lock_held().
 */
#define rcu_dereference_raw_notrace(p) __rcu_dereference_check((p), 1, __rcu)

496 497 498 499
/**
 * rcu_dereference_protected() - fetch RCU pointer when updates prevented
 * @p: The pointer to read, prior to dereferencing
 * @c: The conditions under which the dereference will take place
500 501
 *
 * Return the value of the specified RCU-protected pointer, but omit
502 503 504 505 506
 * the READ_ONCE().  This is useful in cases where update-side locks
 * prevent the value of the pointer from changing.  Please note that this
 * primitive does *not* prevent the compiler from repeating this reference
 * or combining it with other references, so it should not be used without
 * protection of appropriate locks.
507 508 509 510
 *
 * This function is only for update-side use.  Using this function
 * when protected only by rcu_read_lock() will result in infrequent
 * but very ugly failures.
511 512
 */
#define rcu_dereference_protected(p, c) \
513
	__rcu_dereference_protected((p), (c), __rcu)
514

515

516
/**
517 518
 * rcu_dereference() - fetch RCU-protected pointer for dereferencing
 * @p: The pointer to read, prior to dereferencing
519
 *
520
 * This is a simple wrapper around rcu_dereference_check().
521
 */
522
#define rcu_dereference(p) rcu_dereference_check(p, 0)
523

Linus Torvalds's avatar
Linus Torvalds committed
524
/**
525 526 527 528 529 530 531 532 533 534 535 536 537 538 539
 * rcu_dereference_bh() - fetch an RCU-bh-protected pointer for dereferencing
 * @p: The pointer to read, prior to dereferencing
 *
 * Makes rcu_dereference_check() do the dirty work.
 */
#define rcu_dereference_bh(p) rcu_dereference_bh_check(p, 0)

/**
 * rcu_dereference_sched() - fetch RCU-sched-protected pointer for dereferencing
 * @p: The pointer to read, prior to dereferencing
 *
 * Makes rcu_dereference_check() do the dirty work.
 */
#define rcu_dereference_sched(p) rcu_dereference_sched_check(p, 0)

540 541 542 543 544 545 546
/**
 * rcu_pointer_handoff() - Hand off a pointer from RCU to other mechanism
 * @p: The pointer to hand off
 *
 * This is simply an identity function, but it documents where a pointer
 * is handed off from RCU to some other synchronization mechanism, for
 * example, reference counting or locking.  In C11, it would map to
547 548
 * kill_dependency().  It could be used as follows::
 *
549 550 551 552 553 554 555 556 557 558 559 560 561
 *	rcu_read_lock();
 *	p = rcu_dereference(gp);
 *	long_lived = is_long_lived(p);
 *	if (long_lived) {
 *		if (!atomic_inc_not_zero(p->refcnt))
 *			long_lived = false;
 *		else
 *			p = rcu_pointer_handoff(p);
 *	}
 *	rcu_read_unlock();
 */
#define rcu_pointer_handoff(p) (p)

562 563
/**
 * rcu_read_lock() - mark the beginning of an RCU read-side critical section
Linus Torvalds's avatar
Linus Torvalds committed
564
 *
565
 * When synchronize_rcu() is invoked on one CPU while other CPUs
Linus Torvalds's avatar
Linus Torvalds committed
566
 * are within RCU read-side critical sections, then the
567
 * synchronize_rcu() is guaranteed to block until after all the other
Linus Torvalds's avatar
Linus Torvalds committed
568 569 570 571 572 573
 * CPUs exit their critical sections.  Similarly, if call_rcu() is invoked
 * on one CPU while other CPUs are within RCU read-side critical
 * sections, invocation of the corresponding RCU callback is deferred
 * until after the all the other CPUs exit their critical sections.
 *
 * Note, however, that RCU callbacks are permitted to run concurrently
574
 * with new RCU read-side critical sections.  One way that this can happen
Linus Torvalds's avatar
Linus Torvalds committed
575 576 577 578 579 580 581 582 583 584 585 586 587 588
 * is via the following sequence of events: (1) CPU 0 enters an RCU
 * read-side critical section, (2) CPU 1 invokes call_rcu() to register
 * an RCU callback, (3) CPU 0 exits the RCU read-side critical section,
 * (4) CPU 2 enters a RCU read-side critical section, (5) the RCU
 * callback is invoked.  This is legal, because the RCU read-side critical
 * section that was running concurrently with the call_rcu() (and which
 * therefore might be referencing something that the corresponding RCU
 * callback would free up) has completed before the corresponding
 * RCU callback is invoked.
 *
 * RCU read-side critical sections may be nested.  Any deferred actions
 * will be deferred until the outermost RCU read-side critical section
 * completes.
 *
589 590 591 592 593
 * You can avoid reading and understanding the next paragraph by
 * following this rule: don't put anything in an rcu_read_lock() RCU
 * read-side critical section that would block in a !PREEMPT kernel.
 * But if you want the full story, read on!
 *
594 595
 * In non-preemptible RCU implementations (TREE_RCU and TINY_RCU),
 * it is illegal to block while in an RCU read-side critical section.
596
 * In preemptible RCU implementations (PREEMPT_RCU) in CONFIG_PREEMPT
597 598 599 600 601
 * kernel builds, RCU read-side critical sections may be preempted,
 * but explicit blocking is illegal.  Finally, in preemptible RCU
 * implementations in real-time (with -rt patchset) kernel builds, RCU
 * read-side critical sections may be preempted and they may also block, but
 * only when acquiring spinlocks that are subject to priority inheritance.
Linus Torvalds's avatar
Linus Torvalds committed
602
 */
603 604 605 606
static inline void rcu_read_lock(void)
{
	__rcu_read_lock();
	__acquire(RCU);
607
	rcu_lock_acquire(&rcu_lock_map);
608 609
	RCU_LOCKDEP_WARN(!rcu_is_watching(),
			 "rcu_read_lock() used illegally while idle");
610
}
Linus Torvalds's avatar
Linus Torvalds committed
611 612 613 614 615 616 617 618 619 620

/*
 * So where is rcu_write_lock()?  It does not exist, as there is no
 * way for writers to lock out RCU readers.  This is a feature, not
 * a bug -- this property is what provides RCU's performance benefits.
 * Of course, writers must coordinate with each other.  The normal
 * spinlock primitives work well for this, but any other technique may be
 * used as well.  RCU does not care how the writers keep out of each
 * others' way, as long as they do so.
 */
621 622

/**
623
 * rcu_read_unlock() - marks the end of an RCU read-side critical section.
624
 *
625 626 627 628 629 630
 * In most situations, rcu_read_unlock() is immune from deadlock.
 * However, in kernels built with CONFIG_RCU_BOOST, rcu_read_unlock()
 * is responsible for deboosting, which it does via rt_mutex_unlock().
 * Unfortunately, this function acquires the scheduler's runqueue and
 * priority-inheritance spinlocks.  This means that deadlock could result
 * if the caller of rcu_read_unlock() already holds one of these locks or
631
 * any lock that is ever acquired while holding them.
632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652
 *
 * That said, RCU readers are never priority boosted unless they were
 * preempted.  Therefore, one way to avoid deadlock is to make sure
 * that preemption never happens within any RCU read-side critical
 * section whose outermost rcu_read_unlock() is called with one of
 * rt_mutex_unlock()'s locks held.  Such preemption can be avoided in
 * a number of ways, for example, by invoking preempt_disable() before
 * critical section's outermost rcu_read_lock().
 *
 * Given that the set of locks acquired by rt_mutex_unlock() might change
 * at any time, a somewhat more future-proofed approach is to make sure
 * that that preemption never happens within any RCU read-side critical
 * section whose outermost rcu_read_unlock() is called with irqs disabled.
 * This approach relies on the fact that rt_mutex_unlock() currently only
 * acquires irq-disabled locks.
 *
 * The second of these two approaches is best in most situations,
 * however, the first approach can also be useful, at least to those
 * developers willing to keep abreast of the set of locks acquired by
 * rt_mutex_unlock().
 *
653 654
 * See rcu_read_lock() for more information.
 */
655 656
static inline void rcu_read_unlock(void)
{
657 658
	RCU_LOCKDEP_WARN(!rcu_is_watching(),
			 "rcu_read_unlock() used illegally while idle");
659 660
	__release(RCU);
	__rcu_read_unlock();
661
	rcu_lock_release(&rcu_lock_map); /* Keep acq info for rls diags. */
662
}
Linus Torvalds's avatar
Linus Torvalds committed
663 664

/**
665
 * rcu_read_lock_bh() - mark the beginning of an RCU-bh critical section
Linus Torvalds's avatar
Linus Torvalds committed
666
 *
667
 * This is equivalent of rcu_read_lock(), but also disables softirqs.
668 669
 * Note that anything else that disables softirqs can also serve as
 * an RCU read-side critical section.
670 671 672 673 674
 *
 * Note that rcu_read_lock_bh() and the matching rcu_read_unlock_bh()
 * must occur in the same context, for example, it is illegal to invoke
 * rcu_read_unlock_bh() from one task if the matching rcu_read_lock_bh()
 * was invoked from some other task.
Linus Torvalds's avatar
Linus Torvalds committed
675
 */
676 677
static inline void rcu_read_lock_bh(void)
{
678
	local_bh_disable();
679
	__acquire(RCU_BH);
680
	rcu_lock_acquire(&rcu_bh_lock_map);
681 682
	RCU_LOCKDEP_WARN(!rcu_is_watching(),
			 "rcu_read_lock_bh() used illegally while idle");
683
}
Linus Torvalds's avatar
Linus Torvalds committed
684 685 686 687 688 689

/*
 * rcu_read_unlock_bh - marks the end of a softirq-only RCU critical section
 *
 * See rcu_read_lock_bh() for more information.
 */
690 691
static inline void rcu_read_unlock_bh(void)
{
692 693
	RCU_LOCKDEP_WARN(!rcu_is_watching(),
			 "rcu_read_unlock_bh() used illegally while idle");
694
	rcu_lock_release(&rcu_bh_lock_map);
695
	__release(RCU_BH);
696
	local_bh_enable();
697
}
Linus Torvalds's avatar
Linus Torvalds committed
698

699
/**
700
 * rcu_read_lock_sched() - mark the beginning of a RCU-sched critical section
701
 *
702 703 704
 * This is equivalent of rcu_read_lock(), but disables preemption.
 * Read-side critical sections can also be introduced by anything else
 * that disables preemption, including local_irq_disable() and friends.
705 706 707 708 709
 *
 * Note that rcu_read_lock_sched() and the matching rcu_read_unlock_sched()
 * must occur in the same context, for example, it is illegal to invoke
 * rcu_read_unlock_sched() from process context if the matching
 * rcu_read_lock_sched() was invoked from an NMI handler.
710
 */
711 712 713
static inline void rcu_read_lock_sched(void)
{
	preempt_disable();
714
	__acquire(RCU_SCHED);
715
	rcu_lock_acquire(&rcu_sched_lock_map);
716 717
	RCU_LOCKDEP_WARN(!rcu_is_watching(),
			 "rcu_read_lock_sched() used illegally while idle");
718
}
719 720

/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
721
static inline notrace void rcu_read_lock_sched_notrace(void)
722 723
{
	preempt_disable_notrace();
724
	__acquire(RCU_SCHED);
725
}
726 727 728 729 730 731

/*
 * rcu_read_unlock_sched - marks the end of a RCU-classic critical section
 *
 * See rcu_read_lock_sched for more information.
 */
732 733
static inline void rcu_read_unlock_sched(void)
{
734 735
	RCU_LOCKDEP_WARN(!rcu_is_watching(),
			 "rcu_read_unlock_sched() used illegally while idle");
736
	rcu_lock_release(&rcu_sched_lock_map);
737
	__release(RCU_SCHED);
738 739
	preempt_enable();
}
740 741

/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
742
static inline notrace void rcu_read_unlock_sched_notrace(void)
743
{
744
	__release(RCU_SCHED);
745 746
	preempt_enable_notrace();
}
747

748 749
/**
 * RCU_INIT_POINTER() - initialize an RCU protected pointer
750 751
 * @p: The pointer to be initialized.
 * @v: The value to initialized the pointer to.
752
 *
753 754 755 756
 * Initialize an RCU-protected pointer in special cases where readers
 * do not need ordering constraints on the CPU or the compiler.  These
 * special cases are:
 *
757
 * 1.	This use of RCU_INIT_POINTER() is NULLing out the pointer *or*
758
 * 2.	The caller has taken whatever steps are required to prevent
759
 *	RCU readers from concurrently accessing this pointer *or*
760
 * 3.	The referenced data structure has already been exposed to
761 762 763 764
 *	readers either at compile time or via rcu_assign_pointer() *and*
 *
 *	a.	You have not made *any* reader-visible changes to
 *		this structure since then *or*
765 766 767 768 769 770 771 772 773 774 775 776 777 778 779
 *	b.	It is OK for readers accessing this structure from its
 *		new location to see the old state of the structure.  (For
 *		example, the changes were to statistical counters or to
 *		other state where exact synchronization is not required.)
 *
 * Failure to follow these rules governing use of RCU_INIT_POINTER() will
 * result in impossible-to-diagnose memory corruption.  As in the structures
 * will look OK in crash dumps, but any concurrent RCU readers might
 * see pre-initialized values of the referenced data structure.  So
 * please be very careful how you use RCU_INIT_POINTER()!!!
 *
 * If you are creating an RCU-protected linked structure that is accessed
 * by a single external-to-structure RCU-protected pointer, then you may
 * use RCU_INIT_POINTER() to initialize the internal RCU-protected
 * pointers, but you must use rcu_assign_pointer() to initialize the
780
 * external-to-structure pointer *after* you have completely initialized
781
 * the reader-accessible portions of the linked structure.
782 783 784
 *
 * Note that unlike rcu_assign_pointer(), RCU_INIT_POINTER() provides no
 * ordering guarantees for either the CPU or the compiler.
785 786
 */
#define RCU_INIT_POINTER(p, v) \
787
	do { \
788
		rcu_dereference_sparse(p, __rcu); \
789
		WRITE_ONCE(p, RCU_INITIALIZER(v)); \
790
	} while (0)
791

792 793
/**
 * RCU_POINTER_INITIALIZER() - statically initialize an RCU protected pointer
794 795
 * @p: The pointer to be initialized.
 * @v: The value to initialized the pointer to.
796 797 798 799
 *
 * GCC-style initialization for an RCU-protected pointer in a structure field.
 */
#define RCU_POINTER_INITIALIZER(p, v) \
800
		.p = RCU_INITIALIZER(v)
801

802 803 804 805 806 807 808 809 810 811 812 813
/*
 * Does the specified offset indicate that the corresponding rcu_head
 * structure can be handled by kfree_rcu()?
 */
#define __is_kfree_rcu_offset(offset) ((offset) < 4096)

/*
 * Helper macro for kfree_rcu() to prevent argument-expansion eyestrain.
 */
#define __kfree_rcu(head, offset) \
	do { \
		BUILD_BUG_ON(!__is_kfree_rcu_offset(offset)); \
814
		kfree_call_rcu(head, (rcu_callback_t)(unsigned long)(offset)); \
815 816
	} while (0)

817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838
/**
 * kfree_rcu() - kfree an object after a grace period.
 * @ptr:	pointer to kfree
 * @rcu_head:	the name of the struct rcu_head within the type of @ptr.
 *
 * Many rcu callbacks functions just call kfree() on the base structure.
 * These functions are trivial, but their size adds up, and furthermore
 * when they are used in a kernel module, that module must invoke the
 * high-latency rcu_barrier() function at module-unload time.
 *
 * The kfree_rcu() function handles this issue.  Rather than encoding a
 * function address in the embedded rcu_head structure, kfree_rcu() instead
 * encodes the offset of the rcu_head structure within the base structure.
 * Because the functions are not allowed in the low-order 4096 bytes of
 * kernel virtual memory, offsets up to 4095 bytes can be accommodated.
 * If the offset is larger than 4095 bytes, a compile-time error will
 * be generated in __kfree_rcu().  If this error is triggered, you can
 * either fall back to use of call_rcu() or rearrange the structure to
 * position the rcu_head structure into the first 4096 bytes.
 *
 * Note that the allowable offset might decrease in the future, for example,
 * to allow something like kmem_cache_free_rcu().
839 840 841
 *
 * The BUILD_BUG_ON check must not involve any function calls, hence the
 * checks are done in macros here.
842 843 844 845
 */
#define kfree_rcu(ptr, rcu_head)					\
	__kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head))

846

847 848 849 850 851 852
/*
 * Place this after a lock-acquisition primitive to guarantee that
 * an UNLOCK+LOCK pair acts as a full barrier.  This guarantee applies
 * if the UNLOCK and LOCK are executed by the same CPU or if the
 * UNLOCK and LOCK operate on the same lock variable.
 */
853
#ifdef CONFIG_ARCH_WEAK_RELEASE_ACQUIRE
854
#define smp_mb__after_unlock_lock()	smp_mb()  /* Full ordering for lock. */
855
#else /* #ifdef CONFIG_ARCH_WEAK_RELEASE_ACQUIRE */
856
#define smp_mb__after_unlock_lock()	do { } while (0)
857
#endif /* #else #ifdef CONFIG_ARCH_WEAK_RELEASE_ACQUIRE */
858

859

860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899
/* Has the specified rcu_head structure been handed to call_rcu()? */

/*
 * rcu_head_init - Initialize rcu_head for rcu_head_after_call_rcu()
 * @rhp: The rcu_head structure to initialize.
 *
 * If you intend to invoke rcu_head_after_call_rcu() to test whether a
 * given rcu_head structure has already been passed to call_rcu(), then
 * you must also invoke this rcu_head_init() function on it just after
 * allocating that structure.  Calls to this function must not race with
 * calls to call_rcu(), rcu_head_after_call_rcu(), or callback invocation.
 */
static inline void rcu_head_init(struct rcu_head *rhp)
{
	rhp->func = (rcu_callback_t)~0L;
}

/*
 * rcu_head_after_call_rcu - Has this rcu_head been passed to call_rcu()?
 * @rhp: The rcu_head structure to test.
 * @func: The function passed to call_rcu() along with @rhp.
 *
 * Returns @true if the @rhp has been passed to call_rcu() with @func,
 * and @false otherwise.  Emits a warning in any other case, including
 * the case where @rhp has already been invoked after a grace period.
 * Calls to this function must not race with callback invocation.  One way
 * to avoid such races is to enclose the call to rcu_head_after_call_rcu()
 * in an RCU read-side critical section that includes a read-side fetch
 * of the pointer to the structure containing @rhp.
 */
static inline bool
rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f)
{
	if (READ_ONCE(rhp->func) == f)
		return true;
	WARN_ON_ONCE(READ_ONCE(rhp->func) != (rcu_callback_t)~0L);
	return false;
}


900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921
/* Transitional pre-consolidation compatibility definitions. */

static inline void synchronize_rcu_bh(void)
{
	synchronize_rcu();
}

static inline void synchronize_rcu_bh_expedited(void)
{
	synchronize_rcu_expedited();
}

static inline void call_rcu_bh(struct rcu_head *head, rcu_callback_t func)
{
	call_rcu(head, func);
}

static inline void rcu_barrier_bh(void)
{
	rcu_barrier();
}

922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951
static inline void synchronize_sched(void)
{
	synchronize_rcu();
}

static inline void synchronize_sched_expedited(void)
{
	synchronize_rcu_expedited();
}

static inline void call_rcu_sched(struct rcu_head *head, rcu_callback_t func)
{
	call_rcu(head, func);
}

static inline void rcu_barrier_sched(void)
{
	rcu_barrier();
}

static inline unsigned long get_state_synchronize_sched(void)
{
	return get_state_synchronize_rcu();
}

static inline void cond_synchronize_sched(unsigned long oldstate)
{
	cond_synchronize_rcu(oldstate);
}

Linus Torvalds's avatar
Linus Torvalds committed
952
#endif /* __LINUX_RCUPDATE_H */