softirq.c 18.8 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5
/*
 *	linux/kernel/softirq.c
 *
 *	Copyright (C) 1992 Linus Torvalds
 *
6 7 8
 *	Distribute under GPLv2.
 *
 *	Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
Linus Torvalds's avatar
Linus Torvalds committed
9 10
 */

11 12
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

13
#include <linux/export.h>
Linus Torvalds's avatar
Linus Torvalds committed
14 15 16 17 18 19 20
#include <linux/kernel_stat.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/notifier.h>
#include <linux/percpu.h>
#include <linux/cpu.h>
21
#include <linux/freezer.h>
Linus Torvalds's avatar
Linus Torvalds committed
22 23
#include <linux/kthread.h>
#include <linux/rcupdate.h>
24
#include <linux/ftrace.h>
25
#include <linux/smp.h>
26
#include <linux/smpboot.h>
27
#include <linux/tick.h>
28
#include <linux/irq.h>
29 30

#define CREATE_TRACE_POINTS
31
#include <trace/events/irq.h>
Linus Torvalds's avatar
Linus Torvalds committed
32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55

/*
   - No shared variables, all the data are CPU local.
   - If a softirq needs serialization, let it serialize itself
     by its own spinlocks.
   - Even if softirq is serialized, only local cpu is marked for
     execution. Hence, we get something sort of weak cpu binding.
     Though it is still not clear, will it result in better locality
     or will not.

   Examples:
   - NET RX softirq. It is multithreaded and does not require
     any global serialization.
   - NET TX softirq. It kicks software netdevice queues, hence
     it is logically serialized per device, but this serialization
     is invisible to common code.
   - Tasklets: serialized wrt itself.
 */

#ifndef __ARCH_IRQ_STAT
irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
EXPORT_SYMBOL(irq_stat);
#endif

56
static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
Linus Torvalds's avatar
Linus Torvalds committed
57

58
DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
Linus Torvalds's avatar
Linus Torvalds committed
59

60
const char * const softirq_to_name[NR_SOFTIRQS] = {
61
	"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
62
	"TASKLET", "SCHED", "HRTIMER", "RCU"
63 64
};

Linus Torvalds's avatar
Linus Torvalds committed
65 66 67 68 69 70
/*
 * we cannot loop indefinitely here to avoid userspace starvation,
 * but we also don't want to introduce a worst case 1/HZ latency
 * to the pending events, so lets the scheduler to balance
 * the softirq load for us.
 */
71
static void wakeup_softirqd(void)
Linus Torvalds's avatar
Linus Torvalds committed
72 73
{
	/* Interrupts are disabled: no need to stop preemption */
74
	struct task_struct *tsk = __this_cpu_read(ksoftirqd);
Linus Torvalds's avatar
Linus Torvalds committed
75 76 77 78 79

	if (tsk && tsk->state != TASK_RUNNING)
		wake_up_process(tsk);
}

80 81 82 83 84 85 86 87 88 89
/*
 * preempt_count and SOFTIRQ_OFFSET usage:
 * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
 *   softirq processing.
 * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
 *   on local_bh_disable or local_bh_enable.
 * This lets us distinguish between whether we are currently processing
 * softirq and whether we just have bh disabled.
 */

90 91 92 93
/*
 * This one is for softirq.c-internal use,
 * where hardirqs are disabled legitimately:
 */
94
#ifdef CONFIG_TRACE_IRQFLAGS
95
void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
96 97 98 99 100 101
{
	unsigned long flags;

	WARN_ON_ONCE(in_irq());

	raw_local_irq_save(flags);
102
	/*
103
	 * The preempt tracer hooks into preempt_count_add and will break
104 105 106 107 108
	 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
	 * is set and before current->softirq_enabled is cleared.
	 * We must manually increment preempt_count here and manually
	 * call the trace_preempt_off later.
	 */
109
	__preempt_count_add(cnt);
110 111 112
	/*
	 * Were softirqs turned off above:
	 */
113
	if (softirq_count() == (cnt & SOFTIRQ_MASK))
114 115
		trace_softirqs_off(ip);
	raw_local_irq_restore(flags);
116

117
	if (preempt_count() == cnt)
118
		trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
119
}
120
EXPORT_SYMBOL(__local_bh_disable_ip);
121
#endif /* CONFIG_TRACE_IRQFLAGS */
122

123 124 125 126
static void __local_bh_enable(unsigned int cnt)
{
	WARN_ON_ONCE(!irqs_disabled());

127
	if (softirq_count() == (cnt & SOFTIRQ_MASK))
Davidlohr Bueso's avatar
Davidlohr Bueso committed
128
		trace_softirqs_on(_RET_IP_);
129
	preempt_count_sub(cnt);
130 131
}

132 133 134 135 136 137 138
/*
 * Special-case - softirqs can safely be enabled in
 * cond_resched_softirq(), or by __do_softirq(),
 * without processing still-pending softirqs:
 */
void _local_bh_enable(void)
{
139
	WARN_ON_ONCE(in_irq());
140
	__local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
141 142 143
}
EXPORT_SYMBOL(_local_bh_enable);

144
void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
145
{
146
	WARN_ON_ONCE(in_irq() || irqs_disabled());
147
#ifdef CONFIG_TRACE_IRQFLAGS
148
	local_irq_disable();
149
#endif
150 151 152
	/*
	 * Are softirqs going to be turned on now:
	 */
153
	if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
154
		trace_softirqs_on(ip);
155 156 157
	/*
	 * Keep preemption disabled until we are done with
	 * softirq processing:
158
	 */
159
	preempt_count_sub(cnt - 1);
160

161 162 163 164 165
	if (unlikely(!in_interrupt() && local_softirq_pending())) {
		/*
		 * Run softirq if any pending. And do it in its own stack
		 * as we may be calling this deep in a task call stack already.
		 */
166
		do_softirq();
167
	}
168

169
	preempt_count_dec();
170
#ifdef CONFIG_TRACE_IRQFLAGS
171
	local_irq_enable();
172
#endif
173 174
	preempt_check_resched();
}
175
EXPORT_SYMBOL(__local_bh_enable_ip);
176

Linus Torvalds's avatar
Linus Torvalds committed
177
/*
178 179 180 181 182 183
 * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
 * but break the loop if need_resched() is set or after 2 ms.
 * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
 * certain cases, such as stop_machine(), jiffies may cease to
 * increment and so we need the MAX_SOFTIRQ_RESTART limit as
 * well to make sure we eventually return from this method.
Linus Torvalds's avatar
Linus Torvalds committed
184
 *
Eric Dumazet's avatar
Eric Dumazet committed
185
 * These limits have been established via experimentation.
Linus Torvalds's avatar
Linus Torvalds committed
186 187 188 189
 * The two things to balance is latency against fairness -
 * we want to handle softirqs as soon as possible, but they
 * should not be able to lock up the box.
 */
Eric Dumazet's avatar
Eric Dumazet committed
190
#define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
191
#define MAX_SOFTIRQ_RESTART 10
Linus Torvalds's avatar
Linus Torvalds committed
192

193 194 195 196 197 198 199
#ifdef CONFIG_TRACE_IRQFLAGS
/*
 * When we run softirqs from irq_exit() and thus on the hardirq stack we need
 * to keep the lockdep irq context tracking as tight as possible in order to
 * not miss-qualify lock contexts and miss possible deadlocks.
 */

200
static inline bool lockdep_softirq_start(void)
201
{
202
	bool in_hardirq = false;
203

204 205
	if (trace_hardirq_context(current)) {
		in_hardirq = true;
206
		trace_hardirq_exit();
207 208
	}

209
	lockdep_softirq_enter();
210 211

	return in_hardirq;
212 213
}

214
static inline void lockdep_softirq_end(bool in_hardirq)
215 216
{
	lockdep_softirq_exit();
217 218

	if (in_hardirq)
219 220 221
		trace_hardirq_enter();
}
#else
222 223
static inline bool lockdep_softirq_start(void) { return false; }
static inline void lockdep_softirq_end(bool in_hardirq) { }
224 225
#endif

Linus Torvalds's avatar
Linus Torvalds committed
226 227
asmlinkage void __do_softirq(void)
{
Eric Dumazet's avatar
Eric Dumazet committed
228
	unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
229
	unsigned long old_flags = current->flags;
230
	int max_restart = MAX_SOFTIRQ_RESTART;
231
	struct softirq_action *h;
232
	bool in_hardirq;
233
	__u32 pending;
234
	int softirq_bit;
235
	int cpu;
236 237 238 239 240 241 242

	/*
	 * Mask out PF_MEMALLOC s current task context is borrowed for the
	 * softirq. A softirq handled such as network RX might set PF_MEMALLOC
	 * again if the socket is related to swap
	 */
	current->flags &= ~PF_MEMALLOC;
Linus Torvalds's avatar
Linus Torvalds committed
243 244

	pending = local_softirq_pending();
245
	account_irq_enter_time(current);
246

247
	__local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
248
	in_hardirq = lockdep_softirq_start();
Linus Torvalds's avatar
Linus Torvalds committed
249 250 251 252

	cpu = smp_processor_id();
restart:
	/* Reset the pending bitmask before enabling irqs */
253
	set_softirq_pending(0);
Linus Torvalds's avatar
Linus Torvalds committed
254

255
	local_irq_enable();
Linus Torvalds's avatar
Linus Torvalds committed
256 257 258

	h = softirq_vec;

259 260 261 262 263 264 265 266 267 268 269 270 271 272 273
	while ((softirq_bit = ffs(pending))) {
		unsigned int vec_nr;
		int prev_count;

		h += softirq_bit - 1;

		vec_nr = h - softirq_vec;
		prev_count = preempt_count();

		kstat_incr_softirqs_this_cpu(vec_nr);

		trace_softirq_entry(vec_nr);
		h->action(h);
		trace_softirq_exit(vec_nr);
		if (unlikely(prev_count != preempt_count())) {
274
			pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
275 276 277
			       vec_nr, softirq_to_name[vec_nr], h->action,
			       prev_count, preempt_count());
			preempt_count_set(prev_count);
Linus Torvalds's avatar
Linus Torvalds committed
278
		}
279
		rcu_bh_qs(cpu);
Linus Torvalds's avatar
Linus Torvalds committed
280
		h++;
281 282
		pending >>= softirq_bit;
	}
Linus Torvalds's avatar
Linus Torvalds committed
283

284
	local_irq_disable();
Linus Torvalds's avatar
Linus Torvalds committed
285 286

	pending = local_softirq_pending();
Eric Dumazet's avatar
Eric Dumazet committed
287
	if (pending) {
288 289
		if (time_before(jiffies, end) && !need_resched() &&
		    --max_restart)
Eric Dumazet's avatar
Eric Dumazet committed
290
			goto restart;
Linus Torvalds's avatar
Linus Torvalds committed
291 292

		wakeup_softirqd();
Eric Dumazet's avatar
Eric Dumazet committed
293
	}
Linus Torvalds's avatar
Linus Torvalds committed
294

295
	lockdep_softirq_end(in_hardirq);
296
	account_irq_exit_time(current);
297
	__local_bh_enable(SOFTIRQ_OFFSET);
298
	WARN_ON_ONCE(in_interrupt());
299
	tsk_restore_flags(current, old_flags, PF_MEMALLOC);
Linus Torvalds's avatar
Linus Torvalds committed
300 301 302 303 304 305 306 307 308 309 310 311 312 313 314
}

asmlinkage void do_softirq(void)
{
	__u32 pending;
	unsigned long flags;

	if (in_interrupt())
		return;

	local_irq_save(flags);

	pending = local_softirq_pending();

	if (pending)
315
		do_softirq_own_stack();
Linus Torvalds's avatar
Linus Torvalds committed
316 317 318 319

	local_irq_restore(flags);
}

320 321 322 323 324
/*
 * Enter an interrupt context.
 */
void irq_enter(void)
{
325
	rcu_irq_enter();
326
	if (is_idle_task(current) && !in_interrupt()) {
327 328 329 330 331
		/*
		 * Prevent raise_softirq from needlessly waking up ksoftirqd
		 * here, as softirq will be serviced on return from interrupt.
		 */
		local_bh_disable();
332
		tick_irq_enter();
333 334 335 336
		_local_bh_enable();
	}

	__irq_enter();
337 338
}

339 340
static inline void invoke_softirq(void)
{
341
	if (!force_irqthreads) {
342
#ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
343 344 345
		/*
		 * We can safely execute softirq on the current stack if
		 * it is the irq stack, because it should be near empty
346 347 348 349 350 351 352 353
		 * at this stage.
		 */
		__do_softirq();
#else
		/*
		 * Otherwise, irq_exit() is called on the task stack that can
		 * be potentially deep already. So call softirq in its own stack
		 * to prevent from any overrun.
354
		 */
355
		do_softirq_own_stack();
356
#endif
357
	} else {
358
		wakeup_softirqd();
359
	}
360
}
Linus Torvalds's avatar
Linus Torvalds committed
361

362 363 364 365 366 367 368 369 370 371 372 373 374
static inline void tick_irq_exit(void)
{
#ifdef CONFIG_NO_HZ_COMMON
	int cpu = smp_processor_id();

	/* Make sure that timer wheel updates are propagated */
	if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
		if (!in_interrupt())
			tick_nohz_irq_exit();
	}
#endif
}

Linus Torvalds's avatar
Linus Torvalds committed
375 376 377 378 379
/*
 * Exit an interrupt context. Process softirqs if needed and possible:
 */
void irq_exit(void)
{
380
#ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
381
	local_irq_disable();
382 383 384 385
#else
	WARN_ON_ONCE(!irqs_disabled());
#endif

386
	account_irq_exit_time(current);
387
	preempt_count_sub(HARDIRQ_OFFSET);
Linus Torvalds's avatar
Linus Torvalds committed
388 389
	if (!in_interrupt() && local_softirq_pending())
		invoke_softirq();
390

391
	tick_irq_exit();
392
	rcu_irq_exit();
393
	trace_hardirq_exit(); /* must be last! */
Linus Torvalds's avatar
Linus Torvalds committed
394 395 396 397 398
}

/*
 * This function must run with irqs disabled!
 */
399
inline void raise_softirq_irqoff(unsigned int nr)
Linus Torvalds's avatar
Linus Torvalds committed
400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
{
	__raise_softirq_irqoff(nr);

	/*
	 * If we're in an interrupt or softirq, we're done
	 * (this also catches softirq-disabled code). We will
	 * actually run the softirq once we return from
	 * the irq or softirq.
	 *
	 * Otherwise we wake up ksoftirqd to make sure we
	 * schedule the softirq soon.
	 */
	if (!in_interrupt())
		wakeup_softirqd();
}

416
void raise_softirq(unsigned int nr)
Linus Torvalds's avatar
Linus Torvalds committed
417 418 419 420 421 422 423 424
{
	unsigned long flags;

	local_irq_save(flags);
	raise_softirq_irqoff(nr);
	local_irq_restore(flags);
}

425 426 427 428 429 430
void __raise_softirq_irqoff(unsigned int nr)
{
	trace_softirq_raise(nr);
	or_softirq_pending(1UL << nr);
}

431
void open_softirq(int nr, void (*action)(struct softirq_action *))
Linus Torvalds's avatar
Linus Torvalds committed
432 433 434 435
{
	softirq_vec[nr].action = action;
}

436 437 438
/*
 * Tasklets
 */
439
struct tasklet_head {
440 441
	struct tasklet_struct *head;
	struct tasklet_struct **tail;
Linus Torvalds's avatar
Linus Torvalds committed
442 443
};

444 445
static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
Linus Torvalds's avatar
Linus Torvalds committed
446

447
void __tasklet_schedule(struct tasklet_struct *t)
Linus Torvalds's avatar
Linus Torvalds committed
448 449 450 451
{
	unsigned long flags;

	local_irq_save(flags);
452
	t->next = NULL;
453 454
	*__this_cpu_read(tasklet_vec.tail) = t;
	__this_cpu_write(tasklet_vec.tail, &(t->next));
Linus Torvalds's avatar
Linus Torvalds committed
455 456 457 458 459
	raise_softirq_irqoff(TASKLET_SOFTIRQ);
	local_irq_restore(flags);
}
EXPORT_SYMBOL(__tasklet_schedule);

460
void __tasklet_hi_schedule(struct tasklet_struct *t)
Linus Torvalds's avatar
Linus Torvalds committed
461 462 463 464
{
	unsigned long flags;

	local_irq_save(flags);
465
	t->next = NULL;
466 467
	*__this_cpu_read(tasklet_hi_vec.tail) = t;
	__this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
Linus Torvalds's avatar
Linus Torvalds committed
468 469 470 471 472
	raise_softirq_irqoff(HI_SOFTIRQ);
	local_irq_restore(flags);
}
EXPORT_SYMBOL(__tasklet_hi_schedule);

473 474 475 476
void __tasklet_hi_schedule_first(struct tasklet_struct *t)
{
	BUG_ON(!irqs_disabled());

477 478
	t->next = __this_cpu_read(tasklet_hi_vec.head);
	__this_cpu_write(tasklet_hi_vec.head, t);
479 480 481 482
	__raise_softirq_irqoff(HI_SOFTIRQ);
}
EXPORT_SYMBOL(__tasklet_hi_schedule_first);

Linus Torvalds's avatar
Linus Torvalds committed
483 484 485 486 487
static void tasklet_action(struct softirq_action *a)
{
	struct tasklet_struct *list;

	local_irq_disable();
488 489 490
	list = __this_cpu_read(tasklet_vec.head);
	__this_cpu_write(tasklet_vec.head, NULL);
	__this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
Linus Torvalds's avatar
Linus Torvalds committed
491 492 493 494 495 496 497 498 499
	local_irq_enable();

	while (list) {
		struct tasklet_struct *t = list;

		list = list->next;

		if (tasklet_trylock(t)) {
			if (!atomic_read(&t->count)) {
500 501
				if (!test_and_clear_bit(TASKLET_STATE_SCHED,
							&t->state))
Linus Torvalds's avatar
Linus Torvalds committed
502 503 504 505 506 507 508 509 510
					BUG();
				t->func(t->data);
				tasklet_unlock(t);
				continue;
			}
			tasklet_unlock(t);
		}

		local_irq_disable();
511
		t->next = NULL;
512 513
		*__this_cpu_read(tasklet_vec.tail) = t;
		__this_cpu_write(tasklet_vec.tail, &(t->next));
Linus Torvalds's avatar
Linus Torvalds committed
514 515 516 517 518 519 520 521 522 523
		__raise_softirq_irqoff(TASKLET_SOFTIRQ);
		local_irq_enable();
	}
}

static void tasklet_hi_action(struct softirq_action *a)
{
	struct tasklet_struct *list;

	local_irq_disable();
524 525 526
	list = __this_cpu_read(tasklet_hi_vec.head);
	__this_cpu_write(tasklet_hi_vec.head, NULL);
	__this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
Linus Torvalds's avatar
Linus Torvalds committed
527 528 529 530 531 532 533 534 535
	local_irq_enable();

	while (list) {
		struct tasklet_struct *t = list;

		list = list->next;

		if (tasklet_trylock(t)) {
			if (!atomic_read(&t->count)) {
536 537
				if (!test_and_clear_bit(TASKLET_STATE_SCHED,
							&t->state))
Linus Torvalds's avatar
Linus Torvalds committed
538 539 540 541 542 543 544 545 546
					BUG();
				t->func(t->data);
				tasklet_unlock(t);
				continue;
			}
			tasklet_unlock(t);
		}

		local_irq_disable();
547
		t->next = NULL;
548 549
		*__this_cpu_read(tasklet_hi_vec.tail) = t;
		__this_cpu_write(tasklet_hi_vec.tail, &(t->next));
Linus Torvalds's avatar
Linus Torvalds committed
550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568
		__raise_softirq_irqoff(HI_SOFTIRQ);
		local_irq_enable();
	}
}

void tasklet_init(struct tasklet_struct *t,
		  void (*func)(unsigned long), unsigned long data)
{
	t->next = NULL;
	t->state = 0;
	atomic_set(&t->count, 0);
	t->func = func;
	t->data = data;
}
EXPORT_SYMBOL(tasklet_init);

void tasklet_kill(struct tasklet_struct *t)
{
	if (in_interrupt())
569
		pr_notice("Attempt to kill tasklet from interrupt\n");
Linus Torvalds's avatar
Linus Torvalds committed
570 571

	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
572
		do {
Linus Torvalds's avatar
Linus Torvalds committed
573
			yield();
574
		} while (test_bit(TASKLET_STATE_SCHED, &t->state));
Linus Torvalds's avatar
Linus Torvalds committed
575 576 577 578 579 580
	}
	tasklet_unlock_wait(t);
	clear_bit(TASKLET_STATE_SCHED, &t->state);
}
EXPORT_SYMBOL(tasklet_kill);

581 582 583 584 585
/*
 * tasklet_hrtimer
 */

/*
586 587 588
 * The trampoline is called when the hrtimer expires. It schedules a tasklet
 * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 * hrtimer callback, but from softirq context.
589 590 591 592 593 594
 */
static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
{
	struct tasklet_hrtimer *ttimer =
		container_of(timer, struct tasklet_hrtimer, timer);

595 596
	tasklet_hi_schedule(&ttimer->tasklet);
	return HRTIMER_NORESTART;
597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615
}

/*
 * Helper function which calls the hrtimer callback from
 * tasklet/softirq context
 */
static void __tasklet_hrtimer_trampoline(unsigned long data)
{
	struct tasklet_hrtimer *ttimer = (void *)data;
	enum hrtimer_restart restart;

	restart = ttimer->function(&ttimer->timer);
	if (restart != HRTIMER_NORESTART)
		hrtimer_restart(&ttimer->timer);
}

/**
 * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 * @ttimer:	 tasklet_hrtimer which is initialized
616
 * @function:	 hrtimer callback function which gets called from softirq context
617 618 619 620 621 622 623 624 625 626 627 628 629 630 631
 * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 * @mode:	 hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 */
void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
			  enum hrtimer_restart (*function)(struct hrtimer *),
			  clockid_t which_clock, enum hrtimer_mode mode)
{
	hrtimer_init(&ttimer->timer, which_clock, mode);
	ttimer->timer.function = __hrtimer_tasklet_trampoline;
	tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
		     (unsigned long)ttimer);
	ttimer->function = function;
}
EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);

Linus Torvalds's avatar
Linus Torvalds committed
632 633
void __init softirq_init(void)
{
634 635 636 637 638 639 640 641 642
	int cpu;

	for_each_possible_cpu(cpu) {
		per_cpu(tasklet_vec, cpu).tail =
			&per_cpu(tasklet_vec, cpu).head;
		per_cpu(tasklet_hi_vec, cpu).tail =
			&per_cpu(tasklet_hi_vec, cpu).head;
	}

643 644
	open_softirq(TASKLET_SOFTIRQ, tasklet_action);
	open_softirq(HI_SOFTIRQ, tasklet_hi_action);
Linus Torvalds's avatar
Linus Torvalds committed
645 646
}

647
static int ksoftirqd_should_run(unsigned int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
648
{
649 650
	return local_softirq_pending();
}
Linus Torvalds's avatar
Linus Torvalds committed
651

652 653 654 655
static void run_ksoftirqd(unsigned int cpu)
{
	local_irq_disable();
	if (local_softirq_pending()) {
656 657 658 659
		/*
		 * We can safely run softirq on inline stack, as we are not deep
		 * in the task stack here.
		 */
660 661 662 663 664
		__do_softirq();
		rcu_note_context_switch(cpu);
		local_irq_enable();
		cond_resched();
		return;
Linus Torvalds's avatar
Linus Torvalds committed
665
	}
666
	local_irq_enable();
Linus Torvalds's avatar
Linus Torvalds committed
667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689
}

#ifdef CONFIG_HOTPLUG_CPU
/*
 * tasklet_kill_immediate is called to remove a tasklet which can already be
 * scheduled for execution on @cpu.
 *
 * Unlike tasklet_kill, this function removes the tasklet
 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 *
 * When this function is called, @cpu must be in the CPU_DEAD state.
 */
void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
{
	struct tasklet_struct **i;

	BUG_ON(cpu_online(cpu));
	BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));

	if (!test_bit(TASKLET_STATE_SCHED, &t->state))
		return;

	/* CPU is dead, so no lock needed. */
690
	for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
Linus Torvalds's avatar
Linus Torvalds committed
691 692
		if (*i == t) {
			*i = t->next;
693 694 695
			/* If this was the tail element, move the tail ptr */
			if (*i == NULL)
				per_cpu(tasklet_vec, cpu).tail = i;
Linus Torvalds's avatar
Linus Torvalds committed
696 697 698 699 700 701 702 703 704 705 706 707
			return;
		}
	}
	BUG();
}

static void takeover_tasklets(unsigned int cpu)
{
	/* CPU is dead, so no lock needed. */
	local_irq_disable();

	/* Find end, append list for that CPU. */
708
	if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
709 710
		*__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
		this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
711 712 713
		per_cpu(tasklet_vec, cpu).head = NULL;
		per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
	}
Linus Torvalds's avatar
Linus Torvalds committed
714 715
	raise_softirq_irqoff(TASKLET_SOFTIRQ);

716
	if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
717 718
		*__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
		__this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
719 720 721
		per_cpu(tasklet_hi_vec, cpu).head = NULL;
		per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
	}
Linus Torvalds's avatar
Linus Torvalds committed
722 723 724 725 726 727
	raise_softirq_irqoff(HI_SOFTIRQ);

	local_irq_enable();
}
#endif /* CONFIG_HOTPLUG_CPU */

728 729
static int cpu_callback(struct notifier_block *nfb, unsigned long action,
			void *hcpu)
Linus Torvalds's avatar
Linus Torvalds committed
730 731 732 733
{
	switch (action) {
#ifdef CONFIG_HOTPLUG_CPU
	case CPU_DEAD:
734 735
	case CPU_DEAD_FROZEN:
		takeover_tasklets((unsigned long)hcpu);
Linus Torvalds's avatar
Linus Torvalds committed
736 737
		break;
#endif /* CONFIG_HOTPLUG_CPU */
738
	}
Linus Torvalds's avatar
Linus Torvalds committed
739 740 741
	return NOTIFY_OK;
}

742
static struct notifier_block cpu_nfb = {
Linus Torvalds's avatar
Linus Torvalds committed
743 744 745
	.notifier_call = cpu_callback
};

746 747 748 749 750 751 752
static struct smp_hotplug_thread softirq_threads = {
	.store			= &ksoftirqd,
	.thread_should_run	= ksoftirqd_should_run,
	.thread_fn		= run_ksoftirqd,
	.thread_comm		= "ksoftirqd/%u",
};

753
static __init int spawn_ksoftirqd(void)
Linus Torvalds's avatar
Linus Torvalds committed
754 755
{
	register_cpu_notifier(&cpu_nfb);
756 757 758

	BUG_ON(smpboot_register_percpu_thread(&softirq_threads));

Linus Torvalds's avatar
Linus Torvalds committed
759 760
	return 0;
}
761
early_initcall(spawn_ksoftirqd);
762

763 764 765 766 767 768 769 770 771 772
/*
 * [ These __weak aliases are kept in a separate compilation unit, so that
 *   GCC does not inline them incorrectly. ]
 */

int __init __weak early_irq_init(void)
{
	return 0;
}

Yinghai Lu's avatar
Yinghai Lu committed
773 774
int __init __weak arch_probe_nr_irqs(void)
{
775
	return NR_IRQS_LEGACY;
Yinghai Lu's avatar
Yinghai Lu committed
776 777
}

778 779 780 781
int __init __weak arch_early_irq_init(void)
{
	return 0;
}