blk-sysfs.c 25.1 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2 3 4 5
/*
 * Functions related to sysfs handling
 */
#include <linux/kernel.h>
6
#include <linux/slab.h>
7 8 9
#include <linux/module.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
10
#include <linux/backing-dev.h>
11
#include <linux/blktrace_api.h>
12
#include <linux/blk-mq.h>
13
#include <linux/blk-cgroup.h>
14 15

#include "blk.h"
16
#include "blk-mq.h"
17
#include "blk-mq-debugfs.h"
18
#include "blk-wbt.h"
19 20 21 22 23 24 25 26

struct queue_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct request_queue *, char *);
	ssize_t (*store)(struct request_queue *, const char *, size_t);
};

static ssize_t
27
queue_var_show(unsigned long var, char *page)
28
{
29
	return sprintf(page, "%lu\n", var);
30 31 32 33 34
}

static ssize_t
queue_var_store(unsigned long *var, const char *page, size_t count)
{
35 36 37
	int err;
	unsigned long v;

38
	err = kstrtoul(page, 10, &v);
39 40 41 42
	if (err || v > UINT_MAX)
		return -EINVAL;

	*var = v;
43 44 45 46

	return count;
}

47
static ssize_t queue_var_store64(s64 *var, const char *page)
48 49
{
	int err;
50
	s64 v;
51

52
	err = kstrtos64(page, 10, &v);
53 54 55 56 57 58 59
	if (err < 0)
		return err;

	*var = v;
	return 0;
}

60 61 62 63 64 65 66 67 68
static ssize_t queue_requests_show(struct request_queue *q, char *page)
{
	return queue_var_show(q->nr_requests, (page));
}

static ssize_t
queue_requests_store(struct request_queue *q, const char *page, size_t count)
{
	unsigned long nr;
69
	int ret, err;
70

71
	if (!q->request_fn && !q->mq_ops)
72 73 74
		return -EINVAL;

	ret = queue_var_store(&nr, page, count);
75 76 77
	if (ret < 0)
		return ret;

78 79 80
	if (nr < BLKDEV_MIN_RQ)
		nr = BLKDEV_MIN_RQ;

81 82 83 84 85 86 87 88
	if (q->request_fn)
		err = blk_update_nr_requests(q, nr);
	else
		err = blk_mq_update_nr_requests(q, nr);

	if (err)
		return err;

89 90 91 92 93
	return ret;
}

static ssize_t queue_ra_show(struct request_queue *q, char *page)
{
94
	unsigned long ra_kb = q->backing_dev_info->ra_pages <<
95
					(PAGE_SHIFT - 10);
96 97 98 99 100 101 102 103 104 105

	return queue_var_show(ra_kb, (page));
}

static ssize_t
queue_ra_store(struct request_queue *q, const char *page, size_t count)
{
	unsigned long ra_kb;
	ssize_t ret = queue_var_store(&ra_kb, page, count);

106 107 108
	if (ret < 0)
		return ret;

109
	q->backing_dev_info->ra_pages = ra_kb >> (PAGE_SHIFT - 10);
110 111 112 113 114 115

	return ret;
}

static ssize_t queue_max_sectors_show(struct request_queue *q, char *page)
{
116
	int max_sectors_kb = queue_max_sectors(q) >> 1;
117 118 119 120

	return queue_var_show(max_sectors_kb, (page));
}

121 122 123 124 125
static ssize_t queue_max_segments_show(struct request_queue *q, char *page)
{
	return queue_var_show(queue_max_segments(q), (page));
}

126 127 128 129 130 131
static ssize_t queue_max_discard_segments_show(struct request_queue *q,
		char *page)
{
	return queue_var_show(queue_max_discard_segments(q), (page));
}

132 133 134 135 136
static ssize_t queue_max_integrity_segments_show(struct request_queue *q, char *page)
{
	return queue_var_show(q->limits.max_integrity_segments, (page));
}

137 138
static ssize_t queue_max_segment_size_show(struct request_queue *q, char *page)
{
139
	if (blk_queue_cluster(q))
140 141
		return queue_var_show(queue_max_segment_size(q), (page));

142
	return queue_var_show(PAGE_SIZE, (page));
143 144
}

145
static ssize_t queue_logical_block_size_show(struct request_queue *q, char *page)
146
{
147
	return queue_var_show(queue_logical_block_size(q), page);
148 149
}

150 151 152 153 154
static ssize_t queue_physical_block_size_show(struct request_queue *q, char *page)
{
	return queue_var_show(queue_physical_block_size(q), page);
}

155 156 157 158 159
static ssize_t queue_chunk_sectors_show(struct request_queue *q, char *page)
{
	return queue_var_show(q->limits.chunk_sectors, page);
}

160 161 162 163 164 165 166 167
static ssize_t queue_io_min_show(struct request_queue *q, char *page)
{
	return queue_var_show(queue_io_min(q), page);
}

static ssize_t queue_io_opt_show(struct request_queue *q, char *page)
{
	return queue_var_show(queue_io_opt(q), page);
168 169
}

170 171 172 173 174
static ssize_t queue_discard_granularity_show(struct request_queue *q, char *page)
{
	return queue_var_show(q->limits.discard_granularity, page);
}

175 176 177
static ssize_t queue_discard_max_hw_show(struct request_queue *q, char *page)
{

178 179
	return sprintf(page, "%llu\n",
		(unsigned long long)q->limits.max_hw_discard_sectors << 9);
180 181
}

182 183
static ssize_t queue_discard_max_show(struct request_queue *q, char *page)
{
184 185
	return sprintf(page, "%llu\n",
		       (unsigned long long)q->limits.max_discard_sectors << 9);
186 187
}

188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
static ssize_t queue_discard_max_store(struct request_queue *q,
				       const char *page, size_t count)
{
	unsigned long max_discard;
	ssize_t ret = queue_var_store(&max_discard, page, count);

	if (ret < 0)
		return ret;

	if (max_discard & (q->limits.discard_granularity - 1))
		return -EINVAL;

	max_discard >>= 9;
	if (max_discard > UINT_MAX)
		return -EINVAL;

	if (max_discard > q->limits.max_hw_discard_sectors)
		max_discard = q->limits.max_hw_discard_sectors;

	q->limits.max_discard_sectors = max_discard;
	return ret;
}

211 212
static ssize_t queue_discard_zeroes_data_show(struct request_queue *q, char *page)
{
213
	return queue_var_show(0, page);
214 215
}

216 217 218 219 220 221
static ssize_t queue_write_same_max_show(struct request_queue *q, char *page)
{
	return sprintf(page, "%llu\n",
		(unsigned long long)q->limits.max_write_same_sectors << 9);
}

222 223 224 225 226
static ssize_t queue_write_zeroes_max_show(struct request_queue *q, char *page)
{
	return sprintf(page, "%llu\n",
		(unsigned long long)q->limits.max_write_zeroes_sectors << 9);
}
227

228 229 230 231
static ssize_t
queue_max_sectors_store(struct request_queue *q, const char *page, size_t count)
{
	unsigned long max_sectors_kb,
232
		max_hw_sectors_kb = queue_max_hw_sectors(q) >> 1,
233
			page_kb = 1 << (PAGE_SHIFT - 10);
234 235
	ssize_t ret = queue_var_store(&max_sectors_kb, page, count);

236 237 238
	if (ret < 0)
		return ret;

239 240 241
	max_hw_sectors_kb = min_not_zero(max_hw_sectors_kb, (unsigned long)
					 q->limits.max_dev_sectors >> 1);

242 243
	if (max_sectors_kb > max_hw_sectors_kb || max_sectors_kb < page_kb)
		return -EINVAL;
244

245
	spin_lock_irq(q->queue_lock);
246
	q->limits.max_sectors = max_sectors_kb << 1;
247
	q->backing_dev_info->io_pages = max_sectors_kb >> (PAGE_SHIFT - 10);
248 249 250 251 252 253 254
	spin_unlock_irq(q->queue_lock);

	return ret;
}

static ssize_t queue_max_hw_sectors_show(struct request_queue *q, char *page)
{
255
	int max_hw_sectors_kb = queue_max_hw_sectors(q) >> 1;
256 257 258 259

	return queue_var_show(max_hw_sectors_kb, (page));
}

260 261 262 263 264 265 266 267 268 269 270 271 272 273
#define QUEUE_SYSFS_BIT_FNS(name, flag, neg)				\
static ssize_t								\
queue_show_##name(struct request_queue *q, char *page)			\
{									\
	int bit;							\
	bit = test_bit(QUEUE_FLAG_##flag, &q->queue_flags);		\
	return queue_var_show(neg ? !bit : bit, page);			\
}									\
static ssize_t								\
queue_store_##name(struct request_queue *q, const char *page, size_t count) \
{									\
	unsigned long val;						\
	ssize_t ret;							\
	ret = queue_var_store(&val, page, count);			\
274 275
	if (ret < 0)							\
		 return ret;						\
276 277 278 279
	if (neg)							\
		val = !val;						\
									\
	if (val)							\
280
		blk_queue_flag_set(QUEUE_FLAG_##flag, q);		\
281
	else								\
282
		blk_queue_flag_clear(QUEUE_FLAG_##flag, q);		\
283
	return ret;							\
284 285
}

286 287 288 289
QUEUE_SYSFS_BIT_FNS(nonrot, NONROT, 1);
QUEUE_SYSFS_BIT_FNS(random, ADD_RANDOM, 0);
QUEUE_SYSFS_BIT_FNS(iostats, IO_STAT, 0);
#undef QUEUE_SYSFS_BIT_FNS
290

291 292 293 294 295 296 297 298 299 300 301 302
static ssize_t queue_zoned_show(struct request_queue *q, char *page)
{
	switch (blk_queue_zoned_model(q)) {
	case BLK_ZONED_HA:
		return sprintf(page, "host-aware\n");
	case BLK_ZONED_HM:
		return sprintf(page, "host-managed\n");
	default:
		return sprintf(page, "none\n");
	}
}

303 304 305 306 307
static ssize_t queue_nr_zones_show(struct request_queue *q, char *page)
{
	return queue_var_show(blk_queue_nr_zones(q), page);
}

308 309
static ssize_t queue_nomerges_show(struct request_queue *q, char *page)
{
310 311
	return queue_var_show((blk_queue_nomerges(q) << 1) |
			       blk_queue_noxmerges(q), page);
312 313 314 315 316 317 318 319
}

static ssize_t queue_nomerges_store(struct request_queue *q, const char *page,
				    size_t count)
{
	unsigned long nm;
	ssize_t ret = queue_var_store(&nm, page, count);

320 321 322
	if (ret < 0)
		return ret;

323
	spin_lock_irq(q->queue_lock);
324 325 326
	queue_flag_clear(QUEUE_FLAG_NOMERGES, q);
	queue_flag_clear(QUEUE_FLAG_NOXMERGES, q);
	if (nm == 2)
327
		queue_flag_set(QUEUE_FLAG_NOMERGES, q);
328 329
	else if (nm)
		queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
330
	spin_unlock_irq(q->queue_lock);
331

332 333 334
	return ret;
}

335 336
static ssize_t queue_rq_affinity_show(struct request_queue *q, char *page)
{
337
	bool set = test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags);
Dan Williams's avatar
Dan Williams committed
338
	bool force = test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags);
339

Dan Williams's avatar
Dan Williams committed
340
	return queue_var_show(set << force, page);
341 342 343 344 345 346
}

static ssize_t
queue_rq_affinity_store(struct request_queue *q, const char *page, size_t count)
{
	ssize_t ret = -EINVAL;
347
#ifdef CONFIG_SMP
348 349 350
	unsigned long val;

	ret = queue_var_store(&val, page, count);
351 352 353
	if (ret < 0)
		return ret;

354
	spin_lock_irq(q->queue_lock);
355
	if (val == 2) {
356
		queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
357 358 359 360 361
		queue_flag_set(QUEUE_FLAG_SAME_FORCE, q);
	} else if (val == 1) {
		queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
		queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
	} else if (val == 0) {
Dan Williams's avatar
Dan Williams committed
362 363 364
		queue_flag_clear(QUEUE_FLAG_SAME_COMP, q);
		queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
	}
365 366 367 368
	spin_unlock_irq(q->queue_lock);
#endif
	return ret;
}
369

370 371
static ssize_t queue_poll_delay_show(struct request_queue *q, char *page)
{
372 373 374 375 376 377 378 379
	int val;

	if (q->poll_nsec == -1)
		val = -1;
	else
		val = q->poll_nsec / 1000;

	return sprintf(page, "%d\n", val);
380 381 382 383 384
}

static ssize_t queue_poll_delay_store(struct request_queue *q, const char *page,
				size_t count)
{
385
	int err, val;
386 387 388 389

	if (!q->mq_ops || !q->mq_ops->poll)
		return -EINVAL;

390 391 392
	err = kstrtoint(page, 10, &val);
	if (err < 0)
		return err;
393

394 395 396 397 398 399
	if (val == -1)
		q->poll_nsec = -1;
	else
		q->poll_nsec = val * 1000;

	return count;
400 401
}

402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420
static ssize_t queue_poll_show(struct request_queue *q, char *page)
{
	return queue_var_show(test_bit(QUEUE_FLAG_POLL, &q->queue_flags), page);
}

static ssize_t queue_poll_store(struct request_queue *q, const char *page,
				size_t count)
{
	unsigned long poll_on;
	ssize_t ret;

	if (!q->mq_ops || !q->mq_ops->poll)
		return -EINVAL;

	ret = queue_var_store(&poll_on, page, count);
	if (ret < 0)
		return ret;

	if (poll_on)
421
		blk_queue_flag_set(QUEUE_FLAG_POLL, q);
422
	else
423
		blk_queue_flag_clear(QUEUE_FLAG_POLL, q);
424 425 426 427

	return ret;
}

428 429
static ssize_t queue_wb_lat_show(struct request_queue *q, char *page)
{
430
	if (!wbt_rq_qos(q))
431 432
		return -EINVAL;

433
	return sprintf(page, "%llu\n", div_u64(wbt_get_min_lat(q), 1000));
434 435 436 437 438
}

static ssize_t queue_wb_lat_store(struct request_queue *q, const char *page,
				  size_t count)
{
439
	struct rq_qos *rqos;
440
	ssize_t ret;
441
	s64 val;
442 443 444 445

	ret = queue_var_store64(&val, page);
	if (ret < 0)
		return ret;
446 447 448
	if (val < -1)
		return -EINVAL;

449 450
	rqos = wbt_rq_qos(q);
	if (!rqos) {
451 452 453 454
		ret = wbt_init(q);
		if (ret)
			return ret;
	}
455

456
	if (val == -1)
457
		val = wbt_default_latency_nsec(q);
458
	else if (val >= 0)
459
		val *= 1000ULL;
460

461 462 463 464 465 466 467 468 469 470
	/*
	 * Ensure that the queue is idled, in case the latency update
	 * ends up either enabling or disabling wbt completely. We can't
	 * have IO inflight if that happens.
	 */
	if (q->mq_ops) {
		blk_mq_freeze_queue(q);
		blk_mq_quiesce_queue(q);
	} else
		blk_queue_bypass_start(q);
471

472
	wbt_set_min_lat(q, val);
473
	wbt_update_limits(q);
474 475 476 477 478 479 480

	if (q->mq_ops) {
		blk_mq_unquiesce_queue(q);
		blk_mq_unfreeze_queue(q);
	} else
		blk_queue_bypass_end(q);

481 482 483
	return count;
}

484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506
static ssize_t queue_wc_show(struct request_queue *q, char *page)
{
	if (test_bit(QUEUE_FLAG_WC, &q->queue_flags))
		return sprintf(page, "write back\n");

	return sprintf(page, "write through\n");
}

static ssize_t queue_wc_store(struct request_queue *q, const char *page,
			      size_t count)
{
	int set = -1;

	if (!strncmp(page, "write back", 10))
		set = 1;
	else if (!strncmp(page, "write through", 13) ||
		 !strncmp(page, "none", 4))
		set = 0;

	if (set == -1)
		return -EINVAL;

	if (set)
507
		blk_queue_flag_set(QUEUE_FLAG_WC, q);
508
	else
509
		blk_queue_flag_clear(QUEUE_FLAG_WC, q);
510 511 512 513

	return count;
}

514 515 516 517 518
static ssize_t queue_fua_show(struct request_queue *q, char *page)
{
	return sprintf(page, "%u\n", test_bit(QUEUE_FLAG_FUA, &q->queue_flags));
}

519 520 521 522 523
static ssize_t queue_dax_show(struct request_queue *q, char *page)
{
	return queue_var_show(blk_queue_dax(q), page);
}

524
static struct queue_sysfs_entry queue_requests_entry = {
525
	.attr = {.name = "nr_requests", .mode = 0644 },
526 527 528 529 530
	.show = queue_requests_show,
	.store = queue_requests_store,
};

static struct queue_sysfs_entry queue_ra_entry = {
531
	.attr = {.name = "read_ahead_kb", .mode = 0644 },
532 533 534 535 536
	.show = queue_ra_show,
	.store = queue_ra_store,
};

static struct queue_sysfs_entry queue_max_sectors_entry = {
537
	.attr = {.name = "max_sectors_kb", .mode = 0644 },
538 539 540 541 542
	.show = queue_max_sectors_show,
	.store = queue_max_sectors_store,
};

static struct queue_sysfs_entry queue_max_hw_sectors_entry = {
543
	.attr = {.name = "max_hw_sectors_kb", .mode = 0444 },
544 545 546
	.show = queue_max_hw_sectors_show,
};

547
static struct queue_sysfs_entry queue_max_segments_entry = {
548
	.attr = {.name = "max_segments", .mode = 0444 },
549 550 551
	.show = queue_max_segments_show,
};

552
static struct queue_sysfs_entry queue_max_discard_segments_entry = {
553
	.attr = {.name = "max_discard_segments", .mode = 0444 },
554 555 556
	.show = queue_max_discard_segments_show,
};

557
static struct queue_sysfs_entry queue_max_integrity_segments_entry = {
558
	.attr = {.name = "max_integrity_segments", .mode = 0444 },
559 560 561
	.show = queue_max_integrity_segments_show,
};

562
static struct queue_sysfs_entry queue_max_segment_size_entry = {
563
	.attr = {.name = "max_segment_size", .mode = 0444 },
564 565 566
	.show = queue_max_segment_size_show,
};

567
static struct queue_sysfs_entry queue_iosched_entry = {
568
	.attr = {.name = "scheduler", .mode = 0644 },
569 570 571 572
	.show = elv_iosched_show,
	.store = elv_iosched_store,
};

573
static struct queue_sysfs_entry queue_hw_sector_size_entry = {
574
	.attr = {.name = "hw_sector_size", .mode = 0444 },
575 576 577 578
	.show = queue_logical_block_size_show,
};

static struct queue_sysfs_entry queue_logical_block_size_entry = {
579
	.attr = {.name = "logical_block_size", .mode = 0444 },
580
	.show = queue_logical_block_size_show,
581 582
};

583
static struct queue_sysfs_entry queue_physical_block_size_entry = {
584
	.attr = {.name = "physical_block_size", .mode = 0444 },
585 586 587
	.show = queue_physical_block_size_show,
};

588
static struct queue_sysfs_entry queue_chunk_sectors_entry = {
589
	.attr = {.name = "chunk_sectors", .mode = 0444 },
590 591 592
	.show = queue_chunk_sectors_show,
};

593
static struct queue_sysfs_entry queue_io_min_entry = {
594
	.attr = {.name = "minimum_io_size", .mode = 0444 },
595 596 597 598
	.show = queue_io_min_show,
};

static struct queue_sysfs_entry queue_io_opt_entry = {
599
	.attr = {.name = "optimal_io_size", .mode = 0444 },
600
	.show = queue_io_opt_show,
601 602
};

603
static struct queue_sysfs_entry queue_discard_granularity_entry = {
604
	.attr = {.name = "discard_granularity", .mode = 0444 },
605 606 607
	.show = queue_discard_granularity_show,
};

608
static struct queue_sysfs_entry queue_discard_max_hw_entry = {
609
	.attr = {.name = "discard_max_hw_bytes", .mode = 0444 },
610 611 612
	.show = queue_discard_max_hw_show,
};

613
static struct queue_sysfs_entry queue_discard_max_entry = {
614
	.attr = {.name = "discard_max_bytes", .mode = 0644 },
615
	.show = queue_discard_max_show,
616
	.store = queue_discard_max_store,
617 618
};

619
static struct queue_sysfs_entry queue_discard_zeroes_data_entry = {
620
	.attr = {.name = "discard_zeroes_data", .mode = 0444 },
621 622 623
	.show = queue_discard_zeroes_data_show,
};

624
static struct queue_sysfs_entry queue_write_same_max_entry = {
625
	.attr = {.name = "write_same_max_bytes", .mode = 0444 },
626 627 628
	.show = queue_write_same_max_show,
};

629
static struct queue_sysfs_entry queue_write_zeroes_max_entry = {
630
	.attr = {.name = "write_zeroes_max_bytes", .mode = 0444 },
631 632 633
	.show = queue_write_zeroes_max_show,
};

634
static struct queue_sysfs_entry queue_nonrot_entry = {
635
	.attr = {.name = "rotational", .mode = 0644 },
636 637
	.show = queue_show_nonrot,
	.store = queue_store_nonrot,
638 639
};

640
static struct queue_sysfs_entry queue_zoned_entry = {
641
	.attr = {.name = "zoned", .mode = 0444 },
642 643 644
	.show = queue_zoned_show,
};

645 646 647 648 649
static struct queue_sysfs_entry queue_nr_zones_entry = {
	.attr = {.name = "nr_zones", .mode = 0444 },
	.show = queue_nr_zones_show,
};

650
static struct queue_sysfs_entry queue_nomerges_entry = {
651
	.attr = {.name = "nomerges", .mode = 0644 },
652 653 654 655
	.show = queue_nomerges_show,
	.store = queue_nomerges_store,
};

656
static struct queue_sysfs_entry queue_rq_affinity_entry = {
657
	.attr = {.name = "rq_affinity", .mode = 0644 },
658 659 660 661
	.show = queue_rq_affinity_show,
	.store = queue_rq_affinity_store,
};

662
static struct queue_sysfs_entry queue_iostats_entry = {
663
	.attr = {.name = "iostats", .mode = 0644 },
664 665
	.show = queue_show_iostats,
	.store = queue_store_iostats,
666 667
};

668
static struct queue_sysfs_entry queue_random_entry = {
669
	.attr = {.name = "add_random", .mode = 0644 },
670 671
	.show = queue_show_random,
	.store = queue_store_random,
672 673
};

674
static struct queue_sysfs_entry queue_poll_entry = {
675
	.attr = {.name = "io_poll", .mode = 0644 },
676 677 678 679
	.show = queue_poll_show,
	.store = queue_poll_store,
};

680
static struct queue_sysfs_entry queue_poll_delay_entry = {
681
	.attr = {.name = "io_poll_delay", .mode = 0644 },
682 683 684 685
	.show = queue_poll_delay_show,
	.store = queue_poll_delay_store,
};

686
static struct queue_sysfs_entry queue_wc_entry = {
687
	.attr = {.name = "write_cache", .mode = 0644 },
688 689 690 691
	.show = queue_wc_show,
	.store = queue_wc_store,
};

692
static struct queue_sysfs_entry queue_fua_entry = {
693
	.attr = {.name = "fua", .mode = 0444 },
694 695 696
	.show = queue_fua_show,
};

697
static struct queue_sysfs_entry queue_dax_entry = {
698
	.attr = {.name = "dax", .mode = 0444 },
699 700 701
	.show = queue_dax_show,
};

702
static struct queue_sysfs_entry queue_wb_lat_entry = {
703
	.attr = {.name = "wbt_lat_usec", .mode = 0644 },
704 705 706 707
	.show = queue_wb_lat_show,
	.store = queue_wb_lat_store,
};

708 709
#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
static struct queue_sysfs_entry throtl_sample_time_entry = {
710
	.attr = {.name = "throttle_sample_time", .mode = 0644 },
711 712 713 714 715
	.show = blk_throtl_sample_time_show,
	.store = blk_throtl_sample_time_store,
};
#endif

716 717 718 719 720
static struct attribute *default_attrs[] = {
	&queue_requests_entry.attr,
	&queue_ra_entry.attr,
	&queue_max_hw_sectors_entry.attr,
	&queue_max_sectors_entry.attr,
721
	&queue_max_segments_entry.attr,
722
	&queue_max_discard_segments_entry.attr,
723
	&queue_max_integrity_segments_entry.attr,
724
	&queue_max_segment_size_entry.attr,
725
	&queue_iosched_entry.attr,
726
	&queue_hw_sector_size_entry.attr,
727
	&queue_logical_block_size_entry.attr,
728
	&queue_physical_block_size_entry.attr,
729
	&queue_chunk_sectors_entry.attr,
730 731
	&queue_io_min_entry.attr,
	&queue_io_opt_entry.attr,
732 733
	&queue_discard_granularity_entry.attr,
	&queue_discard_max_entry.attr,
734
	&queue_discard_max_hw_entry.attr,
735
	&queue_discard_zeroes_data_entry.attr,
736
	&queue_write_same_max_entry.attr,
737
	&queue_write_zeroes_max_entry.attr,
738
	&queue_nonrot_entry.attr,
739
	&queue_zoned_entry.attr,
740
	&queue_nr_zones_entry.attr,
741
	&queue_nomerges_entry.attr,
742
	&queue_rq_affinity_entry.attr,
743
	&queue_iostats_entry.attr,
744
	&queue_random_entry.attr,
745
	&queue_poll_entry.attr,
746
	&queue_wc_entry.attr,
747
	&queue_fua_entry.attr,
748
	&queue_dax_entry.attr,
749
	&queue_wb_lat_entry.attr,
750
	&queue_poll_delay_entry.attr,
751 752 753
#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
	&throtl_sample_time_entry.attr,
#endif
754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769
	NULL,
};

#define to_queue(atr) container_of((atr), struct queue_sysfs_entry, attr)

static ssize_t
queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
{
	struct queue_sysfs_entry *entry = to_queue(attr);
	struct request_queue *q =
		container_of(kobj, struct request_queue, kobj);
	ssize_t res;

	if (!entry->show)
		return -EIO;
	mutex_lock(&q->sysfs_lock);
770
	if (blk_queue_dying(q)) {
771 772 773 774 775 776 777 778 779 780 781 782 783
		mutex_unlock(&q->sysfs_lock);
		return -ENOENT;
	}
	res = entry->show(q, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

static ssize_t
queue_attr_store(struct kobject *kobj, struct attribute *attr,
		    const char *page, size_t length)
{
	struct queue_sysfs_entry *entry = to_queue(attr);
784
	struct request_queue *q;
785 786 787 788
	ssize_t res;

	if (!entry->store)
		return -EIO;
789 790

	q = container_of(kobj, struct request_queue, kobj);
791
	mutex_lock(&q->sysfs_lock);
792
	if (blk_queue_dying(q)) {
793 794 795 796 797 798 799 800
		mutex_unlock(&q->sysfs_lock);
		return -ENOENT;
	}
	res = entry->store(q, page, length);
	mutex_unlock(&q->sysfs_lock);
	return res;
}

Tejun Heo's avatar
Tejun Heo committed
801 802 803 804 805 806 807
static void blk_free_queue_rcu(struct rcu_head *rcu_head)
{
	struct request_queue *q = container_of(rcu_head, struct request_queue,
					       rcu_head);
	kmem_cache_free(blk_requestq_cachep, q);
}

808
/**
809 810
 * __blk_release_queue - release a request queue when it is no longer needed
 * @work: pointer to the release_work member of the request queue to be released
811 812
 *
 * Description:
813 814 815 816
 *     blk_release_queue is the counterpart of blk_init_queue(). It should be
 *     called when a request queue is being released; typically when a block
 *     device is being de-registered. Its primary task it to free the queue
 *     itself.
817
 *
818
 * Notes:
819 820
 *     The low level driver must have finished any outstanding requests first
 *     via blk_cleanup_queue().
821 822 823 824 825
 *
 *     Although blk_release_queue() may be called with preemption disabled,
 *     __blk_release_queue() may sleep.
 */
static void __blk_release_queue(struct work_struct *work)
826
{
827
	struct request_queue *q = container_of(work, typeof(*q), release_work);
828

829 830 831
	if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags))
		blk_stat_remove_callback(q, q->poll_cb);
	blk_stat_free_callback(q->poll_cb);
832

833 834 835 836 837 838 839 840 841 842 843
	if (!blk_queue_dead(q)) {
		/*
		 * Last reference was dropped without having called
		 * blk_cleanup_queue().
		 */
		WARN_ONCE(blk_queue_init_done(q),
			  "request queue %p has been registered but blk_cleanup_queue() has not been called for that queue\n",
			  q);
		blk_exit_queue(q);
	}

844
	WARN(blk_queue_root_blkg(q),
845 846 847
	     "request queue %p is being released but it has not yet been removed from the blkcg controller\n",
	     q);

848 849
	blk_free_queue_stats(q->stats);

850
	blk_exit_rl(q, &q->root_rl);
851 852 853 854

	if (q->queue_tags)
		__blk_queue_free_tags(q);

855 856
	blk_queue_free_zone_bitmaps(q);

857 858 859
	if (!q->mq_ops) {
		if (q->exit_rq_fn)
			q->exit_rq_fn(q, q->fq->flush_rq);
860
		blk_free_flush_queue(q->fq);
861
	} else {
862
		blk_mq_release(q);
863
	}
864

865 866
	blk_trace_shutdown(q);

867 868 869
	if (q->mq_ops)
		blk_mq_debugfs_unregister(q);

870
	bioset_exit(&q->bio_split);
871

872
	ida_simple_remove(&blk_queue_ida, q->id);
Tejun Heo's avatar
Tejun Heo committed
873
	call_rcu(&q->rcu_head, blk_free_queue_rcu);
874 875
}

876 877 878 879 880 881 882 883 884
static void blk_release_queue(struct kobject *kobj)
{
	struct request_queue *q =
		container_of(kobj, struct request_queue, kobj);

	INIT_WORK(&q->release_work, __blk_release_queue);
	schedule_work(&q->release_work);
}

885
static const struct sysfs_ops queue_sysfs_ops = {
886 887 888 889 890 891 892 893 894 895
	.show	= queue_attr_show,
	.store	= queue_attr_store,
};

struct kobj_type blk_queue_ktype = {
	.sysfs_ops	= &queue_sysfs_ops,
	.default_attrs	= default_attrs,
	.release	= blk_release_queue,
};

896 897 898 899
/**
 * blk_register_queue - register a block layer queue with sysfs
 * @disk: Disk of which the request queue should be registered with sysfs.
 */
900 901 902
int blk_register_queue(struct gendisk *disk)
{
	int ret;
903
	struct device *dev = disk_to_dev(disk);
904 905
	struct request_queue *q = disk->queue;

906
	if (WARN_ON(!q))
907 908
		return -ENXIO;

909 910 911 912 913
	WARN_ONCE(test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags),
		  "%s is registering an already registered queue\n",
		  kobject_name(&dev->kobj));
	queue_flag_set_unlocked(QUEUE_FLAG_REGISTERED, q);

914
	/*
915 916 917 918 919 920 921
	 * SCSI probing may synchronously create and destroy a lot of
	 * request_queues for non-existent devices.  Shutting down a fully
	 * functional queue takes measureable wallclock time as RCU grace
	 * periods are involved.  To avoid excessive latency in these
	 * cases, a request_queue starts out in a degraded mode which is
	 * faster to shut down and is made fully functional here as
	 * request_queues for non-existent devices never get registered.
922
	 */
923 924
	if (!blk_queue_init_done(q)) {
		queue_flag_set_unlocked(QUEUE_FLAG_INIT_DONE, q);
925
		percpu_ref_switch_to_percpu(&q->q_usage_counter);
926 927
		blk_queue_bypass_end(q);
	}
928

929 930 931 932
	ret = blk_trace_init_sysfs(dev);
	if (ret)
		return ret;

933 934 935
	/* Prevent changes through sysfs until registration is completed. */
	mutex_lock(&q->sysfs_lock);

936
	ret = kobject_add(&q->kobj, kobject_get(&dev->kobj), "%s", "queue");
937 938
	if (ret < 0) {
		blk_trace_remove_sysfs(dev);
939
		goto unlock;
940
	}
941

942
	if (q->mq_ops) {
943
		__blk_mq_register_dev(dev, q);
944 945
		blk_mq_debugfs_register(q);
	}
946

947 948
	kobject_uevent(&q->kobj, KOBJ_ADD);

949
	wbt_enable_default(q);
950

951 952
	blk_throtl_register_queue(q);

953 954 955
	if (q->request_fn || (q->mq_ops && q->elevator)) {
		ret = elv_register_queue(q);
		if (ret) {
956
			mutex_unlock(&q->sysfs_lock);
957 958 959 960
			kobject_uevent(&q->kobj, KOBJ_REMOVE);
			kobject_del(&q->kobj);
			blk_trace_remove_sysfs(dev);
			kobject_put(&dev->kobj);
961
			return ret;
962
		}
963
	}
964 965 966 967
	ret = 0;
unlock:
	mutex_unlock(&q->sysfs_lock);
	return ret;
968
}
969
EXPORT_SYMBOL_GPL(blk_register_queue);
970

971 972 973 974 975 976 977
/**
 * blk_unregister_queue - counterpart of blk_register_queue()
 * @disk: Disk of which the request queue should be unregistered from sysfs.
 *
 * Note: the caller is responsible for guaranteeing that this function is called
 * after blk_register_queue() has finished.
 */
978 979 980 981
void blk_unregister_queue(struct gendisk *disk)
{
	struct request_queue *q = disk->queue;

982 983 984
	if (WARN_ON(!q))
		return;

985 986 987 988
	/* Return early if disk->queue was never registered. */
	if (!test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags))
		return;

989
	/*
990 991 992
	 * Since sysfs_remove_dir() prevents adding new directory entries
	 * before removal of existing entries starts, protect against
	 * concurrent elv_iosched_store() calls.
993
	 */
994
	mutex_lock(&q->sysfs_lock);
995

996
	blk_queue_flag_clear(QUEUE_FLAG_REGISTERED, q);
Omar Sandoval's avatar
Omar Sandoval committed
997

998 999 1000 1001
	/*
	 * Remove the sysfs attributes before unregistering the queue data
	 * structures that can be modified through sysfs.
	 */
1002
	if (q->mq_ops)
1003
		blk_mq_unregister_dev(disk_to_dev(disk), q);
1004
	mutex_unlock(&q->sysfs_lock);
1005

1006 1007 1008
	kobject_uevent(&q->kobj, KOBJ_REMOVE);
	kobject_del(&q->kobj);
	blk_trace_remove_sysfs(disk_to_dev(disk));
1009

1010 1011 1012
	mutex_lock(&q->sysfs_lock);
	if (q->request_fn || (q->mq_ops && q->elevator))
		elv_unregister_queue(q);
1013
	mutex_unlock(&q->sysfs_lock);
1014 1015

	kobject_put(&disk_to_dev(disk)->kobj);
1016
}