block_dev.c 45.1 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11 12 13
/*
 *  linux/fs/block_dev.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *  Copyright (C) 2001  Andrea Arcangeli <[email protected]> SuSE
 */

#include <linux/init.h>
#include <linux/mm.h>
#include <linux/fcntl.h>
#include <linux/slab.h>
#include <linux/kmod.h>
#include <linux/major.h>
14
#include <linux/device_cgroup.h>
Linus Torvalds's avatar
Linus Torvalds committed
15 16 17 18
#include <linux/highmem.h>
#include <linux/blkdev.h>
#include <linux/module.h>
#include <linux/blkpg.h>
19
#include <linux/magic.h>
Linus Torvalds's avatar
Linus Torvalds committed
20
#include <linux/buffer_head.h>
Al Viro's avatar
Al Viro committed
21
#include <linux/swap.h>
22
#include <linux/pagevec.h>
23
#include <linux/writeback.h>
Linus Torvalds's avatar
Linus Torvalds committed
24 25 26 27
#include <linux/mpage.h>
#include <linux/mount.h>
#include <linux/uio.h>
#include <linux/namei.h>
28
#include <linux/log2.h>
Al Viro's avatar
Al Viro committed
29
#include <linux/cleancache.h>
Linus Torvalds's avatar
Linus Torvalds committed
30
#include <asm/uaccess.h>
31
#include "internal.h"
Linus Torvalds's avatar
Linus Torvalds committed
32 33 34 35 36 37

struct bdev_inode {
	struct block_device bdev;
	struct inode vfs_inode;
};

38 39
static const struct address_space_operations def_blk_aops;

Linus Torvalds's avatar
Linus Torvalds committed
40 41 42 43 44 45 46 47 48 49 50
static inline struct bdev_inode *BDEV_I(struct inode *inode)
{
	return container_of(inode, struct bdev_inode, vfs_inode);
}

inline struct block_device *I_BDEV(struct inode *inode)
{
	return &BDEV_I(inode)->bdev;
}
EXPORT_SYMBOL(I_BDEV);

51 52 53 54 55 56 57 58 59 60 61
static void bdev_write_inode(struct inode *inode)
{
	spin_lock(&inode->i_lock);
	while (inode->i_state & I_DIRTY) {
		spin_unlock(&inode->i_lock);
		WARN_ON_ONCE(write_inode_now(inode, true));
		spin_lock(&inode->i_lock);
	}
	spin_unlock(&inode->i_lock);
}

Peter Zijlstra's avatar
Peter Zijlstra committed
62
/* Kill _all_ buffers and pagecache , dirty or not.. */
Al Viro's avatar
Al Viro committed
63
void kill_bdev(struct block_device *bdev)
Linus Torvalds's avatar
Linus Torvalds committed
64
{
Al Viro's avatar
Al Viro committed
65 66
	struct address_space *mapping = bdev->bd_inode->i_mapping;

67
	if (mapping->nrpages == 0 && mapping->nrshadows == 0)
Peter Zijlstra's avatar
Peter Zijlstra committed
68
		return;
Al Viro's avatar
Al Viro committed
69

Peter Zijlstra's avatar
Peter Zijlstra committed
70
	invalidate_bh_lrus();
Al Viro's avatar
Al Viro committed
71
	truncate_inode_pages(mapping, 0);
Linus Torvalds's avatar
Linus Torvalds committed
72
}	
Al Viro's avatar
Al Viro committed
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
EXPORT_SYMBOL(kill_bdev);

/* Invalidate clean unused buffers and pagecache. */
void invalidate_bdev(struct block_device *bdev)
{
	struct address_space *mapping = bdev->bd_inode->i_mapping;

	if (mapping->nrpages == 0)
		return;

	invalidate_bh_lrus();
	lru_add_drain_all();	/* make sure all lru add caches are flushed */
	invalidate_mapping_pages(mapping, 0, -1);
	/* 99% of the time, we don't need to flush the cleancache on the bdev.
	 * But, for the strange corners, lets be cautious
	 */
89
	cleancache_invalidate_inode(mapping);
Al Viro's avatar
Al Viro committed
90 91
}
EXPORT_SYMBOL(invalidate_bdev);
Linus Torvalds's avatar
Linus Torvalds committed
92 93 94 95

int set_blocksize(struct block_device *bdev, int size)
{
	/* Size must be a power of two, and between 512 and PAGE_SIZE */
96
	if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size))
Linus Torvalds's avatar
Linus Torvalds committed
97 98 99
		return -EINVAL;

	/* Size cannot be smaller than the size supported by the device */
100
	if (size < bdev_logical_block_size(bdev))
Linus Torvalds's avatar
Linus Torvalds committed
101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121
		return -EINVAL;

	/* Don't change the size if it is same as current */
	if (bdev->bd_block_size != size) {
		sync_blockdev(bdev);
		bdev->bd_block_size = size;
		bdev->bd_inode->i_blkbits = blksize_bits(size);
		kill_bdev(bdev);
	}
	return 0;
}

EXPORT_SYMBOL(set_blocksize);

int sb_set_blocksize(struct super_block *sb, int size)
{
	if (set_blocksize(sb->s_bdev, size))
		return 0;
	/* If we get here, we know size is power of two
	 * and it's value is between 512 and PAGE_SIZE */
	sb->s_blocksize = size;
122
	sb->s_blocksize_bits = blksize_bits(size);
Linus Torvalds's avatar
Linus Torvalds committed
123 124 125 126 127 128 129
	return sb->s_blocksize;
}

EXPORT_SYMBOL(sb_set_blocksize);

int sb_min_blocksize(struct super_block *sb, int size)
{
130
	int minsize = bdev_logical_block_size(sb->s_bdev);
Linus Torvalds's avatar
Linus Torvalds committed
131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
	if (size < minsize)
		size = minsize;
	return sb_set_blocksize(sb, size);
}

EXPORT_SYMBOL(sb_min_blocksize);

static int
blkdev_get_block(struct inode *inode, sector_t iblock,
		struct buffer_head *bh, int create)
{
	bh->b_bdev = I_BDEV(inode);
	bh->b_blocknr = iblock;
	set_buffer_mapped(bh);
	return 0;
}

148
static ssize_t
149
blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, loff_t offset)
150 151 152 153
{
	struct file *file = iocb->ki_filp;
	struct inode *inode = file->f_mapping->host;

154
	return __blockdev_direct_IO(iocb, inode, I_BDEV(inode), iter, offset,
155 156
				    blkdev_get_block, NULL, NULL,
				    DIO_SKIP_DIO_COUNT);
157 158
}

159 160 161 162 163 164 165 166 167
int __sync_blockdev(struct block_device *bdev, int wait)
{
	if (!bdev)
		return 0;
	if (!wait)
		return filemap_flush(bdev->bd_inode->i_mapping);
	return filemap_write_and_wait(bdev->bd_inode->i_mapping);
}

168 169 170 171 172 173
/*
 * Write out and wait upon all the dirty data associated with a block
 * device via its mapping.  Does not take the superblock lock.
 */
int sync_blockdev(struct block_device *bdev)
{
174
	return __sync_blockdev(bdev, 1);
175 176 177 178 179 180 181 182 183 184 185 186
}
EXPORT_SYMBOL(sync_blockdev);

/*
 * Write out and wait upon all dirty data associated with this
 * device.   Filesystem data as well as the underlying block
 * device.  Takes the superblock lock.
 */
int fsync_bdev(struct block_device *bdev)
{
	struct super_block *sb = get_super(bdev);
	if (sb) {
187
		int res = sync_filesystem(sb);
188 189 190 191 192
		drop_super(sb);
		return res;
	}
	return sync_blockdev(bdev);
}
193
EXPORT_SYMBOL(fsync_bdev);
194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212

/**
 * freeze_bdev  --  lock a filesystem and force it into a consistent state
 * @bdev:	blockdevice to lock
 *
 * If a superblock is found on this device, we take the s_umount semaphore
 * on it to make sure nobody unmounts until the snapshot creation is done.
 * The reference counter (bd_fsfreeze_count) guarantees that only the last
 * unfreeze process can unfreeze the frozen filesystem actually when multiple
 * freeze requests arrive simultaneously. It counts up in freeze_bdev() and
 * count down in thaw_bdev(). When it becomes 0, thaw_bdev() will unfreeze
 * actually.
 */
struct super_block *freeze_bdev(struct block_device *bdev)
{
	struct super_block *sb;
	int error = 0;

	mutex_lock(&bdev->bd_fsfreeze_mutex);
213 214 215 216 217 218
	if (++bdev->bd_fsfreeze_count > 1) {
		/*
		 * We don't even need to grab a reference - the first call
		 * to freeze_bdev grab an active reference and only the last
		 * thaw_bdev drops it.
		 */
219
		sb = get_super(bdev);
220 221 222 223 224 225 226 227
		drop_super(sb);
		mutex_unlock(&bdev->bd_fsfreeze_mutex);
		return sb;
	}

	sb = get_active_super(bdev);
	if (!sb)
		goto out;
228 229 230 231
	if (sb->s_op->freeze_super)
		error = sb->s_op->freeze_super(sb);
	else
		error = freeze_super(sb);
232 233 234
	if (error) {
		deactivate_super(sb);
		bdev->bd_fsfreeze_count--;
235
		mutex_unlock(&bdev->bd_fsfreeze_mutex);
236
		return ERR_PTR(error);
237
	}
238
	deactivate_super(sb);
239
 out:
240 241
	sync_blockdev(bdev);
	mutex_unlock(&bdev->bd_fsfreeze_mutex);
242
	return sb;	/* thaw_bdev releases s->s_umount */
243 244 245 246 247 248 249 250 251 252 253 254
}
EXPORT_SYMBOL(freeze_bdev);

/**
 * thaw_bdev  -- unlock filesystem
 * @bdev:	blockdevice to unlock
 * @sb:		associated superblock
 *
 * Unlocks the filesystem and marks it writeable again after freeze_bdev().
 */
int thaw_bdev(struct block_device *bdev, struct super_block *sb)
{
255
	int error = -EINVAL;
256 257

	mutex_lock(&bdev->bd_fsfreeze_mutex);
258
	if (!bdev->bd_fsfreeze_count)
259
		goto out;
260 261 262

	error = 0;
	if (--bdev->bd_fsfreeze_count > 0)
263
		goto out;
264 265

	if (!sb)
266
		goto out;
267

268 269 270 271
	if (sb->s_op->thaw_super)
		error = sb->s_op->thaw_super(sb);
	else
		error = thaw_super(sb);
272 273 274 275 276 277
	if (error) {
		bdev->bd_fsfreeze_count++;
		mutex_unlock(&bdev->bd_fsfreeze_mutex);
		return error;
	}
out:
278 279 280 281 282
	mutex_unlock(&bdev->bd_fsfreeze_mutex);
	return 0;
}
EXPORT_SYMBOL(thaw_bdev);

Linus Torvalds's avatar
Linus Torvalds committed
283 284 285 286 287 288 289 290 291 292
static int blkdev_writepage(struct page *page, struct writeback_control *wbc)
{
	return block_write_full_page(page, blkdev_get_block, wbc);
}

static int blkdev_readpage(struct file * file, struct page * page)
{
	return block_read_full_page(page, blkdev_get_block);
}

293 294 295 296 297 298
static int blkdev_readpages(struct file *file, struct address_space *mapping,
			struct list_head *pages, unsigned nr_pages)
{
	return mpage_readpages(mapping, pages, nr_pages, blkdev_get_block);
}

Nick Piggin's avatar
Nick Piggin committed
299 300 301
static int blkdev_write_begin(struct file *file, struct address_space *mapping,
			loff_t pos, unsigned len, unsigned flags,
			struct page **pagep, void **fsdata)
Linus Torvalds's avatar
Linus Torvalds committed
302
{
303 304
	return block_write_begin(mapping, pos, len, flags, pagep,
				 blkdev_get_block);
Linus Torvalds's avatar
Linus Torvalds committed
305 306
}

Nick Piggin's avatar
Nick Piggin committed
307 308 309
static int blkdev_write_end(struct file *file, struct address_space *mapping,
			loff_t pos, unsigned len, unsigned copied,
			struct page *page, void *fsdata)
Linus Torvalds's avatar
Linus Torvalds committed
310
{
Nick Piggin's avatar
Nick Piggin committed
311 312 313 314 315 316 317
	int ret;
	ret = block_write_end(file, mapping, pos, len, copied, page, fsdata);

	unlock_page(page);
	page_cache_release(page);

	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
318 319 320 321
}

/*
 * private llseek:
Al Viro's avatar
Al Viro committed
322
 * for a block special file file_inode(file)->i_size is zero
Linus Torvalds's avatar
Linus Torvalds committed
323 324
 * so we compute the size by hand (just as in block_read/write above)
 */
325
static loff_t block_llseek(struct file *file, loff_t offset, int whence)
Linus Torvalds's avatar
Linus Torvalds committed
326 327 328 329
{
	struct inode *bd_inode = file->f_mapping->host;
	loff_t retval;

330
	mutex_lock(&bd_inode->i_mutex);
331
	retval = fixed_size_llseek(file, offset, whence, i_size_read(bd_inode));
332
	mutex_unlock(&bd_inode->i_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
333 334 335
	return retval;
}
	
336
int blkdev_fsync(struct file *filp, loff_t start, loff_t end, int datasync)
Linus Torvalds's avatar
Linus Torvalds committed
337
{
338 339
	struct inode *bd_inode = filp->f_mapping->host;
	struct block_device *bdev = I_BDEV(bd_inode);
340
	int error;
341 342 343 344
	
	error = filemap_write_and_wait_range(filp->f_mapping, start, end);
	if (error)
		return error;
345

346 347 348 349 350
	/*
	 * There is no need to serialise calls to blkdev_issue_flush with
	 * i_mutex and doing so causes performance issues with concurrent
	 * O_SYNC writers to a block device.
	 */
351
	error = blkdev_issue_flush(bdev, GFP_KERNEL, NULL);
352 353
	if (error == -EOPNOTSUPP)
		error = 0;
354

355
	return error;
Linus Torvalds's avatar
Linus Torvalds committed
356
}
357
EXPORT_SYMBOL(blkdev_fsync);
Linus Torvalds's avatar
Linus Torvalds committed
358

359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421
/**
 * bdev_read_page() - Start reading a page from a block device
 * @bdev: The device to read the page from
 * @sector: The offset on the device to read the page to (need not be aligned)
 * @page: The page to read
 *
 * On entry, the page should be locked.  It will be unlocked when the page
 * has been read.  If the block driver implements rw_page synchronously,
 * that will be true on exit from this function, but it need not be.
 *
 * Errors returned by this function are usually "soft", eg out of memory, or
 * queue full; callers should try a different route to read this page rather
 * than propagate an error back up the stack.
 *
 * Return: negative errno if an error occurs, 0 if submission was successful.
 */
int bdev_read_page(struct block_device *bdev, sector_t sector,
			struct page *page)
{
	const struct block_device_operations *ops = bdev->bd_disk->fops;
	if (!ops->rw_page)
		return -EOPNOTSUPP;
	return ops->rw_page(bdev, sector + get_start_sect(bdev), page, READ);
}
EXPORT_SYMBOL_GPL(bdev_read_page);

/**
 * bdev_write_page() - Start writing a page to a block device
 * @bdev: The device to write the page to
 * @sector: The offset on the device to write the page to (need not be aligned)
 * @page: The page to write
 * @wbc: The writeback_control for the write
 *
 * On entry, the page should be locked and not currently under writeback.
 * On exit, if the write started successfully, the page will be unlocked and
 * under writeback.  If the write failed already (eg the driver failed to
 * queue the page to the device), the page will still be locked.  If the
 * caller is a ->writepage implementation, it will need to unlock the page.
 *
 * Errors returned by this function are usually "soft", eg out of memory, or
 * queue full; callers should try a different route to write this page rather
 * than propagate an error back up the stack.
 *
 * Return: negative errno if an error occurs, 0 if submission was successful.
 */
int bdev_write_page(struct block_device *bdev, sector_t sector,
			struct page *page, struct writeback_control *wbc)
{
	int result;
	int rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : WRITE;
	const struct block_device_operations *ops = bdev->bd_disk->fops;
	if (!ops->rw_page)
		return -EOPNOTSUPP;
	set_page_writeback(page);
	result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, rw);
	if (result)
		end_page_writeback(page);
	else
		unlock_page(page);
	return result;
}
EXPORT_SYMBOL_GPL(bdev_write_page);

422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461
/**
 * bdev_direct_access() - Get the address for directly-accessibly memory
 * @bdev: The device containing the memory
 * @sector: The offset within the device
 * @addr: Where to put the address of the memory
 * @pfn: The Page Frame Number for the memory
 * @size: The number of bytes requested
 *
 * If a block device is made up of directly addressable memory, this function
 * will tell the caller the PFN and the address of the memory.  The address
 * may be directly dereferenced within the kernel without the need to call
 * ioremap(), kmap() or similar.  The PFN is suitable for inserting into
 * page tables.
 *
 * Return: negative errno if an error occurs, otherwise the number of bytes
 * accessible at this address.
 */
long bdev_direct_access(struct block_device *bdev, sector_t sector,
			void **addr, unsigned long *pfn, long size)
{
	long avail;
	const struct block_device_operations *ops = bdev->bd_disk->fops;

	if (size < 0)
		return size;
	if (!ops->direct_access)
		return -EOPNOTSUPP;
	if ((sector + DIV_ROUND_UP(size, 512)) >
					part_nr_sects_read(bdev->bd_part))
		return -ERANGE;
	sector += get_start_sect(bdev);
	if (sector % (PAGE_SIZE / 512))
		return -EINVAL;
	avail = ops->direct_access(bdev, sector, addr, pfn, size);
	if (!avail)
		return -ERANGE;
	return min(avail, size);
}
EXPORT_SYMBOL_GPL(bdev_direct_access);

Linus Torvalds's avatar
Linus Torvalds committed
462 463 464 465 466
/*
 * pseudo-fs
 */

static  __cacheline_aligned_in_smp DEFINE_SPINLOCK(bdev_lock);
467
static struct kmem_cache * bdev_cachep __read_mostly;
Linus Torvalds's avatar
Linus Torvalds committed
468 469 470

static struct inode *bdev_alloc_inode(struct super_block *sb)
{
471
	struct bdev_inode *ei = kmem_cache_alloc(bdev_cachep, GFP_KERNEL);
Linus Torvalds's avatar
Linus Torvalds committed
472 473 474 475 476
	if (!ei)
		return NULL;
	return &ei->vfs_inode;
}

Nick Piggin's avatar
Nick Piggin committed
477
static void bdev_i_callback(struct rcu_head *head)
Linus Torvalds's avatar
Linus Torvalds committed
478
{
Nick Piggin's avatar
Nick Piggin committed
479
	struct inode *inode = container_of(head, struct inode, i_rcu);
Linus Torvalds's avatar
Linus Torvalds committed
480 481 482 483 484
	struct bdev_inode *bdi = BDEV_I(inode);

	kmem_cache_free(bdev_cachep, bdi);
}

Nick Piggin's avatar
Nick Piggin committed
485 486 487 488 489
static void bdev_destroy_inode(struct inode *inode)
{
	call_rcu(&inode->i_rcu, bdev_i_callback);
}

490
static void init_once(void *foo)
Linus Torvalds's avatar
Linus Torvalds committed
491 492 493 494
{
	struct bdev_inode *ei = (struct bdev_inode *) foo;
	struct block_device *bdev = &ei->bdev;

495 496 497 498
	memset(bdev, 0, sizeof(*bdev));
	mutex_init(&bdev->bd_mutex);
	INIT_LIST_HEAD(&bdev->bd_inodes);
	INIT_LIST_HEAD(&bdev->bd_list);
499 500 501
#ifdef CONFIG_SYSFS
	INIT_LIST_HEAD(&bdev->bd_holder_disks);
#endif
502
	inode_init_once(&ei->vfs_inode);
503 504
	/* Initialize mutex for freeze. */
	mutex_init(&bdev->bd_fsfreeze_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
505 506 507 508 509 510 511 512 513
}

static inline void __bd_forget(struct inode *inode)
{
	list_del_init(&inode->i_devices);
	inode->i_bdev = NULL;
	inode->i_mapping = &inode->i_data;
}

514
static void bdev_evict_inode(struct inode *inode)
Linus Torvalds's avatar
Linus Torvalds committed
515 516 517
{
	struct block_device *bdev = &BDEV_I(inode)->bdev;
	struct list_head *p;
518
	truncate_inode_pages_final(&inode->i_data);
519
	invalidate_inode_buffers(inode); /* is it needed here? */
520
	clear_inode(inode);
Linus Torvalds's avatar
Linus Torvalds committed
521 522 523 524 525 526 527 528
	spin_lock(&bdev_lock);
	while ( (p = bdev->bd_inodes.next) != &bdev->bd_inodes ) {
		__bd_forget(list_entry(p, struct inode, i_devices));
	}
	list_del_init(&bdev->bd_list);
	spin_unlock(&bdev_lock);
}

529
static const struct super_operations bdev_sops = {
Linus Torvalds's avatar
Linus Torvalds committed
530 531 532 533
	.statfs = simple_statfs,
	.alloc_inode = bdev_alloc_inode,
	.destroy_inode = bdev_destroy_inode,
	.drop_inode = generic_delete_inode,
534
	.evict_inode = bdev_evict_inode,
Linus Torvalds's avatar
Linus Torvalds committed
535 536
};

Al Viro's avatar
Al Viro committed
537 538
static struct dentry *bd_mount(struct file_system_type *fs_type,
	int flags, const char *dev_name, void *data)
Linus Torvalds's avatar
Linus Torvalds committed
539
{
540
	return mount_pseudo(fs_type, "bdev:", &bdev_sops, NULL, BDEVFS_MAGIC);
Linus Torvalds's avatar
Linus Torvalds committed
541 542 543 544
}

static struct file_system_type bd_type = {
	.name		= "bdev",
Al Viro's avatar
Al Viro committed
545
	.mount		= bd_mount,
Linus Torvalds's avatar
Linus Torvalds committed
546 547 548
	.kill_sb	= kill_anon_super,
};

Al Viro's avatar
Al Viro committed
549
static struct super_block *blockdev_superblock __read_mostly;
Linus Torvalds's avatar
Linus Torvalds committed
550 551 552 553

void __init bdev_cache_init(void)
{
	int err;
554
	static struct vfsmount *bd_mnt;
555

Linus Torvalds's avatar
Linus Torvalds committed
556
	bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode),
557 558
			0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
				SLAB_MEM_SPREAD|SLAB_PANIC),
559
			init_once);
Linus Torvalds's avatar
Linus Torvalds committed
560 561 562 563 564 565
	err = register_filesystem(&bd_type);
	if (err)
		panic("Cannot register bdev pseudo-fs");
	bd_mnt = kern_mount(&bd_type);
	if (IS_ERR(bd_mnt))
		panic("Cannot create bdev pseudo-fs");
566
	blockdev_superblock = bd_mnt->mnt_sb;   /* For writeback */
Linus Torvalds's avatar
Linus Torvalds committed
567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596
}

/*
 * Most likely _very_ bad one - but then it's hardly critical for small
 * /dev and can be fixed when somebody will need really large one.
 * Keep in mind that it will be fed through icache hash function too.
 */
static inline unsigned long hash(dev_t dev)
{
	return MAJOR(dev)+MINOR(dev);
}

static int bdev_test(struct inode *inode, void *data)
{
	return BDEV_I(inode)->bdev.bd_dev == *(dev_t *)data;
}

static int bdev_set(struct inode *inode, void *data)
{
	BDEV_I(inode)->bdev.bd_dev = *(dev_t *)data;
	return 0;
}

static LIST_HEAD(all_bdevs);

struct block_device *bdget(dev_t dev)
{
	struct block_device *bdev;
	struct inode *inode;

597
	inode = iget5_locked(blockdev_superblock, hash(dev),
Linus Torvalds's avatar
Linus Torvalds committed
598 599 600 601 602 603 604 605 606
			bdev_test, bdev_set, &dev);

	if (!inode)
		return NULL;

	bdev = &BDEV_I(inode)->bdev;

	if (inode->i_state & I_NEW) {
		bdev->bd_contains = NULL;
607
		bdev->bd_super = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626
		bdev->bd_inode = inode;
		bdev->bd_block_size = (1 << inode->i_blkbits);
		bdev->bd_part_count = 0;
		bdev->bd_invalidated = 0;
		inode->i_mode = S_IFBLK;
		inode->i_rdev = dev;
		inode->i_bdev = bdev;
		inode->i_data.a_ops = &def_blk_aops;
		mapping_set_gfp_mask(&inode->i_data, GFP_USER);
		spin_lock(&bdev_lock);
		list_add(&bdev->bd_list, &all_bdevs);
		spin_unlock(&bdev_lock);
		unlock_new_inode(inode);
	}
	return bdev;
}

EXPORT_SYMBOL(bdget);

627 628 629 630 631 632
/**
 * bdgrab -- Grab a reference to an already referenced block device
 * @bdev:	Block device to grab a reference to.
 */
struct block_device *bdgrab(struct block_device *bdev)
{
Al Viro's avatar
Al Viro committed
633
	ihold(bdev->bd_inode);
634 635
	return bdev;
}
636
EXPORT_SYMBOL(bdgrab);
637

Linus Torvalds's avatar
Linus Torvalds committed
638 639
long nr_blockdev_pages(void)
{
640
	struct block_device *bdev;
Linus Torvalds's avatar
Linus Torvalds committed
641 642
	long ret = 0;
	spin_lock(&bdev_lock);
643
	list_for_each_entry(bdev, &all_bdevs, bd_list) {
Linus Torvalds's avatar
Linus Torvalds committed
644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659
		ret += bdev->bd_inode->i_mapping->nrpages;
	}
	spin_unlock(&bdev_lock);
	return ret;
}

void bdput(struct block_device *bdev)
{
	iput(bdev->bd_inode);
}

EXPORT_SYMBOL(bdput);
 
static struct block_device *bd_acquire(struct inode *inode)
{
	struct block_device *bdev;
660

Linus Torvalds's avatar
Linus Torvalds committed
661 662
	spin_lock(&bdev_lock);
	bdev = inode->i_bdev;
663
	if (bdev) {
Al Viro's avatar
Al Viro committed
664
		ihold(bdev->bd_inode);
Linus Torvalds's avatar
Linus Torvalds committed
665 666 667 668
		spin_unlock(&bdev_lock);
		return bdev;
	}
	spin_unlock(&bdev_lock);
669

Linus Torvalds's avatar
Linus Torvalds committed
670 671 672
	bdev = bdget(inode->i_rdev);
	if (bdev) {
		spin_lock(&bdev_lock);
673 674
		if (!inode->i_bdev) {
			/*
Al Viro's avatar
Al Viro committed
675
			 * We take an additional reference to bd_inode,
676 677 678 679
			 * and it's released in clear_inode() of inode.
			 * So, we can access it via ->i_mapping always
			 * without igrab().
			 */
Al Viro's avatar
Al Viro committed
680
			ihold(bdev->bd_inode);
681 682 683 684
			inode->i_bdev = bdev;
			inode->i_mapping = bdev->bd_inode->i_mapping;
			list_add(&inode->i_devices, &bdev->bd_inodes);
		}
Linus Torvalds's avatar
Linus Torvalds committed
685 686 687 688 689
		spin_unlock(&bdev_lock);
	}
	return bdev;
}

690
int sb_is_blkdev_sb(struct super_block *sb)
Al Viro's avatar
Al Viro committed
691 692 693 694
{
	return sb == blockdev_superblock;
}

Linus Torvalds's avatar
Linus Torvalds committed
695 696 697 698
/* Call when you free inode */

void bd_forget(struct inode *inode)
{
699 700
	struct block_device *bdev = NULL;

Linus Torvalds's avatar
Linus Torvalds committed
701
	spin_lock(&bdev_lock);
702 703 704
	if (!sb_is_blkdev_sb(inode->i_sb))
		bdev = inode->i_bdev;
	__bd_forget(inode);
Linus Torvalds's avatar
Linus Torvalds committed
705
	spin_unlock(&bdev_lock);
706 707 708

	if (bdev)
		iput(bdev->bd_inode);
Linus Torvalds's avatar
Linus Torvalds committed
709 710
}

Tejun Heo's avatar
Tejun Heo committed
711 712 713 714 715 716
/**
 * bd_may_claim - test whether a block device can be claimed
 * @bdev: block device of interest
 * @whole: whole block device containing @bdev, may equal @bdev
 * @holder: holder trying to claim @bdev
 *
Lucas De Marchi's avatar
Lucas De Marchi committed
717
 * Test whether @bdev can be claimed by @holder.
Tejun Heo's avatar
Tejun Heo committed
718 719 720 721 722 723 724 725 726
 *
 * CONTEXT:
 * spin_lock(&bdev_lock).
 *
 * RETURNS:
 * %true if @bdev can be claimed, %false otherwise.
 */
static bool bd_may_claim(struct block_device *bdev, struct block_device *whole,
			 void *holder)
Linus Torvalds's avatar
Linus Torvalds committed
727 728
{
	if (bdev->bd_holder == holder)
Tejun Heo's avatar
Tejun Heo committed
729
		return true;	 /* already a holder */
Linus Torvalds's avatar
Linus Torvalds committed
730
	else if (bdev->bd_holder != NULL)
Tejun Heo's avatar
Tejun Heo committed
731
		return false; 	 /* held by someone else */
Linus Torvalds's avatar
Linus Torvalds committed
732
	else if (bdev->bd_contains == bdev)
Tejun Heo's avatar
Tejun Heo committed
733
		return true;  	 /* is a whole device which isn't held */
Linus Torvalds's avatar
Linus Torvalds committed
734

735
	else if (whole->bd_holder == bd_may_claim)
Tejun Heo's avatar
Tejun Heo committed
736 737 738
		return true; 	 /* is a partition of a device that is being partitioned */
	else if (whole->bd_holder != NULL)
		return false;	 /* is a partition of a held device */
Linus Torvalds's avatar
Linus Torvalds committed
739
	else
Tejun Heo's avatar
Tejun Heo committed
740 741 742
		return true;	 /* is a partition of an un-held device */
}

743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768
/**
 * bd_prepare_to_claim - prepare to claim a block device
 * @bdev: block device of interest
 * @whole: the whole device containing @bdev, may equal @bdev
 * @holder: holder trying to claim @bdev
 *
 * Prepare to claim @bdev.  This function fails if @bdev is already
 * claimed by another holder and waits if another claiming is in
 * progress.  This function doesn't actually claim.  On successful
 * return, the caller has ownership of bd_claiming and bd_holder[s].
 *
 * CONTEXT:
 * spin_lock(&bdev_lock).  Might release bdev_lock, sleep and regrab
 * it multiple times.
 *
 * RETURNS:
 * 0 if @bdev can be claimed, -EBUSY otherwise.
 */
static int bd_prepare_to_claim(struct block_device *bdev,
			       struct block_device *whole, void *holder)
{
retry:
	/* if someone else claimed, fail */
	if (!bd_may_claim(bdev, whole, holder))
		return -EBUSY;

769 770
	/* if claiming is already in progress, wait for it to finish */
	if (whole->bd_claiming) {
771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793
		wait_queue_head_t *wq = bit_waitqueue(&whole->bd_claiming, 0);
		DEFINE_WAIT(wait);

		prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
		spin_unlock(&bdev_lock);
		schedule();
		finish_wait(wq, &wait);
		spin_lock(&bdev_lock);
		goto retry;
	}

	/* yay, all mine */
	return 0;
}

/**
 * bd_start_claiming - start claiming a block device
 * @bdev: block device of interest
 * @holder: holder trying to claim @bdev
 *
 * @bdev is about to be opened exclusively.  Check @bdev can be opened
 * exclusively and mark that an exclusive open is in progress.  Each
 * successful call to this function must be matched with a call to
794 795 796 797 798 799 800
 * either bd_finish_claiming() or bd_abort_claiming() (which do not
 * fail).
 *
 * This function is used to gain exclusive access to the block device
 * without actually causing other exclusive open attempts to fail. It
 * should be used when the open sequence itself requires exclusive
 * access but may subsequently fail.
801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825
 *
 * CONTEXT:
 * Might sleep.
 *
 * RETURNS:
 * Pointer to the block device containing @bdev on success, ERR_PTR()
 * value on failure.
 */
static struct block_device *bd_start_claiming(struct block_device *bdev,
					      void *holder)
{
	struct gendisk *disk;
	struct block_device *whole;
	int partno, err;

	might_sleep();

	/*
	 * @bdev might not have been initialized properly yet, look up
	 * and grab the outer block device the hard way.
	 */
	disk = get_gendisk(bdev->bd_dev, &partno);
	if (!disk)
		return ERR_PTR(-ENXIO);

826 827 828 829 830 831 832 833 834 835 836 837 838
	/*
	 * Normally, @bdev should equal what's returned from bdget_disk()
	 * if partno is 0; however, some drivers (floppy) use multiple
	 * bdev's for the same physical device and @bdev may be one of the
	 * aliases.  Keep @bdev if partno is 0.  This means claimer
	 * tracking is broken for those devices but it has always been that
	 * way.
	 */
	if (partno)
		whole = bdget_disk(disk, 0);
	else
		whole = bdgrab(bdev);

839
	module_put(disk->fops->owner);
840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858
	put_disk(disk);
	if (!whole)
		return ERR_PTR(-ENOMEM);

	/* prepare to claim, if successful, mark claiming in progress */
	spin_lock(&bdev_lock);

	err = bd_prepare_to_claim(bdev, whole, holder);
	if (err == 0) {
		whole->bd_claiming = holder;
		spin_unlock(&bdev_lock);
		return whole;
	} else {
		spin_unlock(&bdev_lock);
		bdput(whole);
		return ERR_PTR(err);
	}
}

859
#ifdef CONFIG_SYSFS
860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876
struct bd_holder_disk {
	struct list_head	list;
	struct gendisk		*disk;
	int			refcnt;
};

static struct bd_holder_disk *bd_find_holder_disk(struct block_device *bdev,
						  struct gendisk *disk)
{
	struct bd_holder_disk *holder;

	list_for_each_entry(holder, &bdev->bd_holder_disks, list)
		if (holder->disk == disk)
			return holder;
	return NULL;
}

877
static int add_symlink(struct kobject *from, struct kobject *to)
878
{
879
	return sysfs_create_link(from, to, kobject_name(to));
880 881 882 883 884 885 886
}

static void del_symlink(struct kobject *from, struct kobject *to)
{
	sysfs_remove_link(from, kobject_name(to));
}

887
/**
888 889 890
 * bd_link_disk_holder - create symlinks between holding disk and slave bdev
 * @bdev: the claimed slave bdev
 * @disk: the holding disk
891
 *
892 893
 * DON'T USE THIS UNLESS YOU'RE ALREADY USING IT.
 *
894
 * This functions creates the following sysfs symlinks.
895
 *
896 897
 * - from "slaves" directory of the holder @disk to the claimed @bdev
 * - from "holders" directory of the @bdev to the holder @disk
898
 *
899 900
 * For example, if /dev/dm-0 maps to /dev/sda and disk for dm-0 is
 * passed to bd_link_disk_holder(), then:
901
 *
902 903
 *   /sys/block/dm-0/slaves/sda --> /sys/block/sda
 *   /sys/block/sda/holders/dm-0 --> /sys/block/dm-0
904
 *
905 906 907
 * The caller must have claimed @bdev before calling this function and
 * ensure that both @bdev and @disk are valid during the creation and
 * lifetime of these symlinks.
908
 *
909 910
 * CONTEXT:
 * Might sleep.
911
 *
912 913
 * RETURNS:
 * 0 on success, -errno on failure.
914
 */
915
int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk)
916
{
917
	struct bd_holder_disk *holder;
918
	int ret = 0;
919

920
	mutex_lock(&bdev->bd_mutex);
921

922
	WARN_ON_ONCE(!bdev->bd_holder);
923

924 925 926
	/* FIXME: remove the following once add_disk() handles errors */
	if (WARN_ON(!disk->slave_dir || !bdev->bd_part->holder_dir))
		goto out_unlock;
927

928 929 930
	holder = bd_find_holder_disk(bdev, disk);
	if (holder) {
		holder->refcnt++;
931
		goto out_unlock;
932
	}
933

934 935 936
	holder = kzalloc(sizeof(*holder), GFP_KERNEL);
	if (!holder) {
		ret = -ENOMEM;
937 938
		goto out_unlock;
	}
939

940 941 942 943 944 945 946 947 948 949 950
	INIT_LIST_HEAD(&holder->list);
	holder->disk = disk;
	holder->refcnt = 1;

	ret = add_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj);
	if (ret)
		goto out_free;

	ret = add_symlink(bdev->bd_part->holder_dir, &disk_to_dev(disk)->kobj);
	if (ret)
		goto out_del;
951 952 953 954 955
	/*
	 * bdev could be deleted beneath us which would implicitly destroy
	 * the holder directory.  Hold on to it.
	 */
	kobject_get(bdev->bd_part->holder_dir);
956 957 958 959 960 961 962 963

	list_add(&holder->list, &bdev->bd_holder_disks);
	goto out_unlock;

out_del:
	del_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj);
out_free:
	kfree(holder);
964
out_unlock:
965
	mutex_unlock(&bdev->bd_mutex);
966
	return ret;
967
}
968
EXPORT_SYMBOL_GPL(bd_link_disk_holder);
969

970 971 972 973 974 975 976 977 978 979 980
/**
 * bd_unlink_disk_holder - destroy symlinks created by bd_link_disk_holder()
 * @bdev: the calimed slave bdev
 * @disk: the holding disk
 *
 * DON'T USE THIS UNLESS YOU'RE ALREADY USING IT.
 *
 * CONTEXT:
 * Might sleep.
 */
void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk)
981
{
982
	struct bd_holder_disk *holder;
983

984
	mutex_lock(&bdev->bd_mutex);
985

986 987 988 989 990 991
	holder = bd_find_holder_disk(bdev, disk);

	if (!WARN_ON_ONCE(holder == NULL) && !--holder->refcnt) {
		del_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj);
		del_symlink(bdev->bd_part->holder_dir,
			    &disk_to_dev(disk)->kobj);
992
		kobject_put(bdev->bd_part->holder_dir);
993 994 995 996 997
		list_del_init(&holder->list);
		kfree(holder);
	}

	mutex_unlock(&bdev->bd_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
998
}
999
EXPORT_SYMBOL_GPL(bd_unlink_disk_holder);
1000
#endif
Linus Torvalds's avatar
Linus Torvalds committed
1001

1002 1003 1004 1005
/**
 * flush_disk - invalidates all buffer-cache entries on a disk
 *
 * @bdev:      struct block device to be flushed
1006
 * @kill_dirty: flag to guide handling of dirty inodes
1007 1008 1009 1010 1011
 *
 * Invalidates all buffer-cache entries on a disk. It should be called
 * when a disk has been changed -- either by a media change or online
 * resize.
 */
1012
static void flush_disk(struct block_device *bdev, bool kill_dirty)
1013
{
1014
	if (__invalidate_device(bdev, kill_dirty)) {
1015 1016 1017 1018 1019 1020 1021 1022 1023 1024
		char name[BDEVNAME_SIZE] = "";

		if (bdev->bd_disk)
			disk_name(bdev->bd_disk, 0, name);
		printk(KERN_WARNING "VFS: busy inodes on changed media or "
		       "resized disk %s\n", name);
	}

	if (!bdev->bd_disk)
		return;
Tejun Heo's avatar
Tejun Heo committed
1025
	if (disk_part_scan_enabled(bdev->bd_disk))
1026 1027 1028
		bdev->bd_invalidated = 1;
}

1029
/**
1030
 * check_disk_size_change - checks for disk size change and adjusts bdev size.
1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050
 * @disk: struct gendisk to check
 * @bdev: struct bdev to adjust.
 *
 * This routine checks to see if the bdev size does not match the disk size
 * and adjusts it if it differs.
 */
void check_disk_size_change(struct gendisk *disk, struct block_device *bdev)
{
	loff_t disk_size, bdev_size;

	disk_size = (loff_t)get_capacity(disk) << 9;
	bdev_size = i_size_read(bdev->bd_inode);
	if (disk_size != bdev_size) {
		char name[BDEVNAME_SIZE];

		disk_name(disk, 0, name);
		printk(KERN_INFO
		       "%s: detected capacity change from %lld to %lld\n",
		       name, bdev_size, disk_size);
		i_size_write(bdev->bd_inode, disk_size);
1051
		flush_disk(bdev, false);
1052 1053 1054 1055
	}
}
EXPORT_SYMBOL(check_disk_size_change);

1056
/**
1057
 * revalidate_disk - wrapper for lower-level driver's revalidate_disk call-back
1058 1059 1060 1061 1062 1063 1064 1065
 * @disk: struct gendisk to be revalidated
 *
 * This routine is a wrapper for lower-level driver's revalidate_disk
 * call-backs.  It is used to do common pre and post operations needed
 * for all revalidate_disk operations.
 */
int revalidate_disk(struct gendisk *disk)
{
1066
	struct block_device *bdev;
1067 1068 1069 1070 1071
	int ret = 0;

	if (disk->fops->revalidate_disk)
		ret = disk->fops->revalidate_disk(disk);

1072 1073 1074 1075 1076 1077
	bdev = bdget_disk(disk, 0);
	if (!bdev)
		return ret;

	mutex_lock(&bdev->bd_mutex);
	check_disk_size_change(disk, bdev);
1078
	bdev->bd_invalidated = 0;
1079 1080
	mutex_unlock(&bdev->bd_mutex);
	bdput(bdev);
1081 1082 1083 1084
	return ret;
}
EXPORT_SYMBOL(revalidate_disk);

Linus Torvalds's avatar
Linus Torvalds committed
1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096
/*
 * This routine checks whether a removable media has been changed,
 * and invalidates all buffer-cache-entries in that case. This
 * is a relatively slow routine, so we have to try to minimize using
 * it. Thus it is called only upon a 'mount' or 'open'. This
 * is the best way of combining speed and utility, I think.
 * People changing diskettes in the middle of an operation deserve
 * to lose :-)
 */
int check_disk_change(struct block_device *bdev)
{
	struct gendisk *disk = bdev->bd_disk;
1097
	const struct block_device_operations *bdops = disk->fops;
1098
	unsigned int events;
Linus Torvalds's avatar
Linus Torvalds committed
1099

1100 1101 1102
	events = disk_clear_events(disk, DISK_EVENT_MEDIA_CHANGE |
				   DISK_EVENT_EJECT_REQUEST);
	if (!(events & DISK_EVENT_MEDIA_CHANGE))
Linus Torvalds's avatar
Linus Torvalds committed
1103 1104
		return 0;

1105
	flush_disk(bdev, true);
Linus Torvalds's avatar
Linus Torvalds committed
1106 1107 1108 1109 1110 1111 1112 1113 1114
	if (bdops->revalidate_disk)
		bdops->revalidate_disk(bdev->bd_disk);
	return 1;
}

EXPORT_SYMBOL(check_disk_change);

void bd_set_size(struct block_device *bdev, loff_t size)
{
1115
	unsigned bsize = bdev_logical_block_size(bdev);
Linus Torvalds's avatar
Linus Torvalds committed
1116

1117 1118 1119
	mutex_lock(&bdev->bd_inode->i_mutex);
	i_size_write(bdev->bd_inode, size);
	mutex_unlock(&bdev->bd_inode->i_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
	while (bsize < PAGE_CACHE_SIZE) {
		if (size & bsize)
			break;
		bsize <<= 1;
	}
	bdev->bd_block_size = bsize;
	bdev->bd_inode->i_blkbits = blksize_bits(bsize);
}
EXPORT_SYMBOL(bd_set_size);

Al Viro's avatar
Al Viro committed
1130
static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part);
1131

1132 1133 1134 1135 1136 1137 1138
/*
 * bd_mutex locking:
 *
 *  mutex_lock(part->bd_mutex)
 *    mutex_lock_nested(whole->bd_mutex, 1)
 */

1139
static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part)
Linus Torvalds's avatar
Linus Torvalds committed
1140 1141
{
	struct gendisk *disk;
1142
	struct module *owner;
1143
	int ret;
1144
	int partno;
1145 1146
	int perm = 0;

1147
	if (mode & FMODE_READ)
1148
		perm |= MAY_READ;
1149
	if (mode & FMODE_WRITE)
1150 1151 1152 1153
		perm |= MAY_WRITE;
	/*
	 * hooks: /n/, see "layering violations".
	 */
1154 1155 1156 1157 1158 1159
	if (!for_part) {
		ret = devcgroup_inode_permission(bdev->bd_inode, perm);
		if (ret != 0) {
			bdput(bdev);
			return ret;
		}
1160
	}
1161

1162
 restart:
Tejun Heo's avatar
Tejun Heo committed
1163

1164
	ret = -ENXIO;
1165
	disk = get_gendisk(bdev->bd_dev, &partno);
Tejun Heo's avatar
Tejun Heo committed
1166
	if (!disk)
1167
		goto out;
1168
	owner = disk->fops->owner;
Linus Torvalds's avatar
Linus Torvalds committed
1169

1170
	disk_block_events(disk);
1171
	mutex_lock_nested(&bdev->bd_mutex, for_part);
Linus Torvalds's avatar
Linus Torvalds committed
1172 1173
	if (!bdev->bd_openers) {
		bdev->bd_disk = disk;
1174
		bdev->bd_queue = disk->queue;
Linus Torvalds's avatar
Linus Torvalds committed
1175
		bdev->bd_contains = bdev;
1176
		if (!partno) {
1177 1178 1179 1180 1181
			ret = -ENXIO;
			bdev->bd_part = disk_get_part(disk, partno);
			if (!bdev->bd_part)
				goto out_clear;

1182
			ret = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1183
			if (disk->fops->open) {
1184
				ret = disk->fops->open(bdev, mode);
1185 1186 1187 1188 1189 1190 1191 1192
				if (ret == -ERESTARTSYS) {
					/* Lost a race with 'disk' being
					 * deleted, try again.
					 * See md.c
					 */
					disk_put_part(bdev->bd_part);
					bdev->bd_part = NULL;
					bdev->bd_disk = NULL;
1193
					bdev->bd_queue = NULL;
1194
					mutex_unlock(&bdev->bd_mutex);
1195 1196
					disk_unblock_events(disk);
					put_disk(disk);
1197
					module_put(owner);
1198 1199
					goto restart;
				}
Linus Torvalds's avatar
Linus Torvalds committed
1200
			}
1201

1202
			if (!ret)
1203 1204
				bd_set_size(bdev,(loff_t)get_capacity(disk)<<9);

1205 1206 1207 1208 1209 1210
			/*
			 * If the device is invalidated, rescan partition
			 * if open succeeded or failed with -ENOMEDIUM.
			 * The latter is necessary to prevent ghost
			 * partitions on a removed medium.
			 */
1211 1212 1213 1214 1215 1216
			if (bdev->bd_invalidated) {
				if (!ret)
					rescan_partitions(disk, bdev);
				else if (ret == -ENOMEDIUM)
					invalidate_partitions(disk, bdev);
			}
1217 1218
			if (ret)
				goto out_clear;
Linus Torvalds's avatar
Linus Torvalds committed
1219 1220 1221 1222 1223
		} else {
			struct block_device *whole;
			whole = bdget_disk(disk, 0);
			ret = -ENOMEM;
			if (!whole)
Tejun Heo's avatar
Tejun Heo committed
1224
				goto out_clear;
1225
			BUG_ON(for_part);
1226
			ret = __blkdev_get(whole, mode, 1);
Linus Torvalds's avatar
Linus Torvalds committed
1227
			if (ret)
Tejun Heo's avatar
Tejun Heo committed
1228
				goto out_clear;
Linus Torvalds's avatar
Linus Torvalds committed
1229
			bdev->bd_contains = whole;
1230
			bdev->bd_part = disk_get_part(disk, partno);
1231
			if (!(disk->flags & GENHD_FL_UP) ||
1232
			    !bdev->bd_part || !bdev->bd_part->nr_sects) {
Linus Torvalds's avatar
Linus Torvalds committed
1233
				ret = -ENXIO;
Tejun Heo's avatar
Tejun Heo committed
1234
				goto out_clear;
Linus Torvalds's avatar
Linus Torvalds committed
1235
			}
1236
			bd_set_size(bdev, (loff_t)bdev->bd_part->nr_sects << 9);
Linus Torvalds's avatar
Linus Torvalds committed
1237 1238 1239
		}
	} else {
		if (bdev->bd_contains == bdev) {
1240 1241
			ret = 0;
			if (bdev->bd_disk->fops->open)
1242
				ret = bdev->bd_disk->fops->open(bdev, mode);
1243
			/* the same as first opener case, read comment there */
1244 1245 1246 1247 1248 1249
			if (bdev->bd_invalidated) {
				if (!ret)
					rescan_partitions(bdev->bd_disk, bdev);
				else if (ret == -ENOMEDIUM)
					invalidate_partitions(bdev->bd_disk, bdev);
			}
1250 1251
			if (ret)
				goto out_unlock_bdev;
Linus Torvalds's avatar
Linus Torvalds committed
1252
		}
1253 1254
		/* only one opener holds refs to the module and disk */
		put_disk(disk);
1255
		module_put(owner);
Linus Torvalds's avatar
Linus Torvalds committed
1256 1257
	}
	bdev->bd_openers++;
1258 1259
	if (for_part)
		bdev->bd_part_count++;
1260
	mutex_unlock(&bdev->bd_mutex);
1261
	disk_unblock_events(disk);
Linus Torvalds's avatar
Linus Torvalds committed
1262 1263
	return 0;

Tejun Heo's avatar
Tejun Heo committed
1264
 out_clear:
1265
	disk_put_part(bdev->bd_part);
Linus Torvalds's avatar
Linus Torvalds committed
1266
	bdev->bd_disk = NULL;
Tejun Heo's avatar
Tejun Heo committed
1267
	bdev->bd_part = NULL;
1268
	bdev->bd_queue = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
1269
	if (bdev != bdev->bd_contains)
1270
		__blkdev_put(bdev->bd_contains, mode, 1);
Linus Torvalds's avatar
Linus Torvalds committed
1271
	bdev->bd_contains = NULL;
Tejun Heo's avatar
Tejun Heo committed
1272
 out_unlock_bdev:
1273
	mutex_unlock(&bdev->bd_mutex);
1274
	disk_unblock_events(disk);
Tejun Heo's avatar
Tejun Heo committed
1275
	put_disk(disk);
1276
	module_put(owner);
1277
 out:
Tejun Heo's avatar
Tejun Heo committed
1278 1279
	bdput(bdev);

Linus Torvalds's avatar
Linus Torvalds committed
1280 1281 1282
	return ret;
}

1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301
/**
 * blkdev_get - open a block device
 * @bdev: block_device to open
 * @mode: FMODE_* mask
 * @holder: exclusive holder identifier
 *
 * Open @bdev with @mode.  If @mode includes %FMODE_EXCL, @bdev is
 * open with exclusive access.  Specifying %FMODE_EXCL with %NULL
 * @holder is invalid.  Exclusive opens may nest for the same @holder.
 *
 * On success, the reference count of @bdev is unchanged.  On failure,
 * @bdev is put.
 *
 * CONTEXT:
 * Might sleep.
 *
 * RETURNS:
 * 0 on success, -errno on failure.
 */
1302
int blkdev_get(struct block_device *bdev, fmode_t mode, void *holder)
Linus Torvalds's avatar
Linus Torvalds committed
1303
{
1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319
	struct block_device *whole = NULL;
	int res;

	WARN_ON_ONCE((mode & FMODE_EXCL) && !holder);

	if ((mode & FMODE_EXCL) && holder) {
		whole = bd_start_claiming(bdev, holder);
		if (IS_ERR(whole)) {
			bdput(bdev);
			return PTR_ERR(whole);
		}
	}

	res = __blkdev_get(bdev, mode, 0);

	if (whole) {
1320 1321
		struct gendisk *disk = whole->bd_disk;

1322
		/* finish claiming */
1323
		mutex_lock(&bdev->bd_mutex);
1324 1325
		spin_lock(&bdev_lock);

1326
		if (!res) {
1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345
			BUG_ON(!bd_may_claim(bdev, whole, holder));
			/*
			 * Note that for a whole device bd_holders
			 * will be incremented twice, and bd_holder