fmr_pool.c 13.5 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2
/*
 * Copyright (c) 2004 Topspin Communications.  All rights reserved.
3
 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
Linus Torvalds's avatar
Linus Torvalds committed
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/errno.h>
#include <linux/spinlock.h>
36
#include <linux/export.h>
Linus Torvalds's avatar
Linus Torvalds committed
37 38 39 40
#include <linux/slab.h>
#include <linux/jhash.h>
#include <linux/kthread.h>

41
#include <rdma/ib_fmr_pool.h>
Linus Torvalds's avatar
Linus Torvalds committed
42 43 44

#include "core_priv.h"

45 46
#define PFX "fmr_pool: "

Linus Torvalds's avatar
Linus Torvalds committed
47 48 49 50 51 52 53 54 55 56 57
enum {
	IB_FMR_MAX_REMAPS = 32,

	IB_FMR_HASH_BITS  = 8,
	IB_FMR_HASH_SIZE  = 1 << IB_FMR_HASH_BITS,
	IB_FMR_HASH_MASK  = IB_FMR_HASH_SIZE - 1
};

/*
 * If an FMR is not in use, then the list member will point to either
 * its pool's free_list (if the FMR can be mapped again; that is,
58
 * remap_count < pool->max_remaps) or its pool's dirty_list (if the
Linus Torvalds's avatar
Linus Torvalds committed
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87
 * FMR needs to be unmapped before being remapped).  In either of
 * these cases it is a bug if the ref_count is not 0.  In other words,
 * if ref_count is > 0, then the list member must not be linked into
 * either free_list or dirty_list.
 *
 * The cache_node member is used to link the FMR into a cache bucket
 * (if caching is enabled).  This is independent of the reference
 * count of the FMR.  When a valid FMR is released, its ref_count is
 * decremented, and if ref_count reaches 0, the FMR is placed in
 * either free_list or dirty_list as appropriate.  However, it is not
 * removed from the cache and may be "revived" if a call to
 * ib_fmr_register_physical() occurs before the FMR is remapped.  In
 * this case we just increment the ref_count and remove the FMR from
 * free_list/dirty_list.
 *
 * Before we remap an FMR from free_list, we remove it from the cache
 * (to prevent another user from obtaining a stale FMR).  When an FMR
 * is released, we add it to the tail of the free list, so that our
 * cache eviction policy is "least recently used."
 *
 * All manipulation of ref_count, list and cache_node is protected by
 * pool_lock to maintain consistency.
 */

struct ib_fmr_pool {
	spinlock_t                pool_lock;

	int                       pool_size;
	int                       max_pages;
88
	int			  max_remaps;
Linus Torvalds's avatar
Linus Torvalds committed
89 90 91 92 93 94 95 96 97 98
	int                       dirty_watermark;
	int                       dirty_len;
	struct list_head          free_list;
	struct list_head          dirty_list;
	struct hlist_head        *cache_bucket;

	void                     (*flush_function)(struct ib_fmr_pool *pool,
						   void *              arg);
	void                     *flush_arg;

99 100
	struct kthread_worker	  *worker;
	struct kthread_work	  work;
Linus Torvalds's avatar
Linus Torvalds committed
101 102 103 104 105 106 107 108 109

	atomic_t                  req_ser;
	atomic_t                  flush_ser;

	wait_queue_head_t         force_wait;
};

static inline u32 ib_fmr_hash(u64 first_page)
{
110 111
	return jhash_2words((u32) first_page, (u32) (first_page >> 32), 0) &
		(IB_FMR_HASH_SIZE - 1);
Linus Torvalds's avatar
Linus Torvalds committed
112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
}

/* Caller must hold pool_lock */
static inline struct ib_pool_fmr *ib_fmr_cache_lookup(struct ib_fmr_pool *pool,
						      u64 *page_list,
						      int  page_list_len,
						      u64  io_virtual_address)
{
	struct hlist_head *bucket;
	struct ib_pool_fmr *fmr;

	if (!pool->cache_bucket)
		return NULL;

	bucket = pool->cache_bucket + ib_fmr_hash(*page_list);

128
	hlist_for_each_entry(fmr, bucket, cache_node)
Linus Torvalds's avatar
Linus Torvalds committed
129 130 131 132 133 134 135 136 137 138 139 140
		if (io_virtual_address == fmr->io_virtual_address &&
		    page_list_len      == fmr->page_list_len      &&
		    !memcmp(page_list, fmr->page_list,
			    page_list_len * sizeof *page_list))
			return fmr;

	return NULL;
}

static void ib_fmr_batch_release(struct ib_fmr_pool *pool)
{
	int                 ret;
141
	struct ib_pool_fmr *fmr;
Linus Torvalds's avatar
Linus Torvalds committed
142 143 144 145 146 147 148 149 150 151 152 153
	LIST_HEAD(unmap_list);
	LIST_HEAD(fmr_list);

	spin_lock_irq(&pool->pool_lock);

	list_for_each_entry(fmr, &pool->dirty_list, list) {
		hlist_del_init(&fmr->cache_node);
		fmr->remap_count = 0;
		list_add_tail(&fmr->fmr->list, &fmr_list);

#ifdef DEBUG
		if (fmr->ref_count !=0) {
154 155
			pr_warn(PFX "Unmapping FMR 0x%08x with ref count %d\n",
				fmr, fmr->ref_count);
Linus Torvalds's avatar
Linus Torvalds committed
156 157 158 159
		}
#endif
	}

160
	list_splice_init(&pool->dirty_list, &unmap_list);
Linus Torvalds's avatar
Linus Torvalds committed
161 162 163 164 165 166 167 168 169 170
	pool->dirty_len = 0;

	spin_unlock_irq(&pool->pool_lock);

	if (list_empty(&unmap_list)) {
		return;
	}

	ret = ib_unmap_fmr(&fmr_list);
	if (ret)
171
		pr_warn(PFX "ib_unmap_fmr returned %d\n", ret);
Linus Torvalds's avatar
Linus Torvalds committed
172 173 174 175 176 177

	spin_lock_irq(&pool->pool_lock);
	list_splice(&unmap_list, &pool->free_list);
	spin_unlock_irq(&pool->pool_lock);
}

178
static void ib_fmr_cleanup_func(struct kthread_work *work)
Linus Torvalds's avatar
Linus Torvalds committed
179
{
180
	struct ib_fmr_pool *pool = container_of(work, struct ib_fmr_pool, work);
Linus Torvalds's avatar
Linus Torvalds committed
181

182 183 184
	ib_fmr_batch_release(pool);
	atomic_inc(&pool->flush_ser);
	wake_up_interruptible(&pool->force_wait);
Linus Torvalds's avatar
Linus Torvalds committed
185

186 187
	if (pool->flush_function)
		pool->flush_function(pool, pool->flush_arg);
Linus Torvalds's avatar
Linus Torvalds committed
188

189 190
	if (atomic_read(&pool->flush_ser) - atomic_read(&pool->req_ser) < 0)
		kthread_queue_work(pool->worker, &pool->work);
Linus Torvalds's avatar
Linus Torvalds committed
191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
}

/**
 * ib_create_fmr_pool - Create an FMR pool
 * @pd:Protection domain for FMRs
 * @params:FMR pool parameters
 *
 * Create a pool of FMRs.  Return value is pointer to new pool or
 * error code if creation failed.
 */
struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd             *pd,
				       struct ib_fmr_pool_param *params)
{
	struct ib_device   *device;
	struct ib_fmr_pool *pool;
	int i;
	int ret;
208
	int max_remaps;
Linus Torvalds's avatar
Linus Torvalds committed
209 210 211 212 213 214 215

	if (!params)
		return ERR_PTR(-EINVAL);

	device = pd->device;
	if (!device->alloc_fmr    || !device->dealloc_fmr  ||
	    !device->map_phys_fmr || !device->unmap_fmr) {
216
		dev_info(&device->dev, "Device does not support FMRs\n");
Linus Torvalds's avatar
Linus Torvalds committed
217 218 219
		return ERR_PTR(-ENOSYS);
	}

220
	if (!device->attrs.max_map_per_fmr)
221 222
		max_remaps = IB_FMR_MAX_REMAPS;
	else
223
		max_remaps = device->attrs.max_map_per_fmr;
224

Linus Torvalds's avatar
Linus Torvalds committed
225
	pool = kmalloc(sizeof *pool, GFP_KERNEL);
226
	if (!pool)
Linus Torvalds's avatar
Linus Torvalds committed
227 228 229 230 231 232 233 234 235 236 237
		return ERR_PTR(-ENOMEM);

	pool->cache_bucket   = NULL;
	pool->flush_function = params->flush_function;
	pool->flush_arg      = params->flush_arg;

	INIT_LIST_HEAD(&pool->free_list);
	INIT_LIST_HEAD(&pool->dirty_list);

	if (params->cache) {
		pool->cache_bucket =
238 239 240
			kmalloc_array(IB_FMR_HASH_SIZE,
				      sizeof(*pool->cache_bucket),
				      GFP_KERNEL);
Linus Torvalds's avatar
Linus Torvalds committed
241 242 243 244 245 246 247 248 249 250 251
		if (!pool->cache_bucket) {
			ret = -ENOMEM;
			goto out_free_pool;
		}

		for (i = 0; i < IB_FMR_HASH_SIZE; ++i)
			INIT_HLIST_HEAD(pool->cache_bucket + i);
	}

	pool->pool_size       = 0;
	pool->max_pages       = params->max_pages_per_fmr;
252
	pool->max_remaps      = max_remaps;
Linus Torvalds's avatar
Linus Torvalds committed
253 254 255 256 257 258 259
	pool->dirty_watermark = params->dirty_watermark;
	pool->dirty_len       = 0;
	spin_lock_init(&pool->pool_lock);
	atomic_set(&pool->req_ser,   0);
	atomic_set(&pool->flush_ser, 0);
	init_waitqueue_head(&pool->force_wait);

260 261
	pool->worker =
		kthread_create_worker(0, "ib_fmr(%s)", dev_name(&device->dev));
262 263 264
	if (IS_ERR(pool->worker)) {
		pr_warn(PFX "couldn't start cleanup kthread worker\n");
		ret = PTR_ERR(pool->worker);
Linus Torvalds's avatar
Linus Torvalds committed
265 266
		goto out_free_pool;
	}
267
	kthread_init_work(&pool->work, ib_fmr_cleanup_func);
Linus Torvalds's avatar
Linus Torvalds committed
268 269 270

	{
		struct ib_pool_fmr *fmr;
271
		struct ib_fmr_attr fmr_attr = {
272
			.max_pages  = params->max_pages_per_fmr,
273
			.max_maps   = pool->max_remaps,
274
			.page_shift = params->page_shift
Linus Torvalds's avatar
Linus Torvalds committed
275
		};
276 277 278 279
		int bytes_per_fmr = sizeof *fmr;

		if (pool->cache_bucket)
			bytes_per_fmr += params->max_pages_per_fmr * sizeof (u64);
Linus Torvalds's avatar
Linus Torvalds committed
280 281

		for (i = 0; i < params->pool_size; ++i) {
282
			fmr = kmalloc(bytes_per_fmr, GFP_KERNEL);
283
			if (!fmr)
Linus Torvalds's avatar
Linus Torvalds committed
284 285 286 287 288 289 290
				goto out_fail;

			fmr->pool             = pool;
			fmr->remap_count      = 0;
			fmr->ref_count        = 0;
			INIT_HLIST_NODE(&fmr->cache_node);

291
			fmr->fmr = ib_alloc_fmr(pd, params->access, &fmr_attr);
Linus Torvalds's avatar
Linus Torvalds committed
292
			if (IS_ERR(fmr->fmr)) {
293 294
				pr_warn(PFX "fmr_create failed for FMR %d\n",
					i);
Linus Torvalds's avatar
Linus Torvalds committed
295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
				kfree(fmr);
				goto out_fail;
			}

			list_add_tail(&fmr->list, &pool->free_list);
			++pool->pool_size;
		}
	}

	return pool;

 out_free_pool:
	kfree(pool->cache_bucket);
	kfree(pool);

	return ERR_PTR(ret);

 out_fail:
	ib_destroy_fmr_pool(pool);

	return ERR_PTR(-ENOMEM);
}
EXPORT_SYMBOL(ib_create_fmr_pool);

/**
 * ib_destroy_fmr_pool - Free FMR pool
 * @pool:FMR pool to free
 *
 * Destroy an FMR pool and free all associated resources.
 */
325
void ib_destroy_fmr_pool(struct ib_fmr_pool *pool)
Linus Torvalds's avatar
Linus Torvalds committed
326 327 328
{
	struct ib_pool_fmr *fmr;
	struct ib_pool_fmr *tmp;
329
	LIST_HEAD(fmr_list);
Linus Torvalds's avatar
Linus Torvalds committed
330 331
	int                 i;

332
	kthread_destroy_worker(pool->worker);
Linus Torvalds's avatar
Linus Torvalds committed
333 334 335 336
	ib_fmr_batch_release(pool);

	i = 0;
	list_for_each_entry_safe(fmr, tmp, &pool->free_list, list) {
337 338 339 340 341
		if (fmr->remap_count) {
			INIT_LIST_HEAD(&fmr_list);
			list_add_tail(&fmr->fmr->list, &fmr_list);
			ib_unmap_fmr(&fmr_list);
		}
Linus Torvalds's avatar
Linus Torvalds committed
342 343 344 345 346 347 348
		ib_dealloc_fmr(fmr->fmr);
		list_del(&fmr->list);
		kfree(fmr);
		++i;
	}

	if (i < pool->pool_size)
349 350
		pr_warn(PFX "pool still has %d regions registered\n",
			pool->pool_size - i);
Linus Torvalds's avatar
Linus Torvalds committed
351 352 353 354 355 356 357 358 359 360 361 362 363 364

	kfree(pool->cache_bucket);
	kfree(pool);
}
EXPORT_SYMBOL(ib_destroy_fmr_pool);

/**
 * ib_flush_fmr_pool - Invalidate all unmapped FMRs
 * @pool:FMR pool to flush
 *
 * Ensure that all unmapped FMRs are fully invalidated.
 */
int ib_flush_fmr_pool(struct ib_fmr_pool *pool)
{
365 366 367 368 369 370 371 372 373 374 375 376 377 378 379
	int serial;
	struct ib_pool_fmr *fmr, *next;

	/*
	 * The free_list holds FMRs that may have been used
	 * but have not been remapped enough times to be dirty.
	 * Put them on the dirty list now so that the cleanup
	 * thread will reap them too.
	 */
	spin_lock_irq(&pool->pool_lock);
	list_for_each_entry_safe(fmr, next, &pool->free_list, list) {
		if (fmr->remap_count > 0)
			list_move(&fmr->list, &pool->dirty_list);
	}
	spin_unlock_irq(&pool->pool_lock);
Linus Torvalds's avatar
Linus Torvalds committed
380

381
	serial = atomic_inc_return(&pool->req_ser);
382
	kthread_queue_work(pool->worker, &pool->work);
Linus Torvalds's avatar
Linus Torvalds committed
383 384

	if (wait_event_interruptible(pool->force_wait,
385
				     atomic_read(&pool->flush_ser) - serial >= 0))
Linus Torvalds's avatar
Linus Torvalds committed
386 387 388 389 390 391 392
		return -EINTR;

	return 0;
}
EXPORT_SYMBOL(ib_flush_fmr_pool);

/**
393 394 395 396 397
 * ib_fmr_pool_map_phys - Map an FMR from an FMR pool.
 * @pool_handle: FMR pool to allocate FMR from
 * @page_list: List of pages to map
 * @list_len: Number of pages in @page_list
 * @io_virtual_address: I/O virtual address for new FMR
Linus Torvalds's avatar
Linus Torvalds committed
398 399 400 401
 */
struct ib_pool_fmr *ib_fmr_pool_map_phys(struct ib_fmr_pool *pool_handle,
					 u64                *page_list,
					 int                 list_len,
402
					 u64                 io_virtual_address)
Linus Torvalds's avatar
Linus Torvalds committed
403 404 405 406 407 408 409 410 411 412 413 414 415
{
	struct ib_fmr_pool *pool = pool_handle;
	struct ib_pool_fmr *fmr;
	unsigned long       flags;
	int                 result;

	if (list_len < 1 || list_len > pool->max_pages)
		return ERR_PTR(-EINVAL);

	spin_lock_irqsave(&pool->pool_lock, flags);
	fmr = ib_fmr_cache_lookup(pool,
				  page_list,
				  list_len,
416
				  io_virtual_address);
Linus Torvalds's avatar
Linus Torvalds committed
417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439
	if (fmr) {
		/* found in cache */
		++fmr->ref_count;
		if (fmr->ref_count == 1) {
			list_del(&fmr->list);
		}

		spin_unlock_irqrestore(&pool->pool_lock, flags);

		return fmr;
	}

	if (list_empty(&pool->free_list)) {
		spin_unlock_irqrestore(&pool->pool_lock, flags);
		return ERR_PTR(-EAGAIN);
	}

	fmr = list_entry(pool->free_list.next, struct ib_pool_fmr, list);
	list_del(&fmr->list);
	hlist_del_init(&fmr->cache_node);
	spin_unlock_irqrestore(&pool->pool_lock, flags);

	result = ib_map_phys_fmr(fmr->fmr, page_list, list_len,
440
				 io_virtual_address);
Linus Torvalds's avatar
Linus Torvalds committed
441 442 443 444 445 446

	if (result) {
		spin_lock_irqsave(&pool->pool_lock, flags);
		list_add(&fmr->list, &pool->free_list);
		spin_unlock_irqrestore(&pool->pool_lock, flags);

447
		pr_warn(PFX "fmr_map returns %d\n", result);
Linus Torvalds's avatar
Linus Torvalds committed
448 449 450 451 452 453 454 455

		return ERR_PTR(result);
	}

	++fmr->remap_count;
	fmr->ref_count = 1;

	if (pool->cache_bucket) {
456
		fmr->io_virtual_address = io_virtual_address;
Linus Torvalds's avatar
Linus Torvalds committed
457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487
		fmr->page_list_len      = list_len;
		memcpy(fmr->page_list, page_list, list_len * sizeof(*page_list));

		spin_lock_irqsave(&pool->pool_lock, flags);
		hlist_add_head(&fmr->cache_node,
			       pool->cache_bucket + ib_fmr_hash(fmr->page_list[0]));
		spin_unlock_irqrestore(&pool->pool_lock, flags);
	}

	return fmr;
}
EXPORT_SYMBOL(ib_fmr_pool_map_phys);

/**
 * ib_fmr_pool_unmap - Unmap FMR
 * @fmr:FMR to unmap
 *
 * Unmap an FMR.  The FMR mapping may remain valid until the FMR is
 * reused (or until ib_flush_fmr_pool() is called).
 */
int ib_fmr_pool_unmap(struct ib_pool_fmr *fmr)
{
	struct ib_fmr_pool *pool;
	unsigned long flags;

	pool = fmr->pool;

	spin_lock_irqsave(&pool->pool_lock, flags);

	--fmr->ref_count;
	if (!fmr->ref_count) {
488
		if (fmr->remap_count < pool->max_remaps) {
Linus Torvalds's avatar
Linus Torvalds committed
489 490 491
			list_add_tail(&fmr->list, &pool->free_list);
		} else {
			list_add_tail(&fmr->list, &pool->dirty_list);
492 493
			if (++pool->dirty_len >= pool->dirty_watermark) {
				atomic_inc(&pool->req_ser);
494
				kthread_queue_work(pool->worker, &pool->work);
495
			}
Linus Torvalds's avatar
Linus Torvalds committed
496 497 498 499 500
		}
	}

#ifdef DEBUG
	if (fmr->ref_count < 0)
501 502
		pr_warn(PFX "FMR %p has ref count %d < 0\n",
			fmr, fmr->ref_count);
Linus Torvalds's avatar
Linus Torvalds committed
503 504 505 506 507 508 509
#endif

	spin_unlock_irqrestore(&pool->pool_lock, flags);

	return 0;
}
EXPORT_SYMBOL(ib_fmr_pool_unmap);