dma-mapping.h 25.4 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
2 3
#ifndef _LINUX_DMA_MAPPING_H
#define _LINUX_DMA_MAPPING_H
Linus Torvalds's avatar
Linus Torvalds committed
4

5
#include <linux/sizes.h>
6
#include <linux/string.h>
Linus Torvalds's avatar
Linus Torvalds committed
7 8
#include <linux/device.h>
#include <linux/err.h>
9
#include <linux/dma-debug.h>
10
#include <linux/dma-direction.h>
11
#include <linux/scatterlist.h>
12
#include <linux/bug.h>
13
#include <linux/mem_encrypt.h>
Linus Torvalds's avatar
Linus Torvalds committed
14

15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
/**
 * List of possible attributes associated with a DMA mapping. The semantics
 * of each attribute should be defined in Documentation/DMA-attributes.txt.
 *
 * DMA_ATTR_WRITE_BARRIER: DMA to a memory region with this attribute
 * forces all pending DMA writes to complete.
 */
#define DMA_ATTR_WRITE_BARRIER		(1UL << 0)
/*
 * DMA_ATTR_WEAK_ORDERING: Specifies that reads and writes to the mapping
 * may be weakly ordered, that is that reads and writes may pass each other.
 */
#define DMA_ATTR_WEAK_ORDERING		(1UL << 1)
/*
 * DMA_ATTR_WRITE_COMBINE: Specifies that writes to the mapping may be
 * buffered to improve performance.
 */
#define DMA_ATTR_WRITE_COMBINE		(1UL << 2)
/*
 * DMA_ATTR_NON_CONSISTENT: Lets the platform to choose to return either
 * consistent or non-consistent memory as it sees fit.
 */
#define DMA_ATTR_NON_CONSISTENT		(1UL << 3)
/*
 * DMA_ATTR_NO_KERNEL_MAPPING: Lets the platform to avoid creating a kernel
 * virtual mapping for the allocated buffer.
 */
#define DMA_ATTR_NO_KERNEL_MAPPING	(1UL << 4)
/*
 * DMA_ATTR_SKIP_CPU_SYNC: Allows platform code to skip synchronization of
 * the CPU cache for the given buffer assuming that it has been already
 * transferred to 'device' domain.
 */
#define DMA_ATTR_SKIP_CPU_SYNC		(1UL << 5)
/*
 * DMA_ATTR_FORCE_CONTIGUOUS: Forces contiguous allocation of the buffer
 * in physical memory.
 */
#define DMA_ATTR_FORCE_CONTIGUOUS	(1UL << 6)
/*
 * DMA_ATTR_ALLOC_SINGLE_PAGES: This is a hint to the DMA-mapping subsystem
 * that it's probably not worth the time to try to allocate memory to in a way
 * that gives better TLB efficiency.
 */
#define DMA_ATTR_ALLOC_SINGLE_PAGES	(1UL << 7)
60 61 62 63 64
/*
 * DMA_ATTR_NO_WARN: This tells the DMA-mapping subsystem to suppress
 * allocation failure reports (similarly to __GFP_NOWARN).
 */
#define DMA_ATTR_NO_WARN	(1UL << 8)
65

66 67 68 69 70 71 72
/*
 * DMA_ATTR_PRIVILEGED: used to indicate that the buffer is fully
 * accessible at an elevated privilege level (and ideally inaccessible or
 * at least read-only at lesser-privileged levels).
 */
#define DMA_ATTR_PRIVILEGED		(1UL << 9)

73 74 75 76 77 78
/*
 * A dma_addr_t can hold any valid DMA or bus address for the platform.
 * It can be given to a device to use as a DMA source or target.  A CPU cannot
 * reference a dma_addr_t directly because there may be translation between
 * its physical address space and the bus address space.
 */
79
struct dma_map_ops {
80 81
	void* (*alloc)(struct device *dev, size_t size,
				dma_addr_t *dma_handle, gfp_t gfp,
82
				unsigned long attrs);
83 84
	void (*free)(struct device *dev, size_t size,
			      void *vaddr, dma_addr_t dma_handle,
85
			      unsigned long attrs);
86
	int (*mmap)(struct device *, struct vm_area_struct *,
87 88
			  void *, dma_addr_t, size_t,
			  unsigned long attrs);
89

90
	int (*get_sgtable)(struct device *dev, struct sg_table *sgt, void *,
91
			   dma_addr_t, size_t, unsigned long attrs);
92

93 94 95
	dma_addr_t (*map_page)(struct device *dev, struct page *page,
			       unsigned long offset, size_t size,
			       enum dma_data_direction dir,
96
			       unsigned long attrs);
97 98
	void (*unmap_page)(struct device *dev, dma_addr_t dma_handle,
			   size_t size, enum dma_data_direction dir,
99
			   unsigned long attrs);
100 101 102 103
	/*
	 * map_sg returns 0 on error and a value > 0 on success.
	 * It should never return a value < 0.
	 */
104 105
	int (*map_sg)(struct device *dev, struct scatterlist *sg,
		      int nents, enum dma_data_direction dir,
106
		      unsigned long attrs);
107 108 109
	void (*unmap_sg)(struct device *dev,
			 struct scatterlist *sg, int nents,
			 enum dma_data_direction dir,
110
			 unsigned long attrs);
111 112 113 114 115 116
	dma_addr_t (*map_resource)(struct device *dev, phys_addr_t phys_addr,
			       size_t size, enum dma_data_direction dir,
			       unsigned long attrs);
	void (*unmap_resource)(struct device *dev, dma_addr_t dma_handle,
			   size_t size, enum dma_data_direction dir,
			   unsigned long attrs);
117 118 119 120 121 122 123 124 125 126 127 128
	void (*sync_single_for_cpu)(struct device *dev,
				    dma_addr_t dma_handle, size_t size,
				    enum dma_data_direction dir);
	void (*sync_single_for_device)(struct device *dev,
				       dma_addr_t dma_handle, size_t size,
				       enum dma_data_direction dir);
	void (*sync_sg_for_cpu)(struct device *dev,
				struct scatterlist *sg, int nents,
				enum dma_data_direction dir);
	void (*sync_sg_for_device)(struct device *dev,
				   struct scatterlist *sg, int nents,
				   enum dma_data_direction dir);
129 130
	void (*cache_sync)(struct device *dev, void *vaddr, size_t size,
			enum dma_data_direction direction);
131 132
	int (*mapping_error)(struct device *dev, dma_addr_t dma_addr);
	int (*dma_supported)(struct device *dev, u64 mask);
133
	u64 (*get_required_mask)(struct device *dev);
134 135
};

136
extern const struct dma_map_ops dma_direct_ops;
137
extern const struct dma_map_ops dma_virt_ops;
138

139
#define DMA_BIT_MASK(n)	(((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
140

141 142
#define DMA_MASK_NONE	0x0ULL

143 144 145 146 147 148 149
static inline int valid_dma_direction(int dma_direction)
{
	return ((dma_direction == DMA_BIDIRECTIONAL) ||
		(dma_direction == DMA_TO_DEVICE) ||
		(dma_direction == DMA_FROM_DEVICE));
}

150 151 152 153 154
static inline int is_device_dma_capable(struct device *dev)
{
	return dev->dma_mask != NULL && *dev->dma_mask != DMA_MASK_NONE;
}

155 156 157 158 159
#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT
/*
 * These three functions are only for dma allocator.
 * Don't use them in device drivers.
 */
160
int dma_alloc_from_dev_coherent(struct device *dev, ssize_t size,
161
				       dma_addr_t *dma_handle, void **ret);
162
int dma_release_from_dev_coherent(struct device *dev, int order, void *vaddr);
163

164
int dma_mmap_from_dev_coherent(struct device *dev, struct vm_area_struct *vma,
165
			    void *cpu_addr, size_t size, int *ret);
166 167 168 169 170 171

void *dma_alloc_from_global_coherent(ssize_t size, dma_addr_t *dma_handle);
int dma_release_from_global_coherent(int order, void *vaddr);
int dma_mmap_from_global_coherent(struct vm_area_struct *vma, void *cpu_addr,
				  size_t size, int *ret);

172
#else
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193
#define dma_alloc_from_dev_coherent(dev, size, handle, ret) (0)
#define dma_release_from_dev_coherent(dev, order, vaddr) (0)
#define dma_mmap_from_dev_coherent(dev, vma, vaddr, order, ret) (0)

static inline void *dma_alloc_from_global_coherent(ssize_t size,
						   dma_addr_t *dma_handle)
{
	return NULL;
}

static inline int dma_release_from_global_coherent(int order, void *vaddr)
{
	return 0;
}

static inline int dma_mmap_from_global_coherent(struct vm_area_struct *vma,
						void *cpu_addr, size_t size,
						int *ret)
{
	return 0;
}
194 195
#endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */

196
#ifdef CONFIG_HAS_DMA
Linus Torvalds's avatar
Linus Torvalds committed
197
#include <asm/dma-mapping.h>
198 199 200 201 202 203 204
static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
{
	if (dev && dev->dma_ops)
		return dev->dma_ops;
	return get_arch_dma_ops(dev ? dev->bus : NULL);
}

205 206 207 208 209
static inline void set_dma_ops(struct device *dev,
			       const struct dma_map_ops *dma_ops)
{
	dev->dma_ops = dma_ops;
}
210
#else
211
/*
212 213 214 215
 * Define the dma api to allow compilation of dma dependent code.
 * Code that depends on the dma-mapping API needs to set 'depends on HAS_DMA'
 * in its Kconfig, unless it already depends on <something> || COMPILE_TEST,
 * where <something> guarantuees the availability of the dma-mapping API.
216
 */
217
static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
218
{
219
	return NULL;
220 221 222 223 224 225
}
#endif

static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr,
					      size_t size,
					      enum dma_data_direction dir,
226
					      unsigned long attrs)
227
{
228
	const struct dma_map_ops *ops = get_dma_ops(dev);
229 230 231
	dma_addr_t addr;

	BUG_ON(!valid_dma_direction(dir));
232
	debug_dma_map_single(dev, ptr, size);
233
	addr = ops->map_page(dev, virt_to_page(ptr),
234
			     offset_in_page(ptr), size,
235 236
			     dir, attrs);
	debug_dma_map_page(dev, virt_to_page(ptr),
237
			   offset_in_page(ptr), size,
238 239 240 241 242 243 244
			   dir, addr, true);
	return addr;
}

static inline void dma_unmap_single_attrs(struct device *dev, dma_addr_t addr,
					  size_t size,
					  enum dma_data_direction dir,
245
					  unsigned long attrs)
246
{
247
	const struct dma_map_ops *ops = get_dma_ops(dev);
248 249 250 251 252 253 254 255 256 257 258 259 260

	BUG_ON(!valid_dma_direction(dir));
	if (ops->unmap_page)
		ops->unmap_page(dev, addr, size, dir, attrs);
	debug_dma_unmap_page(dev, addr, size, dir, true);
}

/*
 * dma_maps_sg_attrs returns 0 on error and > 0 on success.
 * It should never return a value < 0.
 */
static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
				   int nents, enum dma_data_direction dir,
261
				   unsigned long attrs)
262
{
263
	const struct dma_map_ops *ops = get_dma_ops(dev);
264
	int ents;
265 266 267 268 269 270 271 272 273 274 275

	BUG_ON(!valid_dma_direction(dir));
	ents = ops->map_sg(dev, sg, nents, dir, attrs);
	BUG_ON(ents < 0);
	debug_dma_map_sg(dev, sg, nents, ents, dir);

	return ents;
}

static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
				      int nents, enum dma_data_direction dir,
276
				      unsigned long attrs)
277
{
278
	const struct dma_map_ops *ops = get_dma_ops(dev);
279 280 281 282 283 284 285

	BUG_ON(!valid_dma_direction(dir));
	debug_dma_unmap_sg(dev, sg, nents, dir);
	if (ops->unmap_sg)
		ops->unmap_sg(dev, sg, nents, dir, attrs);
}

286 287 288 289 290
static inline dma_addr_t dma_map_page_attrs(struct device *dev,
					    struct page *page,
					    size_t offset, size_t size,
					    enum dma_data_direction dir,
					    unsigned long attrs)
291
{
292
	const struct dma_map_ops *ops = get_dma_ops(dev);
293 294 295
	dma_addr_t addr;

	BUG_ON(!valid_dma_direction(dir));
296
	addr = ops->map_page(dev, page, offset, size, dir, attrs);
297 298 299 300 301
	debug_dma_map_page(dev, page, offset, size, dir, addr, false);

	return addr;
}

302 303 304 305
static inline void dma_unmap_page_attrs(struct device *dev,
					dma_addr_t addr, size_t size,
					enum dma_data_direction dir,
					unsigned long attrs)
306
{
307
	const struct dma_map_ops *ops = get_dma_ops(dev);
308 309 310

	BUG_ON(!valid_dma_direction(dir));
	if (ops->unmap_page)
311
		ops->unmap_page(dev, addr, size, dir, attrs);
312 313 314
	debug_dma_unmap_page(dev, addr, size, dir, false);
}

315 316 317 318 319 320
static inline dma_addr_t dma_map_resource(struct device *dev,
					  phys_addr_t phys_addr,
					  size_t size,
					  enum dma_data_direction dir,
					  unsigned long attrs)
{
321
	const struct dma_map_ops *ops = get_dma_ops(dev);
322 323 324 325 326
	dma_addr_t addr;

	BUG_ON(!valid_dma_direction(dir));

	/* Don't allow RAM to be mapped */
327
	BUG_ON(pfn_valid(PHYS_PFN(phys_addr)));
328 329 330 331 332 333 334 335 336 337 338 339 340 341

	addr = phys_addr;
	if (ops->map_resource)
		addr = ops->map_resource(dev, phys_addr, size, dir, attrs);

	debug_dma_map_resource(dev, phys_addr, size, dir, addr);

	return addr;
}

static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr,
				      size_t size, enum dma_data_direction dir,
				      unsigned long attrs)
{
342
	const struct dma_map_ops *ops = get_dma_ops(dev);
343 344 345 346 347 348 349

	BUG_ON(!valid_dma_direction(dir));
	if (ops->unmap_resource)
		ops->unmap_resource(dev, addr, size, dir, attrs);
	debug_dma_unmap_resource(dev, addr, size, dir);
}

350 351 352 353
static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
					   size_t size,
					   enum dma_data_direction dir)
{
354
	const struct dma_map_ops *ops = get_dma_ops(dev);
355 356 357 358 359 360 361 362 363 364 365

	BUG_ON(!valid_dma_direction(dir));
	if (ops->sync_single_for_cpu)
		ops->sync_single_for_cpu(dev, addr, size, dir);
	debug_dma_sync_single_for_cpu(dev, addr, size, dir);
}

static inline void dma_sync_single_for_device(struct device *dev,
					      dma_addr_t addr, size_t size,
					      enum dma_data_direction dir)
{
366
	const struct dma_map_ops *ops = get_dma_ops(dev);
367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405

	BUG_ON(!valid_dma_direction(dir));
	if (ops->sync_single_for_device)
		ops->sync_single_for_device(dev, addr, size, dir);
	debug_dma_sync_single_for_device(dev, addr, size, dir);
}

static inline void dma_sync_single_range_for_cpu(struct device *dev,
						 dma_addr_t addr,
						 unsigned long offset,
						 size_t size,
						 enum dma_data_direction dir)
{
	const struct dma_map_ops *ops = get_dma_ops(dev);

	BUG_ON(!valid_dma_direction(dir));
	if (ops->sync_single_for_cpu)
		ops->sync_single_for_cpu(dev, addr + offset, size, dir);
	debug_dma_sync_single_range_for_cpu(dev, addr, offset, size, dir);
}

static inline void dma_sync_single_range_for_device(struct device *dev,
						    dma_addr_t addr,
						    unsigned long offset,
						    size_t size,
						    enum dma_data_direction dir)
{
	const struct dma_map_ops *ops = get_dma_ops(dev);

	BUG_ON(!valid_dma_direction(dir));
	if (ops->sync_single_for_device)
		ops->sync_single_for_device(dev, addr + offset, size, dir);
	debug_dma_sync_single_range_for_device(dev, addr, offset, size, dir);
}

static inline void
dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
		    int nelems, enum dma_data_direction dir)
{
406
	const struct dma_map_ops *ops = get_dma_ops(dev);
407 408 409 410 411 412 413 414 415 416 417

	BUG_ON(!valid_dma_direction(dir));
	if (ops->sync_sg_for_cpu)
		ops->sync_sg_for_cpu(dev, sg, nelems, dir);
	debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
}

static inline void
dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
		       int nelems, enum dma_data_direction dir)
{
418
	const struct dma_map_ops *ops = get_dma_ops(dev);
419 420 421 422 423 424 425 426

	BUG_ON(!valid_dma_direction(dir));
	if (ops->sync_sg_for_device)
		ops->sync_sg_for_device(dev, sg, nelems, dir);
	debug_dma_sync_sg_for_device(dev, sg, nelems, dir);

}

427 428 429 430
#define dma_map_single(d, a, s, r) dma_map_single_attrs(d, a, s, r, 0)
#define dma_unmap_single(d, a, s, r) dma_unmap_single_attrs(d, a, s, r, 0)
#define dma_map_sg(d, s, n, r) dma_map_sg_attrs(d, s, n, r, 0)
#define dma_unmap_sg(d, s, n, r) dma_unmap_sg_attrs(d, s, n, r, 0)
431 432
#define dma_map_page(d, p, o, s, r) dma_map_page_attrs(d, p, o, s, r, 0)
#define dma_unmap_page(d, a, s, r) dma_unmap_page_attrs(d, a, s, r, 0)
433

434 435 436 437 438 439 440 441 442 443 444
static inline void
dma_cache_sync(struct device *dev, void *vaddr, size_t size,
		enum dma_data_direction dir)
{
	const struct dma_map_ops *ops = get_dma_ops(dev);

	BUG_ON(!valid_dma_direction(dir));
	if (ops->cache_sync)
		ops->cache_sync(dev, vaddr, size, dir);
}

445
extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
446 447
		void *cpu_addr, dma_addr_t dma_addr, size_t size,
		unsigned long attrs);
448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472

void *dma_common_contiguous_remap(struct page *page, size_t size,
			unsigned long vm_flags,
			pgprot_t prot, const void *caller);

void *dma_common_pages_remap(struct page **pages, size_t size,
			unsigned long vm_flags, pgprot_t prot,
			const void *caller);
void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags);

/**
 * dma_mmap_attrs - map a coherent DMA allocation into user space
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @vma: vm_area_struct describing requested user mapping
 * @cpu_addr: kernel CPU-view address returned from dma_alloc_attrs
 * @handle: device-view address returned from dma_alloc_attrs
 * @size: size of memory originally requested in dma_alloc_attrs
 * @attrs: attributes of mapping properties requested in dma_alloc_attrs
 *
 * Map a coherent DMA buffer previously allocated by dma_alloc_attrs
 * into user space.  The coherent DMA buffer must not be freed by the
 * driver until the user space mapping has been released.
 */
static inline int
dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, void *cpu_addr,
473
	       dma_addr_t dma_addr, size_t size, unsigned long attrs)
474
{
475
	const struct dma_map_ops *ops = get_dma_ops(dev);
476 477 478
	BUG_ON(!ops);
	if (ops->mmap)
		return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
479
	return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
480 481
}

482
#define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, 0)
483 484

int
485 486
dma_common_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr,
		dma_addr_t dma_addr, size_t size, unsigned long attrs);
487 488 489

static inline int
dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, void *cpu_addr,
490 491
		      dma_addr_t dma_addr, size_t size,
		      unsigned long attrs)
492
{
493
	const struct dma_map_ops *ops = get_dma_ops(dev);
494 495 496 497
	BUG_ON(!ops);
	if (ops->get_sgtable)
		return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size,
					attrs);
498 499
	return dma_common_get_sgtable(dev, sgt, cpu_addr, dma_addr, size,
			attrs);
500 501
}

502
#define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, 0)
503 504

#ifndef arch_dma_alloc_attrs
505
#define arch_dma_alloc_attrs(dev)	(true)
506 507 508 509
#endif

static inline void *dma_alloc_attrs(struct device *dev, size_t size,
				       dma_addr_t *dma_handle, gfp_t flag,
510
				       unsigned long attrs)
511
{
512
	const struct dma_map_ops *ops = get_dma_ops(dev);
513 514 515
	void *cpu_addr;

	BUG_ON(!ops);
516
	WARN_ON_ONCE(dev && !dev->coherent_dma_mask);
517

518
	if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr))
519 520
		return cpu_addr;

521 522
	/* let the implementation decide on the zone to allocate from: */
	flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
523

524
	if (!arch_dma_alloc_attrs(&dev))
525 526 527 528 529 530 531 532 533 534 535
		return NULL;
	if (!ops->alloc)
		return NULL;

	cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
	debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr);
	return cpu_addr;
}

static inline void dma_free_attrs(struct device *dev, size_t size,
				     void *cpu_addr, dma_addr_t dma_handle,
536
				     unsigned long attrs)
537
{
538
	const struct dma_map_ops *ops = get_dma_ops(dev);
539 540 541

	BUG_ON(!ops);

542
	if (dma_release_from_dev_coherent(dev, get_order(size), cpu_addr))
543
		return;
544 545 546 547 548 549 550 551
	/*
	 * On non-coherent platforms which implement DMA-coherent buffers via
	 * non-cacheable remaps, ops->free() may call vunmap(). Thus getting
	 * this far in IRQ context is a) at risk of a BUG_ON() or trying to
	 * sleep on some machines, and b) an indication that the driver is
	 * probably misusing the coherent API anyway.
	 */
	WARN_ON(irqs_disabled());
552

553
	if (!ops->free || !cpu_addr)
554 555 556 557 558 559 560
		return;

	debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);
	ops->free(dev, size, cpu_addr, dma_handle, attrs);
}

static inline void *dma_alloc_coherent(struct device *dev, size_t size,
561
		dma_addr_t *dma_handle, gfp_t gfp)
562
{
563 564 565

	return dma_alloc_attrs(dev, size, dma_handle, gfp,
			(gfp & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0);
566 567 568 569 570
}

static inline void dma_free_coherent(struct device *dev, size_t size,
		void *cpu_addr, dma_addr_t dma_handle)
{
571
	return dma_free_attrs(dev, size, cpu_addr, dma_handle, 0);
572 573 574 575
}

static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
{
576
	const struct dma_map_ops *ops = get_dma_ops(dev);
577

578 579 580
	debug_dma_mapping_error(dev, dma_addr);
	if (ops->mapping_error)
		return ops->mapping_error(dev, dma_addr);
581 582 583
	return 0;
}

584 585 586 587 588 589
static inline void dma_check_mask(struct device *dev, u64 mask)
{
	if (sme_active() && (mask < (((u64)sme_get_me_mask() << 1) - 1)))
		dev_warn(dev, "SME is active, device will require DMA bounce buffers\n");
}

590 591
static inline int dma_supported(struct device *dev, u64 mask)
{
592
	const struct dma_map_ops *ops = get_dma_ops(dev);
593 594 595 596 597 598 599 600 601 602 603 604 605

	if (!ops)
		return 0;
	if (!ops->dma_supported)
		return 1;
	return ops->dma_supported(dev, mask);
}

#ifndef HAVE_ARCH_DMA_SET_MASK
static inline int dma_set_mask(struct device *dev, u64 mask)
{
	if (!dev->dma_mask || !dma_supported(dev, mask))
		return -EIO;
606 607 608

	dma_check_mask(dev, mask);

609 610 611
	*dev->dma_mask = mask;
	return 0;
}
612
#endif
Linus Torvalds's avatar
Linus Torvalds committed
613

614 615
static inline u64 dma_get_mask(struct device *dev)
{
616
	if (dev && dev->dma_mask && *dev->dma_mask)
617
		return *dev->dma_mask;
618
	return DMA_BIT_MASK(32);
619 620
}

621
#ifdef CONFIG_ARCH_HAS_DMA_SET_COHERENT_MASK
622 623
int dma_set_coherent_mask(struct device *dev, u64 mask);
#else
624 625 626 627
static inline int dma_set_coherent_mask(struct device *dev, u64 mask)
{
	if (!dma_supported(dev, mask))
		return -EIO;
628 629 630

	dma_check_mask(dev, mask);

631 632 633
	dev->coherent_dma_mask = mask;
	return 0;
}
634
#endif
635

636 637 638 639 640 641 642 643 644 645 646 647 648 649
/*
 * Set both the DMA mask and the coherent DMA mask to the same thing.
 * Note that we don't check the return value from dma_set_coherent_mask()
 * as the DMA API guarantees that the coherent DMA mask can be set to
 * the same or smaller than the streaming DMA mask.
 */
static inline int dma_set_mask_and_coherent(struct device *dev, u64 mask)
{
	int rc = dma_set_mask(dev, mask);
	if (rc == 0)
		dma_set_coherent_mask(dev, mask);
	return rc;
}

650 651 652 653 654 655 656 657 658 659
/*
 * Similar to the above, except it deals with the case where the device
 * does not have dev->dma_mask appropriately setup.
 */
static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask)
{
	dev->dma_mask = &dev->coherent_dma_mask;
	return dma_set_mask_and_coherent(dev, mask);
}

Linus Torvalds's avatar
Linus Torvalds committed
660 661
extern u64 dma_get_required_mask(struct device *dev);

662
#ifndef arch_setup_dma_ops
663
static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
664
				      u64 size, const struct iommu_ops *iommu,
665 666 667 668
				      bool coherent) { }
#endif

#ifndef arch_teardown_dma_ops
669
static inline void arch_teardown_dma_ops(struct device *dev) { }
670 671
#endif

672 673
static inline unsigned int dma_get_max_seg_size(struct device *dev)
{
674 675 676
	if (dev->dma_parms && dev->dma_parms->max_segment_size)
		return dev->dma_parms->max_segment_size;
	return SZ_64K;
677 678 679 680 681 682 683 684
}

static inline unsigned int dma_set_max_seg_size(struct device *dev,
						unsigned int size)
{
	if (dev->dma_parms) {
		dev->dma_parms->max_segment_size = size;
		return 0;
685 686
	}
	return -EIO;
687 688
}

689 690
static inline unsigned long dma_get_seg_boundary(struct device *dev)
{
691 692 693
	if (dev->dma_parms && dev->dma_parms->segment_boundary_mask)
		return dev->dma_parms->segment_boundary_mask;
	return DMA_BIT_MASK(32);
694 695 696 697 698 699 700
}

static inline int dma_set_seg_boundary(struct device *dev, unsigned long mask)
{
	if (dev->dma_parms) {
		dev->dma_parms->segment_boundary_mask = mask;
		return 0;
701 702
	}
	return -EIO;
703 704
}

705 706 707
#ifndef dma_max_pfn
static inline unsigned long dma_max_pfn(struct device *dev)
{
708
	return (*dev->dma_mask >> PAGE_SHIFT) + dev->dma_pfn_offset;
709 710 711
}
#endif

712 713 714
static inline void *dma_zalloc_coherent(struct device *dev, size_t size,
					dma_addr_t *dma_handle, gfp_t flag)
{
715 716
	void *ret = dma_alloc_coherent(dev, size, dma_handle,
				       flag | __GFP_ZERO);
717 718 719
	return ret;
}

720 721 722 723 724 725 726 727
static inline int dma_get_cache_alignment(void)
{
#ifdef ARCH_DMA_MINALIGN
	return ARCH_DMA_MINALIGN;
#endif
	return 1;
}

Linus Torvalds's avatar
Linus Torvalds committed
728
/* flags for the coherent memory api */
729
#define DMA_MEMORY_EXCLUSIVE		0x01
Linus Torvalds's avatar
Linus Torvalds committed
730

731 732 733 734 735 736 737
#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT
int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
				dma_addr_t device_addr, size_t size, int flags);
void dma_release_declared_memory(struct device *dev);
void *dma_mark_declared_memory_occupied(struct device *dev,
					dma_addr_t device_addr, size_t size);
#else
Linus Torvalds's avatar
Linus Torvalds committed
738
static inline int
739
dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
Linus Torvalds's avatar
Linus Torvalds committed
740 741
			    dma_addr_t device_addr, size_t size, int flags)
{
742
	return -ENOSYS;
Linus Torvalds's avatar
Linus Torvalds committed
743 744 745 746 747 748 749 750 751 752 753 754 755
}

static inline void
dma_release_declared_memory(struct device *dev)
{
}

static inline void *
dma_mark_declared_memory_occupied(struct device *dev,
				  dma_addr_t device_addr, size_t size)
{
	return ERR_PTR(-EBUSY);
}
756
#endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */
Linus Torvalds's avatar
Linus Torvalds committed
757

758 759 760
/*
 * Managed DMA API
 */
761
#ifdef CONFIG_HAS_DMA
762 763 764 765
extern void *dmam_alloc_coherent(struct device *dev, size_t size,
				 dma_addr_t *dma_handle, gfp_t gfp);
extern void dmam_free_coherent(struct device *dev, size_t size, void *vaddr,
			       dma_addr_t dma_handle);
766 767 768 769 770 771 772 773
#else /* !CONFIG_HAS_DMA */
static inline void *dmam_alloc_coherent(struct device *dev, size_t size,
					dma_addr_t *dma_handle, gfp_t gfp)
{ return NULL; }
static inline void dmam_free_coherent(struct device *dev, size_t size,
				      void *vaddr, dma_addr_t dma_handle) { }
#endif /* !CONFIG_HAS_DMA */

774 775 776
extern void *dmam_alloc_attrs(struct device *dev, size_t size,
			      dma_addr_t *dma_handle, gfp_t gfp,
			      unsigned long attrs);
777
#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT
778 779
extern int dmam_declare_coherent_memory(struct device *dev,
					phys_addr_t phys_addr,
780 781 782
					dma_addr_t device_addr, size_t size,
					int flags);
extern void dmam_release_declared_memory(struct device *dev);
783
#else /* CONFIG_HAVE_GENERIC_DMA_COHERENT */
784
static inline int dmam_declare_coherent_memory(struct device *dev,
785
				phys_addr_t phys_addr, dma_addr_t device_addr,
786 787 788 789
				size_t size, gfp_t gfp)
{
	return 0;
}
Linus Torvalds's avatar
Linus Torvalds committed
790

791 792 793
static inline void dmam_release_declared_memory(struct device *dev)
{
}
794
#endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */
Linus Torvalds's avatar
Linus Torvalds committed
795

796 797
static inline void *dma_alloc_wc(struct device *dev, size_t size,
				 dma_addr_t *dma_addr, gfp_t gfp)
798
{
799
	unsigned long attrs = DMA_ATTR_WRITE_COMBINE;
800 801 802 803 804

	if (gfp & __GFP_NOWARN)
		attrs |= DMA_ATTR_NO_WARN;

	return dma_alloc_attrs(dev, size, dma_addr, gfp, attrs);
805
}
806 807 808
#ifndef dma_alloc_writecombine
#define dma_alloc_writecombine dma_alloc_wc
#endif
809

810 811
static inline void dma_free_wc(struct device *dev, size_t size,
			       void *cpu_addr, dma_addr_t dma_addr)
812
{
813 814
	return dma_free_attrs(dev, size, cpu_addr, dma_addr,
			      DMA_ATTR_WRITE_COMBINE);
815
}
816 817 818
#ifndef dma_free_writecombine
#define dma_free_writecombine dma_free_wc
#endif
819

820 821 822 823
static inline int dma_mmap_wc(struct device *dev,
			      struct vm_area_struct *vma,
			      void *cpu_addr, dma_addr_t dma_addr,
			      size_t size)
824
{
825 826
	return dma_mmap_attrs(dev, vma, cpu_addr, dma_addr, size,
			      DMA_ATTR_WRITE_COMBINE);
827
}
828 829 830
#ifndef dma_mmap_writecombine
#define dma_mmap_writecombine dma_mmap_wc
#endif
831

832
#ifdef CONFIG_NEED_DMA_MAP_STATE
833 834 835 836 837 838 839 840 841 842 843 844 845 846 847
#define DEFINE_DMA_UNMAP_ADDR(ADDR_NAME)        dma_addr_t ADDR_NAME
#define DEFINE_DMA_UNMAP_LEN(LEN_NAME)          __u32 LEN_NAME
#define dma_unmap_addr(PTR, ADDR_NAME)           ((PTR)->ADDR_NAME)
#define dma_unmap_addr_set(PTR, ADDR_NAME, VAL)  (((PTR)->ADDR_NAME) = (VAL))
#define dma_unmap_len(PTR, LEN_NAME)             ((PTR)->LEN_NAME)
#define dma_unmap_len_set(PTR, LEN_NAME, VAL)    (((PTR)->LEN_NAME) = (VAL))
#else
#define DEFINE_DMA_UNMAP_ADDR(ADDR_NAME)
#define DEFINE_DMA_UNMAP_LEN(LEN_NAME)
#define dma_unmap_addr(PTR, ADDR_NAME)           (0)
#define dma_unmap_addr_set(PTR, ADDR_NAME, VAL)  do { } while (0)
#define dma_unmap_len(PTR, LEN_NAME)             (0)
#define dma_unmap_len_set(PTR, LEN_NAME, VAL)    do { } while (0)
#endif

848
#endif