dma-mapping.h 25.5 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
2 3
#ifndef _LINUX_DMA_MAPPING_H
#define _LINUX_DMA_MAPPING_H
Linus Torvalds's avatar
Linus Torvalds committed
4

5
#include <linux/sizes.h>
6
#include <linux/string.h>
Linus Torvalds's avatar
Linus Torvalds committed
7 8
#include <linux/device.h>
#include <linux/err.h>
9
#include <linux/dma-debug.h>
10
#include <linux/dma-direction.h>
11
#include <linux/scatterlist.h>
12
#include <linux/bug.h>
13
#include <linux/mem_encrypt.h>
Linus Torvalds's avatar
Linus Torvalds committed
14

15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
/**
 * List of possible attributes associated with a DMA mapping. The semantics
 * of each attribute should be defined in Documentation/DMA-attributes.txt.
 *
 * DMA_ATTR_WRITE_BARRIER: DMA to a memory region with this attribute
 * forces all pending DMA writes to complete.
 */
#define DMA_ATTR_WRITE_BARRIER		(1UL << 0)
/*
 * DMA_ATTR_WEAK_ORDERING: Specifies that reads and writes to the mapping
 * may be weakly ordered, that is that reads and writes may pass each other.
 */
#define DMA_ATTR_WEAK_ORDERING		(1UL << 1)
/*
 * DMA_ATTR_WRITE_COMBINE: Specifies that writes to the mapping may be
 * buffered to improve performance.
 */
#define DMA_ATTR_WRITE_COMBINE		(1UL << 2)
/*
 * DMA_ATTR_NON_CONSISTENT: Lets the platform to choose to return either
 * consistent or non-consistent memory as it sees fit.
 */
#define DMA_ATTR_NON_CONSISTENT		(1UL << 3)
/*
 * DMA_ATTR_NO_KERNEL_MAPPING: Lets the platform to avoid creating a kernel
 * virtual mapping for the allocated buffer.
 */
#define DMA_ATTR_NO_KERNEL_MAPPING	(1UL << 4)
/*
 * DMA_ATTR_SKIP_CPU_SYNC: Allows platform code to skip synchronization of
 * the CPU cache for the given buffer assuming that it has been already
 * transferred to 'device' domain.
 */
#define DMA_ATTR_SKIP_CPU_SYNC		(1UL << 5)
/*
 * DMA_ATTR_FORCE_CONTIGUOUS: Forces contiguous allocation of the buffer
 * in physical memory.
 */
#define DMA_ATTR_FORCE_CONTIGUOUS	(1UL << 6)
/*
 * DMA_ATTR_ALLOC_SINGLE_PAGES: This is a hint to the DMA-mapping subsystem
 * that it's probably not worth the time to try to allocate memory to in a way
 * that gives better TLB efficiency.
 */
#define DMA_ATTR_ALLOC_SINGLE_PAGES	(1UL << 7)
60 61 62 63 64
/*
 * DMA_ATTR_NO_WARN: This tells the DMA-mapping subsystem to suppress
 * allocation failure reports (similarly to __GFP_NOWARN).
 */
#define DMA_ATTR_NO_WARN	(1UL << 8)
65

66 67 68 69 70 71 72
/*
 * DMA_ATTR_PRIVILEGED: used to indicate that the buffer is fully
 * accessible at an elevated privilege level (and ideally inaccessible or
 * at least read-only at lesser-privileged levels).
 */
#define DMA_ATTR_PRIVILEGED		(1UL << 9)

73 74 75 76 77 78
/*
 * A dma_addr_t can hold any valid DMA or bus address for the platform.
 * It can be given to a device to use as a DMA source or target.  A CPU cannot
 * reference a dma_addr_t directly because there may be translation between
 * its physical address space and the bus address space.
 */
79
struct dma_map_ops {
80 81
	void* (*alloc)(struct device *dev, size_t size,
				dma_addr_t *dma_handle, gfp_t gfp,
82
				unsigned long attrs);
83 84
	void (*free)(struct device *dev, size_t size,
			      void *vaddr, dma_addr_t dma_handle,
85
			      unsigned long attrs);
86
	int (*mmap)(struct device *, struct vm_area_struct *,
87 88
			  void *, dma_addr_t, size_t,
			  unsigned long attrs);
89

90
	int (*get_sgtable)(struct device *dev, struct sg_table *sgt, void *,
91
			   dma_addr_t, size_t, unsigned long attrs);
92

93 94 95
	dma_addr_t (*map_page)(struct device *dev, struct page *page,
			       unsigned long offset, size_t size,
			       enum dma_data_direction dir,
96
			       unsigned long attrs);
97 98
	void (*unmap_page)(struct device *dev, dma_addr_t dma_handle,
			   size_t size, enum dma_data_direction dir,
99
			   unsigned long attrs);
100 101 102 103
	/*
	 * map_sg returns 0 on error and a value > 0 on success.
	 * It should never return a value < 0.
	 */
104 105
	int (*map_sg)(struct device *dev, struct scatterlist *sg,
		      int nents, enum dma_data_direction dir,
106
		      unsigned long attrs);
107 108 109
	void (*unmap_sg)(struct device *dev,
			 struct scatterlist *sg, int nents,
			 enum dma_data_direction dir,
110
			 unsigned long attrs);
111 112 113 114 115 116
	dma_addr_t (*map_resource)(struct device *dev, phys_addr_t phys_addr,
			       size_t size, enum dma_data_direction dir,
			       unsigned long attrs);
	void (*unmap_resource)(struct device *dev, dma_addr_t dma_handle,
			   size_t size, enum dma_data_direction dir,
			   unsigned long attrs);
117 118 119 120 121 122 123 124 125 126 127 128
	void (*sync_single_for_cpu)(struct device *dev,
				    dma_addr_t dma_handle, size_t size,
				    enum dma_data_direction dir);
	void (*sync_single_for_device)(struct device *dev,
				       dma_addr_t dma_handle, size_t size,
				       enum dma_data_direction dir);
	void (*sync_sg_for_cpu)(struct device *dev,
				struct scatterlist *sg, int nents,
				enum dma_data_direction dir);
	void (*sync_sg_for_device)(struct device *dev,
				   struct scatterlist *sg, int nents,
				   enum dma_data_direction dir);
129 130
	void (*cache_sync)(struct device *dev, void *vaddr, size_t size,
			enum dma_data_direction direction);
131 132
	int (*mapping_error)(struct device *dev, dma_addr_t dma_addr);
	int (*dma_supported)(struct device *dev, u64 mask);
133 134 135
#ifdef ARCH_HAS_DMA_GET_REQUIRED_MASK
	u64 (*get_required_mask)(struct device *dev);
#endif
136 137
};

138
extern const struct dma_map_ops dma_direct_ops;
139
extern const struct dma_map_ops dma_noncoherent_ops;
140
extern const struct dma_map_ops dma_virt_ops;
141

142
#define DMA_BIT_MASK(n)	(((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
143

144 145
#define DMA_MASK_NONE	0x0ULL

146 147 148 149 150 151 152
static inline int valid_dma_direction(int dma_direction)
{
	return ((dma_direction == DMA_BIDIRECTIONAL) ||
		(dma_direction == DMA_TO_DEVICE) ||
		(dma_direction == DMA_FROM_DEVICE));
}

153 154 155 156 157
static inline int is_device_dma_capable(struct device *dev)
{
	return dev->dma_mask != NULL && *dev->dma_mask != DMA_MASK_NONE;
}

158 159 160 161 162
#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT
/*
 * These three functions are only for dma allocator.
 * Don't use them in device drivers.
 */
163
int dma_alloc_from_dev_coherent(struct device *dev, ssize_t size,
164
				       dma_addr_t *dma_handle, void **ret);
165
int dma_release_from_dev_coherent(struct device *dev, int order, void *vaddr);
166

167
int dma_mmap_from_dev_coherent(struct device *dev, struct vm_area_struct *vma,
168
			    void *cpu_addr, size_t size, int *ret);
169 170 171 172 173 174

void *dma_alloc_from_global_coherent(ssize_t size, dma_addr_t *dma_handle);
int dma_release_from_global_coherent(int order, void *vaddr);
int dma_mmap_from_global_coherent(struct vm_area_struct *vma, void *cpu_addr,
				  size_t size, int *ret);

175
#else
176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196
#define dma_alloc_from_dev_coherent(dev, size, handle, ret) (0)
#define dma_release_from_dev_coherent(dev, order, vaddr) (0)
#define dma_mmap_from_dev_coherent(dev, vma, vaddr, order, ret) (0)

static inline void *dma_alloc_from_global_coherent(ssize_t size,
						   dma_addr_t *dma_handle)
{
	return NULL;
}

static inline int dma_release_from_global_coherent(int order, void *vaddr)
{
	return 0;
}

static inline int dma_mmap_from_global_coherent(struct vm_area_struct *vma,
						void *cpu_addr, size_t size,
						int *ret)
{
	return 0;
}
197 198
#endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */

199
#ifdef CONFIG_HAS_DMA
Linus Torvalds's avatar
Linus Torvalds committed
200
#include <asm/dma-mapping.h>
201 202 203 204 205 206 207
static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
{
	if (dev && dev->dma_ops)
		return dev->dma_ops;
	return get_arch_dma_ops(dev ? dev->bus : NULL);
}

208 209 210 211 212
static inline void set_dma_ops(struct device *dev,
			       const struct dma_map_ops *dma_ops)
{
	dev->dma_ops = dma_ops;
}
213
#else
214
/*
215 216 217 218
 * Define the dma api to allow compilation of dma dependent code.
 * Code that depends on the dma-mapping API needs to set 'depends on HAS_DMA'
 * in its Kconfig, unless it already depends on <something> || COMPILE_TEST,
 * where <something> guarantuees the availability of the dma-mapping API.
219
 */
220
static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
221
{
222
	return NULL;
223 224 225 226 227 228
}
#endif

static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr,
					      size_t size,
					      enum dma_data_direction dir,
229
					      unsigned long attrs)
230
{
231
	const struct dma_map_ops *ops = get_dma_ops(dev);
232 233 234 235
	dma_addr_t addr;

	BUG_ON(!valid_dma_direction(dir));
	addr = ops->map_page(dev, virt_to_page(ptr),
236
			     offset_in_page(ptr), size,
237 238
			     dir, attrs);
	debug_dma_map_page(dev, virt_to_page(ptr),
239
			   offset_in_page(ptr), size,
240 241 242 243 244 245 246
			   dir, addr, true);
	return addr;
}

static inline void dma_unmap_single_attrs(struct device *dev, dma_addr_t addr,
					  size_t size,
					  enum dma_data_direction dir,
247
					  unsigned long attrs)
248
{
249
	const struct dma_map_ops *ops = get_dma_ops(dev);
250 251 252 253 254 255 256 257 258 259 260 261 262

	BUG_ON(!valid_dma_direction(dir));
	if (ops->unmap_page)
		ops->unmap_page(dev, addr, size, dir, attrs);
	debug_dma_unmap_page(dev, addr, size, dir, true);
}

/*
 * dma_maps_sg_attrs returns 0 on error and > 0 on success.
 * It should never return a value < 0.
 */
static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
				   int nents, enum dma_data_direction dir,
263
				   unsigned long attrs)
264
{
265
	const struct dma_map_ops *ops = get_dma_ops(dev);
266
	int ents;
267 268 269 270 271 272 273 274 275 276 277

	BUG_ON(!valid_dma_direction(dir));
	ents = ops->map_sg(dev, sg, nents, dir, attrs);
	BUG_ON(ents < 0);
	debug_dma_map_sg(dev, sg, nents, ents, dir);

	return ents;
}

static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
				      int nents, enum dma_data_direction dir,
278
				      unsigned long attrs)
279
{
280
	const struct dma_map_ops *ops = get_dma_ops(dev);
281 282 283 284 285 286 287

	BUG_ON(!valid_dma_direction(dir));
	debug_dma_unmap_sg(dev, sg, nents, dir);
	if (ops->unmap_sg)
		ops->unmap_sg(dev, sg, nents, dir, attrs);
}

288 289 290 291 292
static inline dma_addr_t dma_map_page_attrs(struct device *dev,
					    struct page *page,
					    size_t offset, size_t size,
					    enum dma_data_direction dir,
					    unsigned long attrs)
293
{
294
	const struct dma_map_ops *ops = get_dma_ops(dev);
295 296 297
	dma_addr_t addr;

	BUG_ON(!valid_dma_direction(dir));
298
	addr = ops->map_page(dev, page, offset, size, dir, attrs);
299 300 301 302 303
	debug_dma_map_page(dev, page, offset, size, dir, addr, false);

	return addr;
}

304 305 306 307
static inline void dma_unmap_page_attrs(struct device *dev,
					dma_addr_t addr, size_t size,
					enum dma_data_direction dir,
					unsigned long attrs)
308
{
309
	const struct dma_map_ops *ops = get_dma_ops(dev);
310 311 312

	BUG_ON(!valid_dma_direction(dir));
	if (ops->unmap_page)
313
		ops->unmap_page(dev, addr, size, dir, attrs);
314 315 316
	debug_dma_unmap_page(dev, addr, size, dir, false);
}

317 318 319 320 321 322
static inline dma_addr_t dma_map_resource(struct device *dev,
					  phys_addr_t phys_addr,
					  size_t size,
					  enum dma_data_direction dir,
					  unsigned long attrs)
{
323
	const struct dma_map_ops *ops = get_dma_ops(dev);
324 325 326 327 328
	dma_addr_t addr;

	BUG_ON(!valid_dma_direction(dir));

	/* Don't allow RAM to be mapped */
329
	BUG_ON(pfn_valid(PHYS_PFN(phys_addr)));
330 331 332 333 334 335 336 337 338 339 340 341 342 343

	addr = phys_addr;
	if (ops->map_resource)
		addr = ops->map_resource(dev, phys_addr, size, dir, attrs);

	debug_dma_map_resource(dev, phys_addr, size, dir, addr);

	return addr;
}

static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr,
				      size_t size, enum dma_data_direction dir,
				      unsigned long attrs)
{
344
	const struct dma_map_ops *ops = get_dma_ops(dev);
345 346 347 348 349 350 351

	BUG_ON(!valid_dma_direction(dir));
	if (ops->unmap_resource)
		ops->unmap_resource(dev, addr, size, dir, attrs);
	debug_dma_unmap_resource(dev, addr, size, dir);
}

352 353 354 355
static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
					   size_t size,
					   enum dma_data_direction dir)
{
356
	const struct dma_map_ops *ops = get_dma_ops(dev);
357 358 359 360 361 362 363 364 365 366 367

	BUG_ON(!valid_dma_direction(dir));
	if (ops->sync_single_for_cpu)
		ops->sync_single_for_cpu(dev, addr, size, dir);
	debug_dma_sync_single_for_cpu(dev, addr, size, dir);
}

static inline void dma_sync_single_for_device(struct device *dev,
					      dma_addr_t addr, size_t size,
					      enum dma_data_direction dir)
{
368
	const struct dma_map_ops *ops = get_dma_ops(dev);
369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407

	BUG_ON(!valid_dma_direction(dir));
	if (ops->sync_single_for_device)
		ops->sync_single_for_device(dev, addr, size, dir);
	debug_dma_sync_single_for_device(dev, addr, size, dir);
}

static inline void dma_sync_single_range_for_cpu(struct device *dev,
						 dma_addr_t addr,
						 unsigned long offset,
						 size_t size,
						 enum dma_data_direction dir)
{
	const struct dma_map_ops *ops = get_dma_ops(dev);

	BUG_ON(!valid_dma_direction(dir));
	if (ops->sync_single_for_cpu)
		ops->sync_single_for_cpu(dev, addr + offset, size, dir);
	debug_dma_sync_single_range_for_cpu(dev, addr, offset, size, dir);
}

static inline void dma_sync_single_range_for_device(struct device *dev,
						    dma_addr_t addr,
						    unsigned long offset,
						    size_t size,
						    enum dma_data_direction dir)
{
	const struct dma_map_ops *ops = get_dma_ops(dev);

	BUG_ON(!valid_dma_direction(dir));
	if (ops->sync_single_for_device)
		ops->sync_single_for_device(dev, addr + offset, size, dir);
	debug_dma_sync_single_range_for_device(dev, addr, offset, size, dir);
}

static inline void
dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
		    int nelems, enum dma_data_direction dir)
{
408
	const struct dma_map_ops *ops = get_dma_ops(dev);
409 410 411 412 413 414 415 416 417 418 419

	BUG_ON(!valid_dma_direction(dir));
	if (ops->sync_sg_for_cpu)
		ops->sync_sg_for_cpu(dev, sg, nelems, dir);
	debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
}

static inline void
dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
		       int nelems, enum dma_data_direction dir)
{
420
	const struct dma_map_ops *ops = get_dma_ops(dev);
421 422 423 424 425 426 427 428

	BUG_ON(!valid_dma_direction(dir));
	if (ops->sync_sg_for_device)
		ops->sync_sg_for_device(dev, sg, nelems, dir);
	debug_dma_sync_sg_for_device(dev, sg, nelems, dir);

}

429 430 431 432
#define dma_map_single(d, a, s, r) dma_map_single_attrs(d, a, s, r, 0)
#define dma_unmap_single(d, a, s, r) dma_unmap_single_attrs(d, a, s, r, 0)
#define dma_map_sg(d, s, n, r) dma_map_sg_attrs(d, s, n, r, 0)
#define dma_unmap_sg(d, s, n, r) dma_unmap_sg_attrs(d, s, n, r, 0)
433 434
#define dma_map_page(d, p, o, s, r) dma_map_page_attrs(d, p, o, s, r, 0)
#define dma_unmap_page(d, a, s, r) dma_unmap_page_attrs(d, a, s, r, 0)
435

436 437 438 439 440 441 442 443 444 445 446
static inline void
dma_cache_sync(struct device *dev, void *vaddr, size_t size,
		enum dma_data_direction dir)
{
	const struct dma_map_ops *ops = get_dma_ops(dev);

	BUG_ON(!valid_dma_direction(dir));
	if (ops->cache_sync)
		ops->cache_sync(dev, vaddr, size, dir);
}

447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473
extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
			   void *cpu_addr, dma_addr_t dma_addr, size_t size);

void *dma_common_contiguous_remap(struct page *page, size_t size,
			unsigned long vm_flags,
			pgprot_t prot, const void *caller);

void *dma_common_pages_remap(struct page **pages, size_t size,
			unsigned long vm_flags, pgprot_t prot,
			const void *caller);
void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags);

/**
 * dma_mmap_attrs - map a coherent DMA allocation into user space
 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 * @vma: vm_area_struct describing requested user mapping
 * @cpu_addr: kernel CPU-view address returned from dma_alloc_attrs
 * @handle: device-view address returned from dma_alloc_attrs
 * @size: size of memory originally requested in dma_alloc_attrs
 * @attrs: attributes of mapping properties requested in dma_alloc_attrs
 *
 * Map a coherent DMA buffer previously allocated by dma_alloc_attrs
 * into user space.  The coherent DMA buffer must not be freed by the
 * driver until the user space mapping has been released.
 */
static inline int
dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, void *cpu_addr,
474
	       dma_addr_t dma_addr, size_t size, unsigned long attrs)
475
{
476
	const struct dma_map_ops *ops = get_dma_ops(dev);
477 478 479 480 481 482
	BUG_ON(!ops);
	if (ops->mmap)
		return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
	return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
}

483
#define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, 0)
484 485 486 487 488 489 490

int
dma_common_get_sgtable(struct device *dev, struct sg_table *sgt,
		       void *cpu_addr, dma_addr_t dma_addr, size_t size);

static inline int
dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, void *cpu_addr,
491 492
		      dma_addr_t dma_addr, size_t size,
		      unsigned long attrs)
493
{
494
	const struct dma_map_ops *ops = get_dma_ops(dev);
495 496 497 498 499 500 501
	BUG_ON(!ops);
	if (ops->get_sgtable)
		return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size,
					attrs);
	return dma_common_get_sgtable(dev, sgt, cpu_addr, dma_addr, size);
}

502
#define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, 0)
503 504

#ifndef arch_dma_alloc_attrs
505
#define arch_dma_alloc_attrs(dev)	(true)
506 507 508 509
#endif

static inline void *dma_alloc_attrs(struct device *dev, size_t size,
				       dma_addr_t *dma_handle, gfp_t flag,
510
				       unsigned long attrs)
511
{
512
	const struct dma_map_ops *ops = get_dma_ops(dev);
513 514 515
	void *cpu_addr;

	BUG_ON(!ops);
516
	WARN_ON_ONCE(dev && !dev->coherent_dma_mask);
517

518
	if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr))
519 520
		return cpu_addr;

521 522
	/* let the implementation decide on the zone to allocate from: */
	flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
523

524
	if (!arch_dma_alloc_attrs(&dev))
525 526 527 528 529 530 531 532 533 534 535
		return NULL;
	if (!ops->alloc)
		return NULL;

	cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
	debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr);
	return cpu_addr;
}

static inline void dma_free_attrs(struct device *dev, size_t size,
				     void *cpu_addr, dma_addr_t dma_handle,
536
				     unsigned long attrs)
537
{
538
	const struct dma_map_ops *ops = get_dma_ops(dev);
539 540 541

	BUG_ON(!ops);

542
	if (dma_release_from_dev_coherent(dev, get_order(size), cpu_addr))
543
		return;
544 545 546 547 548 549 550 551
	/*
	 * On non-coherent platforms which implement DMA-coherent buffers via
	 * non-cacheable remaps, ops->free() may call vunmap(). Thus getting
	 * this far in IRQ context is a) at risk of a BUG_ON() or trying to
	 * sleep on some machines, and b) an indication that the driver is
	 * probably misusing the coherent API anyway.
	 */
	WARN_ON(irqs_disabled());
552

553
	if (!ops->free || !cpu_addr)
554 555 556 557 558 559 560 561 562
		return;

	debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);
	ops->free(dev, size, cpu_addr, dma_handle, attrs);
}

static inline void *dma_alloc_coherent(struct device *dev, size_t size,
		dma_addr_t *dma_handle, gfp_t flag)
{
563
	return dma_alloc_attrs(dev, size, dma_handle, flag, 0);
564 565 566 567 568
}

static inline void dma_free_coherent(struct device *dev, size_t size,
		void *cpu_addr, dma_addr_t dma_handle)
{
569
	return dma_free_attrs(dev, size, cpu_addr, dma_handle, 0);
570 571 572 573
}

static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
{
574
	const struct dma_map_ops *ops = get_dma_ops(dev);
575

576 577 578
	debug_dma_mapping_error(dev, dma_addr);
	if (ops->mapping_error)
		return ops->mapping_error(dev, dma_addr);
579 580 581
	return 0;
}

582 583 584 585 586 587
static inline void dma_check_mask(struct device *dev, u64 mask)
{
	if (sme_active() && (mask < (((u64)sme_get_me_mask() << 1) - 1)))
		dev_warn(dev, "SME is active, device will require DMA bounce buffers\n");
}

588 589
static inline int dma_supported(struct device *dev, u64 mask)
{
590
	const struct dma_map_ops *ops = get_dma_ops(dev);
591 592 593 594 595 596 597 598 599 600 601 602 603

	if (!ops)
		return 0;
	if (!ops->dma_supported)
		return 1;
	return ops->dma_supported(dev, mask);
}

#ifndef HAVE_ARCH_DMA_SET_MASK
static inline int dma_set_mask(struct device *dev, u64 mask)
{
	if (!dev->dma_mask || !dma_supported(dev, mask))
		return -EIO;
604 605 606

	dma_check_mask(dev, mask);

607 608 609
	*dev->dma_mask = mask;
	return 0;
}
610
#endif
Linus Torvalds's avatar
Linus Torvalds committed
611

612 613
static inline u64 dma_get_mask(struct device *dev)
{
614
	if (dev && dev->dma_mask && *dev->dma_mask)
615
		return *dev->dma_mask;
616
	return DMA_BIT_MASK(32);
617 618
}

619
#ifdef CONFIG_ARCH_HAS_DMA_SET_COHERENT_MASK
620 621
int dma_set_coherent_mask(struct device *dev, u64 mask);
#else
622 623 624 625
static inline int dma_set_coherent_mask(struct device *dev, u64 mask)
{
	if (!dma_supported(dev, mask))
		return -EIO;
626 627 628

	dma_check_mask(dev, mask);

629 630 631
	dev->coherent_dma_mask = mask;
	return 0;
}
632
#endif
633

634 635 636 637 638 639 640 641 642 643 644 645 646 647
/*
 * Set both the DMA mask and the coherent DMA mask to the same thing.
 * Note that we don't check the return value from dma_set_coherent_mask()
 * as the DMA API guarantees that the coherent DMA mask can be set to
 * the same or smaller than the streaming DMA mask.
 */
static inline int dma_set_mask_and_coherent(struct device *dev, u64 mask)
{
	int rc = dma_set_mask(dev, mask);
	if (rc == 0)
		dma_set_coherent_mask(dev, mask);
	return rc;
}

648 649 650 651 652 653 654 655 656 657
/*
 * Similar to the above, except it deals with the case where the device
 * does not have dev->dma_mask appropriately setup.
 */
static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask)
{
	dev->dma_mask = &dev->coherent_dma_mask;
	return dma_set_mask_and_coherent(dev, mask);
}

Linus Torvalds's avatar
Linus Torvalds committed
658 659
extern u64 dma_get_required_mask(struct device *dev);

660
#ifndef arch_setup_dma_ops
661
static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
662
				      u64 size, const struct iommu_ops *iommu,
663 664 665 666 667
				      bool coherent) { }
#endif

#ifndef arch_teardown_dma_ops
static inline void arch_teardown_dma_ops(struct device *dev) { }
668 669
#endif

670 671
static inline unsigned int dma_get_max_seg_size(struct device *dev)
{
672 673 674
	if (dev->dma_parms && dev->dma_parms->max_segment_size)
		return dev->dma_parms->max_segment_size;
	return SZ_64K;
675 676 677 678 679 680 681 682
}

static inline unsigned int dma_set_max_seg_size(struct device *dev,
						unsigned int size)
{
	if (dev->dma_parms) {
		dev->dma_parms->max_segment_size = size;
		return 0;
683 684
	}
	return -EIO;
685 686
}

687 688
static inline unsigned long dma_get_seg_boundary(struct device *dev)
{
689 690 691
	if (dev->dma_parms && dev->dma_parms->segment_boundary_mask)
		return dev->dma_parms->segment_boundary_mask;
	return DMA_BIT_MASK(32);
692 693 694 695 696 697 698
}

static inline int dma_set_seg_boundary(struct device *dev, unsigned long mask)
{
	if (dev->dma_parms) {
		dev->dma_parms->segment_boundary_mask = mask;
		return 0;
699 700
	}
	return -EIO;
701 702
}

703 704 705
#ifndef dma_max_pfn
static inline unsigned long dma_max_pfn(struct device *dev)
{
706
	return (*dev->dma_mask >> PAGE_SHIFT) + dev->dma_pfn_offset;
707 708 709
}
#endif

710 711 712
static inline void *dma_zalloc_coherent(struct device *dev, size_t size,
					dma_addr_t *dma_handle, gfp_t flag)
{
713 714
	void *ret = dma_alloc_coherent(dev, size, dma_handle,
				       flag | __GFP_ZERO);
715 716 717
	return ret;
}

718 719 720 721 722 723 724 725
static inline int dma_get_cache_alignment(void)
{
#ifdef ARCH_DMA_MINALIGN
	return ARCH_DMA_MINALIGN;
#endif
	return 1;
}

Linus Torvalds's avatar
Linus Torvalds committed
726
/* flags for the coherent memory api */
727
#define DMA_MEMORY_EXCLUSIVE		0x01
Linus Torvalds's avatar
Linus Torvalds committed
728

729 730 731 732 733 734 735
#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT
int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
				dma_addr_t device_addr, size_t size, int flags);
void dma_release_declared_memory(struct device *dev);
void *dma_mark_declared_memory_occupied(struct device *dev,
					dma_addr_t device_addr, size_t size);
#else
Linus Torvalds's avatar
Linus Torvalds committed
736
static inline int
737
dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
Linus Torvalds's avatar
Linus Torvalds committed
738 739
			    dma_addr_t device_addr, size_t size, int flags)
{
740
	return -ENOSYS;
Linus Torvalds's avatar
Linus Torvalds committed
741 742 743 744 745 746 747 748 749 750 751 752 753
}

static inline void
dma_release_declared_memory(struct device *dev)
{
}

static inline void *
dma_mark_declared_memory_occupied(struct device *dev,
				  dma_addr_t device_addr, size_t size)
{
	return ERR_PTR(-EBUSY);
}
754
#endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */
Linus Torvalds's avatar
Linus Torvalds committed
755

756 757 758 759 760 761 762 763 764 765 766 767
#ifdef CONFIG_HAS_DMA
int dma_configure(struct device *dev);
void dma_deconfigure(struct device *dev);
#else
static inline int dma_configure(struct device *dev)
{
	return 0;
}

static inline void dma_deconfigure(struct device *dev) {}
#endif

768 769 770
/*
 * Managed DMA API
 */
771
#ifdef CONFIG_HAS_DMA
772 773 774 775
extern void *dmam_alloc_coherent(struct device *dev, size_t size,
				 dma_addr_t *dma_handle, gfp_t gfp);
extern void dmam_free_coherent(struct device *dev, size_t size, void *vaddr,
			       dma_addr_t dma_handle);
776 777 778 779 780 781 782 783
#else /* !CONFIG_HAS_DMA */
static inline void *dmam_alloc_coherent(struct device *dev, size_t size,
					dma_addr_t *dma_handle, gfp_t gfp)
{ return NULL; }
static inline void dmam_free_coherent(struct device *dev, size_t size,
				      void *vaddr, dma_addr_t dma_handle) { }
#endif /* !CONFIG_HAS_DMA */

784 785 786
extern void *dmam_alloc_attrs(struct device *dev, size_t size,
			      dma_addr_t *dma_handle, gfp_t gfp,
			      unsigned long attrs);
787
#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT
788 789
extern int dmam_declare_coherent_memory(struct device *dev,
					phys_addr_t phys_addr,
790 791 792
					dma_addr_t device_addr, size_t size,
					int flags);
extern void dmam_release_declared_memory(struct device *dev);
793
#else /* CONFIG_HAVE_GENERIC_DMA_COHERENT */
794
static inline int dmam_declare_coherent_memory(struct device *dev,
795
				phys_addr_t phys_addr, dma_addr_t device_addr,
796 797 798 799
				size_t size, gfp_t gfp)
{
	return 0;
}
Linus Torvalds's avatar
Linus Torvalds committed
800

801 802 803
static inline void dmam_release_declared_memory(struct device *dev)
{
}
804
#endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */
Linus Torvalds's avatar
Linus Torvalds committed
805

806 807
static inline void *dma_alloc_wc(struct device *dev, size_t size,
				 dma_addr_t *dma_addr, gfp_t gfp)
808
{
809 810
	return dma_alloc_attrs(dev, size, dma_addr, gfp,
			       DMA_ATTR_WRITE_COMBINE);
811
}
812 813 814
#ifndef dma_alloc_writecombine
#define dma_alloc_writecombine dma_alloc_wc
#endif
815

816 817
static inline void dma_free_wc(struct device *dev, size_t size,
			       void *cpu_addr, dma_addr_t dma_addr)
818
{
819 820
	return dma_free_attrs(dev, size, cpu_addr, dma_addr,
			      DMA_ATTR_WRITE_COMBINE);
821
}
822 823 824
#ifndef dma_free_writecombine
#define dma_free_writecombine dma_free_wc
#endif
825

826 827 828 829
static inline int dma_mmap_wc(struct device *dev,
			      struct vm_area_struct *vma,
			      void *cpu_addr, dma_addr_t dma_addr,
			      size_t size)
830
{
831 832
	return dma_mmap_attrs(dev, vma, cpu_addr, dma_addr, size,
			      DMA_ATTR_WRITE_COMBINE);
833
}
834 835 836
#ifndef dma_mmap_writecombine
#define dma_mmap_writecombine dma_mmap_wc
#endif
837

838
#ifdef CONFIG_NEED_DMA_MAP_STATE
839 840 841 842 843 844 845 846 847 848 849 850 851 852 853
#define DEFINE_DMA_UNMAP_ADDR(ADDR_NAME)        dma_addr_t ADDR_NAME
#define DEFINE_DMA_UNMAP_LEN(LEN_NAME)          __u32 LEN_NAME
#define dma_unmap_addr(PTR, ADDR_NAME)           ((PTR)->ADDR_NAME)
#define dma_unmap_addr_set(PTR, ADDR_NAME, VAL)  (((PTR)->ADDR_NAME) = (VAL))
#define dma_unmap_len(PTR, LEN_NAME)             ((PTR)->LEN_NAME)
#define dma_unmap_len_set(PTR, LEN_NAME, VAL)    (((PTR)->LEN_NAME) = (VAL))
#else
#define DEFINE_DMA_UNMAP_ADDR(ADDR_NAME)
#define DEFINE_DMA_UNMAP_LEN(LEN_NAME)
#define dma_unmap_addr(PTR, ADDR_NAME)           (0)
#define dma_unmap_addr_set(PTR, ADDR_NAME, VAL)  do { } while (0)
#define dma_unmap_len(PTR, LEN_NAME)             (0)
#define dma_unmap_len_set(PTR, LEN_NAME, VAL)    do { } while (0)
#endif

854
#endif