radeon.h 94.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
/*
 * Copyright 2008 Advanced Micro Devices, Inc.
 * Copyright 2008 Red Hat Inc.
 * Copyright 2009 Jerome Glisse.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Dave Airlie
 *          Alex Deucher
 *          Jerome Glisse
 */
#ifndef __RADEON_H__
#define __RADEON_H__

/* TODO: Here are things that needs to be done :
 *	- surface allocator & initializer : (bit like scratch reg) should
 *	  initialize HDP_ stuff on RS600, R600, R700 hw, well anythings
 *	  related to surface
 *	- WB : write back stuff (do it bit like scratch reg things)
 *	- Vblank : look at Jesse's rework and what we should do
 *	- r600/r700: gart & cp
 *	- cs : clean cs ioctl use bitmap & things like that.
 *	- power management stuff
 *	- Barrier in gart code
 *	- Unmappabled vram ?
 *	- TESTING, TESTING, TESTING
 */

45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
/* Initialization path:
 *  We expect that acceleration initialization might fail for various
 *  reasons even thought we work hard to make it works on most
 *  configurations. In order to still have a working userspace in such
 *  situation the init path must succeed up to the memory controller
 *  initialization point. Failure before this point are considered as
 *  fatal error. Here is the init callchain :
 *      radeon_device_init  perform common structure, mutex initialization
 *      asic_init           setup the GPU memory layout and perform all
 *                          one time initialization (failure in this
 *                          function are considered fatal)
 *      asic_startup        setup the GPU acceleration, in order to
 *                          follow guideline the first thing this
 *                          function should do is setting the GPU
 *                          memory controller (only MC setup failure
 *                          are considered as fatal)
 */

63
#include <linux/atomic.h>
64 65 66
#include <linux/wait.h>
#include <linux/list.h>
#include <linux/kref.h>
67
#include <linux/interval_tree.h>
68
#include <linux/hashtable.h>
69
#include <linux/dma-fence.h>
70

71 72 73 74
#include <ttm/ttm_bo_api.h>
#include <ttm/ttm_bo_driver.h>
#include <ttm/ttm_placement.h>
#include <ttm/ttm_module.h>
75
#include <ttm/ttm_execbuf_util.h>
76

77 78
#include <drm/drm_gem.h>

79
#include "radeon_family.h"
80 81 82 83 84 85 86 87 88 89 90 91 92 93
#include "radeon_mode.h"
#include "radeon_reg.h"

/*
 * Modules parameters.
 */
extern int radeon_no_wb;
extern int radeon_modeset;
extern int radeon_dynclks;
extern int radeon_r4xx_atom;
extern int radeon_agpmode;
extern int radeon_vram_limit;
extern int radeon_gart_size;
extern int radeon_benchmarking;
94
extern int radeon_testing;
95
extern int radeon_connector_table;
96
extern int radeon_tv;
97
extern int radeon_audio;
98
extern int radeon_disp_priority;
99
extern int radeon_hw_i2c;
100
extern int radeon_pcie_gen2;
101
extern int radeon_msi;
102
extern int radeon_lockup_timeout;
103
extern int radeon_fastfb;
104
extern int radeon_dpm;
105
extern int radeon_aspm;
106
extern int radeon_runtime_pm;
107
extern int radeon_hard_reset;
108
extern int radeon_vm_size;
109
extern int radeon_vm_block_size;
110
extern int radeon_deep_color;
111
extern int radeon_use_pflipirq;
112
extern int radeon_bapm;
113
extern int radeon_backlight;
114
extern int radeon_auxch;
115
extern int radeon_mst;
116
extern int radeon_uvd;
117
extern int radeon_vce;
118 119 120 121 122

/*
 * Copy from radeon_drv.h so we don't have to include both and have conflicting
 * symbol;
 */
123 124
#define RADEON_MAX_USEC_TIMEOUT			100000	/* 100 ms */
#define RADEON_FENCE_JIFFIES_TIMEOUT		(HZ / 2)
125
#define RADEON_USEC_IB_TEST_TIMEOUT		1000000 /* 1s */
126
/* RADEON_IB_POOL_SIZE must be a power of 2 */
127 128 129 130
#define RADEON_IB_POOL_SIZE			16
#define RADEON_DEBUGFS_MAX_COMPONENTS		32
#define RADEONFB_CONN_LIMIT			4
#define RADEON_BIOS_NUM_SCRATCH			8
131

132 133
/* internal ring indices */
/* r1xx+ has gfx CP ring */
134
#define RADEON_RING_TYPE_GFX_INDEX		0
135 136

/* cayman has 2 compute CP rings */
137 138
#define CAYMAN_RING_TYPE_CP1_INDEX		1
#define CAYMAN_RING_TYPE_CP2_INDEX		2
139

140 141
/* R600+ has an async dma ring */
#define R600_RING_TYPE_DMA_INDEX		3
142 143
/* cayman add a second async dma ring */
#define CAYMAN_RING_TYPE_DMA1_INDEX		4
144

145
/* R600+ */
146 147 148 149 150 151 152 153
#define R600_RING_TYPE_UVD_INDEX		5

/* TN+ */
#define TN_RING_TYPE_VCE1_INDEX			6
#define TN_RING_TYPE_VCE2_INDEX			7

/* max number of rings */
#define RADEON_NUM_RINGS			8
154

155 156
/* number of hw syncs before falling back on blocking */
#define RADEON_NUM_SYNCS			4
157

158
/* hardcode those limit for now */
159
#define RADEON_VA_IB_OFFSET			(1 << 20)
160 161
#define RADEON_VA_RESERVED_SIZE			(8 << 20)
#define RADEON_IB_VM_MAX_SIZE			(64 << 10)
162

163 164 165
/* hard reset data */
#define RADEON_ASIC_RESET_DATA                  0x39d5e86b

166 167 168 169
/* reset flags */
#define RADEON_RESET_GFX			(1 << 0)
#define RADEON_RESET_COMPUTE			(1 << 1)
#define RADEON_RESET_DMA			(1 << 2)
170 171 172 173 174 175 176 177 178
#define RADEON_RESET_CP				(1 << 3)
#define RADEON_RESET_GRBM			(1 << 4)
#define RADEON_RESET_DMA1			(1 << 5)
#define RADEON_RESET_RLC			(1 << 6)
#define RADEON_RESET_SEM			(1 << 7)
#define RADEON_RESET_IH				(1 << 8)
#define RADEON_RESET_VMC			(1 << 9)
#define RADEON_RESET_MC				(1 << 10)
#define RADEON_RESET_DISPLAY			(1 << 11)
179

180 181 182 183 184 185 186
/* CG block flags */
#define RADEON_CG_BLOCK_GFX			(1 << 0)
#define RADEON_CG_BLOCK_MC			(1 << 1)
#define RADEON_CG_BLOCK_SDMA			(1 << 2)
#define RADEON_CG_BLOCK_UVD			(1 << 3)
#define RADEON_CG_BLOCK_VCE			(1 << 4)
#define RADEON_CG_BLOCK_HDP			(1 << 5)
187
#define RADEON_CG_BLOCK_BIF			(1 << 6)
188

189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208
/* CG flags */
#define RADEON_CG_SUPPORT_GFX_MGCG		(1 << 0)
#define RADEON_CG_SUPPORT_GFX_MGLS		(1 << 1)
#define RADEON_CG_SUPPORT_GFX_CGCG		(1 << 2)
#define RADEON_CG_SUPPORT_GFX_CGLS		(1 << 3)
#define RADEON_CG_SUPPORT_GFX_CGTS		(1 << 4)
#define RADEON_CG_SUPPORT_GFX_CGTS_LS		(1 << 5)
#define RADEON_CG_SUPPORT_GFX_CP_LS		(1 << 6)
#define RADEON_CG_SUPPORT_GFX_RLC_LS		(1 << 7)
#define RADEON_CG_SUPPORT_MC_LS			(1 << 8)
#define RADEON_CG_SUPPORT_MC_MGCG		(1 << 9)
#define RADEON_CG_SUPPORT_SDMA_LS		(1 << 10)
#define RADEON_CG_SUPPORT_SDMA_MGCG		(1 << 11)
#define RADEON_CG_SUPPORT_BIF_LS		(1 << 12)
#define RADEON_CG_SUPPORT_UVD_MGCG		(1 << 13)
#define RADEON_CG_SUPPORT_VCE_MGCG		(1 << 14)
#define RADEON_CG_SUPPORT_HDP_LS		(1 << 15)
#define RADEON_CG_SUPPORT_HDP_MGCG		(1 << 16)

/* PG flags */
209
#define RADEON_PG_SUPPORT_GFX_PG		(1 << 0)
210 211 212 213 214 215 216 217 218 219 220
#define RADEON_PG_SUPPORT_GFX_SMG		(1 << 1)
#define RADEON_PG_SUPPORT_GFX_DMG		(1 << 2)
#define RADEON_PG_SUPPORT_UVD			(1 << 3)
#define RADEON_PG_SUPPORT_VCE			(1 << 4)
#define RADEON_PG_SUPPORT_CP			(1 << 5)
#define RADEON_PG_SUPPORT_GDS			(1 << 6)
#define RADEON_PG_SUPPORT_RLC_SMU_HS		(1 << 7)
#define RADEON_PG_SUPPORT_SDMA			(1 << 8)
#define RADEON_PG_SUPPORT_ACP			(1 << 9)
#define RADEON_PG_SUPPORT_SAMU			(1 << 10)

221 222 223 224 225 226 227
/* max cursor sizes (in pixels) */
#define CURSOR_WIDTH 64
#define CURSOR_HEIGHT 64

#define CIK_CURSOR_WIDTH 128
#define CIK_CURSOR_HEIGHT 128

228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
/*
 * Errata workarounds.
 */
enum radeon_pll_errata {
	CHIP_ERRATA_R300_CG             = 0x00000001,
	CHIP_ERRATA_PLL_DUMMYREADS      = 0x00000002,
	CHIP_ERRATA_PLL_DELAY           = 0x00000004
};


struct radeon_device;


/*
 * BIOS.
 */
bool radeon_get_bios(struct radeon_device *rdev);

/*
247
 * Dummy page
248
 */
249
struct radeon_dummy_page {
250
	uint64_t	entry;
251 252 253 254 255 256
	struct page	*page;
	dma_addr_t	addr;
};
int radeon_dummy_page_init(struct radeon_device *rdev);
void radeon_dummy_page_fini(struct radeon_device *rdev);

257

258 259 260
/*
 * Clocks
 */
261 262 263
struct radeon_clock {
	struct radeon_pll p1pll;
	struct radeon_pll p2pll;
264
	struct radeon_pll dcpll;
265 266 267 268 269
	struct radeon_pll spll;
	struct radeon_pll mpll;
	/* 10 Khz units */
	uint32_t default_mclk;
	uint32_t default_sclk;
270
	uint32_t default_dispclk;
271
	uint32_t current_dispclk;
272
	uint32_t dp_extclk;
273
	uint32_t max_pixel_clock;
274
	uint32_t vco_freq;
275 276
};

277 278 279 280
/*
 * Power management
 */
int radeon_pm_init(struct radeon_device *rdev);
281
int radeon_pm_late_init(struct radeon_device *rdev);
282
void radeon_pm_fini(struct radeon_device *rdev);
283
void radeon_pm_compute_clocks(struct radeon_device *rdev);
284 285
void radeon_pm_suspend(struct radeon_device *rdev);
void radeon_pm_resume(struct radeon_device *rdev);
286 287
void radeon_combios_get_power_modes(struct radeon_device *rdev);
void radeon_atombios_get_power_modes(struct radeon_device *rdev);
288 289 290 291 292
int radeon_atom_get_clock_dividers(struct radeon_device *rdev,
				   u8 clock_type,
				   u32 clock,
				   bool strobe_mode,
				   struct atom_clock_dividers *dividers);
293 294 295 296
int radeon_atom_get_memory_pll_dividers(struct radeon_device *rdev,
					u32 clock,
					bool strobe_mode,
					struct atom_mpll_param *mpll_param);
297
void radeon_atom_set_voltage(struct radeon_device *rdev, u16 voltage_level, u8 voltage_type);
298 299 300 301 302 303 304
int radeon_atom_get_voltage_gpio_settings(struct radeon_device *rdev,
					  u16 voltage_level, u8 voltage_type,
					  u32 *gpio_value, u32 *gpio_mask);
void radeon_atom_set_engine_dram_timings(struct radeon_device *rdev,
					 u32 eng_clock, u32 mem_clock);
int radeon_atom_get_voltage_step(struct radeon_device *rdev,
				 u8 voltage_type, u16 *voltage_step);
305 306
int radeon_atom_get_max_vddc(struct radeon_device *rdev, u8 voltage_type,
			     u16 voltage_id, u16 *voltage);
307 308 309
int radeon_atom_get_leakage_vddc_based_on_leakage_idx(struct radeon_device *rdev,
						      u16 *voltage,
						      u16 leakage_idx);
310 311 312 313 314 315
int radeon_atom_get_leakage_id_from_vbios(struct radeon_device *rdev,
					  u16 *leakage_id);
int radeon_atom_get_leakage_vddc_based_on_leakage_params(struct radeon_device *rdev,
							 u16 *vddc, u16 *vddci,
							 u16 virtual_voltage_id,
							 u16 vbios_voltage_id);
316 317 318
int radeon_atom_get_voltage_evv(struct radeon_device *rdev,
				u16 virtual_voltage_id,
				u16 *voltage);
319 320 321 322 323 324 325 326 327
int radeon_atom_round_to_true_voltage(struct radeon_device *rdev,
				      u8 voltage_type,
				      u16 nominal_voltage,
				      u16 *true_voltage);
int radeon_atom_get_min_voltage(struct radeon_device *rdev,
				u8 voltage_type, u16 *min_voltage);
int radeon_atom_get_max_voltage(struct radeon_device *rdev,
				u8 voltage_type, u16 *max_voltage);
int radeon_atom_get_voltage_table(struct radeon_device *rdev,
328
				  u8 voltage_type, u8 voltage_mode,
329
				  struct atom_voltage_table *voltage_table);
330 331
bool radeon_atom_is_voltage_gpio(struct radeon_device *rdev,
				 u8 voltage_type, u8 voltage_mode);
332 333 334
int radeon_atom_get_svi2_info(struct radeon_device *rdev,
			      u8 voltage_type,
			      u8 *svd_gpio_id, u8 *svc_gpio_id);
335 336 337 338 339 340 341 342 343 344 345 346 347 348
void radeon_atom_update_memory_dll(struct radeon_device *rdev,
				   u32 mem_clock);
void radeon_atom_set_ac_timing(struct radeon_device *rdev,
			       u32 mem_clock);
int radeon_atom_init_mc_reg_table(struct radeon_device *rdev,
				  u8 module_index,
				  struct atom_mc_reg_table *reg_table);
int radeon_atom_get_memory_info(struct radeon_device *rdev,
				u8 module_index, struct atom_memory_info *mem_info);
int radeon_atom_get_mclk_range_table(struct radeon_device *rdev,
				     bool gddr5, u8 module_index,
				     struct atom_memory_clock_range_table *mclk_range_table);
int radeon_atom_get_max_vddc(struct radeon_device *rdev, u8 voltage_type,
			     u16 voltage_id, u16 *voltage);
349
void rs690_pm_info(struct radeon_device *rdev);
350 351 352
extern void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
				    unsigned *bankh, unsigned *mtaspect,
				    unsigned *tile_split);
353

354 355 356 357
/*
 * Fences.
 */
struct radeon_fence_driver {
358
	struct radeon_device		*rdev;
359
	uint32_t			scratch_reg;
360 361
	uint64_t			gpu_addr;
	volatile uint32_t		*cpu_addr;
362 363
	/* sync_seq is protected by ring emission lock */
	uint64_t			sync_seq[RADEON_NUM_RINGS];
364
	atomic64_t			last_seq;
365
	bool				initialized, delayed_irq;
366
	struct delayed_work		lockup_work;
367 368 369
};

struct radeon_fence {
370
	struct dma_fence		base;
371

372 373
	struct radeon_device	*rdev;
	uint64_t		seq;
374
	/* RB, DMA, etc. */
375 376
	unsigned		ring;
	bool			is_vm_update;
377

378
	wait_queue_entry_t		fence_wake;
379 380
};

381 382
int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring);
int radeon_fence_driver_init(struct radeon_device *rdev);
383
void radeon_fence_driver_fini(struct radeon_device *rdev);
384
void radeon_fence_driver_force_completion(struct radeon_device *rdev, int ring);
385
int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence **fence, int ring);
386
void radeon_fence_process(struct radeon_device *rdev, int ring);
387
bool radeon_fence_signaled(struct radeon_fence *fence);
388
long radeon_fence_wait_timeout(struct radeon_fence *fence, bool interruptible, long timeout);
389
int radeon_fence_wait(struct radeon_fence *fence, bool interruptible);
390 391
int radeon_fence_wait_next(struct radeon_device *rdev, int ring);
int radeon_fence_wait_empty(struct radeon_device *rdev, int ring);
392 393 394
int radeon_fence_wait_any(struct radeon_device *rdev,
			  struct radeon_fence **fences,
			  bool intr);
395 396
struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence);
void radeon_fence_unref(struct radeon_fence **fence);
397
unsigned radeon_fence_count_emitted(struct radeon_device *rdev, int ring);
398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418
bool radeon_fence_need_sync(struct radeon_fence *fence, int ring);
void radeon_fence_note_sync(struct radeon_fence *fence, int ring);
static inline struct radeon_fence *radeon_fence_later(struct radeon_fence *a,
						      struct radeon_fence *b)
{
	if (!a) {
		return b;
	}

	if (!b) {
		return a;
	}

	BUG_ON(a->ring != b->ring);

	if (a->seq > b->seq) {
		return a;
	} else {
		return b;
	}
}
419

420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
static inline bool radeon_fence_is_earlier(struct radeon_fence *a,
					   struct radeon_fence *b)
{
	if (!a) {
		return false;
	}

	if (!b) {
		return true;
	}

	BUG_ON(a->ring != b->ring);

	return a->seq < b->seq;
}

436 437 438 439
/*
 * Tiling registers
 */
struct radeon_surface_reg {
440
	struct radeon_bo *bo;
441 442 443
};

#define RADEON_GEM_MAX_SURFACES 8
444 445

/*
446
 * TTM.
447
 */
448 449
struct radeon_mman {
	struct ttm_bo_global_ref        bo_global_ref;
450
	struct drm_global_reference	mem_global_ref;
451
	struct ttm_bo_device		bdev;
452 453
	bool				mem_global_referenced;
	bool				initialized;
454 455 456

#if defined(CONFIG_DEBUG_FS)
	struct dentry			*vram;
457
	struct dentry			*gtt;
458
#endif
459 460
};

461 462 463 464 465 466 467 468 469
struct radeon_bo_list {
	struct radeon_bo		*robj;
	struct ttm_validate_buffer	tv;
	uint64_t			gpu_offset;
	unsigned			prefered_domains;
	unsigned			allowed_domains;
	uint32_t			tiling_flags;
};

470 471
/* bo virtual address in a specific vm */
struct radeon_bo_va {
472
	/* protected by bo being reserved */
473 474
	struct list_head		bo_list;
	uint32_t			flags;
475
	struct radeon_fence		*last_pt_update;
476 477 478
	unsigned			ref_count;

	/* protected by vm mutex */
479
	struct interval_tree_node	it;
480
	struct list_head		vm_status;
481 482 483 484

	/* constant after initialization */
	struct radeon_vm		*vm;
	struct radeon_bo		*bo;
485 486
};

487 488 489 490
struct radeon_bo {
	/* Protected by gem.mutex */
	struct list_head		list;
	/* Protected by tbo.reserved */
491
	u32				initial_domain;
492
	struct ttm_place		placements[4];
493
	struct ttm_placement		placement;
494 495
	struct ttm_buffer_object	tbo;
	struct ttm_bo_kmap_obj		kmap;
496
	u32				flags;
497 498 499 500 501
	unsigned			pin_count;
	void				*kptr;
	u32				tiling_flags;
	u32				pitch;
	int				surface_reg;
502
	unsigned			prime_shared_count;
503 504 505 506
	/* list of all virtual address to which this bo
	 * is associated to
	 */
	struct list_head		va;
507 508
	/* Constant after initialization */
	struct radeon_device		*rdev;
509
	struct drm_gem_object		gem_base;
510

511 512
	struct ttm_bo_kmap_obj		dma_buf_vmap;
	pid_t				pid;
513 514

	struct radeon_mn		*mn;
515
	struct list_head		mn_list;
516
};
517
#define gem_to_radeon_bo(gobj) container_of((gobj), struct radeon_bo, gem_base)
518

519 520
int radeon_gem_debugfs_init(struct radeon_device *rdev);

521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544
/* sub-allocation manager, it has to be protected by another lock.
 * By conception this is an helper for other part of the driver
 * like the indirect buffer or semaphore, which both have their
 * locking.
 *
 * Principe is simple, we keep a list of sub allocation in offset
 * order (first entry has offset == 0, last entry has the highest
 * offset).
 *
 * When allocating new object we first check if there is room at
 * the end total_size - (last_object_offset + last_object_size) >=
 * alloc_size. If so we allocate new object there.
 *
 * When there is not enough room at the end, we start waiting for
 * each sub object until we reach object_offset+object_size >=
 * alloc_size, this object then become the sub object we return.
 *
 * Alignment can't be bigger than page size.
 *
 * Hole are not considered for allocation to keep things simple.
 * Assumption is that there won't be hole (all object on same
 * alignment).
 */
struct radeon_sa_manager {
545
	wait_queue_head_t	wq;
546
	struct radeon_bo	*bo;
547 548 549
	struct list_head	*hole;
	struct list_head	flist[RADEON_NUM_RINGS];
	struct list_head	olist;
550 551 552 553
	unsigned		size;
	uint64_t		gpu_addr;
	void			*cpu_ptr;
	uint32_t		domain;
554
	uint32_t		align;
555 556 557 558 559 560
};

struct radeon_sa_bo;

/* sub-allocation buffer */
struct radeon_sa_bo {
561 562
	struct list_head		olist;
	struct list_head		flist;
563
	struct radeon_sa_manager	*manager;
564 565
	unsigned			soffset;
	unsigned			eoffset;
566
	struct radeon_fence		*fence;
567 568
};

569 570 571 572
/*
 * GEM objects.
 */
struct radeon_gem {
573
	struct mutex		mutex;
574 575 576 577 578
	struct list_head	objects;
};

int radeon_gem_init(struct radeon_device *rdev);
void radeon_gem_fini(struct radeon_device *rdev);
579
int radeon_gem_object_create(struct radeon_device *rdev, unsigned long size,
580
				int alignment, int initial_domain,
581
				u32 flags, bool kernel,
582
				struct drm_gem_object **obj);
583

584 585 586 587 588 589
int radeon_mode_dumb_create(struct drm_file *file_priv,
			    struct drm_device *dev,
			    struct drm_mode_create_dumb *args);
int radeon_mode_dumb_mmap(struct drm_file *filp,
			  struct drm_device *dev,
			  uint32_t handle, uint64_t *offset_p);
590

591 592 593 594
/*
 * Semaphores.
 */
struct radeon_semaphore {
595 596 597
	struct radeon_sa_bo	*sa_bo;
	signed			waiters;
	uint64_t		gpu_addr;
598 599 600 601
};

int radeon_semaphore_create(struct radeon_device *rdev,
			    struct radeon_semaphore **semaphore);
602
bool radeon_semaphore_emit_signal(struct radeon_device *rdev, int ring,
603
				  struct radeon_semaphore *semaphore);
604
bool radeon_semaphore_emit_wait(struct radeon_device *rdev, int ring,
605 606
				struct radeon_semaphore *semaphore);
void radeon_semaphore_free(struct radeon_device *rdev,
607
			   struct radeon_semaphore **semaphore,
608
			   struct radeon_fence *fence);
609

610 611 612 613 614 615
/*
 * Synchronization
 */
struct radeon_sync {
	struct radeon_semaphore *semaphores[RADEON_NUM_SYNCS];
	struct radeon_fence	*sync_to[RADEON_NUM_RINGS];
616
	struct radeon_fence	*last_vm_update;
617 618 619 620 621 622 623 624 625 626 627 628 629 630 631
};

void radeon_sync_create(struct radeon_sync *sync);
void radeon_sync_fence(struct radeon_sync *sync,
		       struct radeon_fence *fence);
int radeon_sync_resv(struct radeon_device *rdev,
		     struct radeon_sync *sync,
		     struct reservation_object *resv,
		     bool shared);
int radeon_sync_rings(struct radeon_device *rdev,
		      struct radeon_sync *sync,
		      int waiting_ring);
void radeon_sync_free(struct radeon_device *rdev, struct radeon_sync *sync,
		      struct radeon_fence *fence);

632 633 634 635 636
/*
 * GART structures, functions & helpers
 */
struct radeon_mc;

637
#define RADEON_GPU_PAGE_SIZE 4096
638
#define RADEON_GPU_PAGE_MASK (RADEON_GPU_PAGE_SIZE - 1)
639
#define RADEON_GPU_PAGE_SHIFT 12
640
#define RADEON_GPU_PAGE_ALIGN(a) (((a) + RADEON_GPU_PAGE_MASK) & ~RADEON_GPU_PAGE_MASK)
641

642 643 644 645 646 647
#define RADEON_GART_PAGE_DUMMY  0
#define RADEON_GART_PAGE_VALID	(1 << 0)
#define RADEON_GART_PAGE_READ	(1 << 1)
#define RADEON_GART_PAGE_WRITE	(1 << 2)
#define RADEON_GART_PAGE_SNOOP	(1 << 3)

648 649
struct radeon_gart {
	dma_addr_t			table_addr;
650 651
	struct radeon_bo		*robj;
	void				*ptr;
652 653 654 655
	unsigned			num_gpu_pages;
	unsigned			num_cpu_pages;
	unsigned			table_size;
	struct page			**pages;
656
	uint64_t			*pages_entry;
657 658 659 660 661 662 663
	bool				ready;
};

int radeon_gart_table_ram_alloc(struct radeon_device *rdev);
void radeon_gart_table_ram_free(struct radeon_device *rdev);
int radeon_gart_table_vram_alloc(struct radeon_device *rdev);
void radeon_gart_table_vram_free(struct radeon_device *rdev);
664 665
int radeon_gart_table_vram_pin(struct radeon_device *rdev);
void radeon_gart_table_vram_unpin(struct radeon_device *rdev);
666 667 668 669 670
int radeon_gart_init(struct radeon_device *rdev);
void radeon_gart_fini(struct radeon_device *rdev);
void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset,
			int pages);
int radeon_gart_bind(struct radeon_device *rdev, unsigned offset,
671
		     int pages, struct page **pagelist,
672
		     dma_addr_t *dma_addr, uint32_t flags);
673 674 675 676 677 678 679 680 681


/*
 * GPU MC structures, functions & helpers
 */
struct radeon_mc {
	resource_size_t		aper_size;
	resource_size_t		aper_base;
	resource_size_t		agp_base;
682 683
	/* for some chips with <= 32MB we need to lie
	 * about vram size near mc fb location */
684
	u64			mc_vram_size;
685
	u64			visible_vram_size;
686 687 688 689 690
	u64			gtt_size;
	u64			gtt_start;
	u64			gtt_end;
	u64			vram_start;
	u64			vram_end;
691
	unsigned		vram_width;
692
	u64			real_vram_size;
693 694
	int			vram_mtrr;
	bool			vram_is_ddr;
695
	bool			igp_sideport_enabled;
696
	u64                     gtt_base_align;
697
	u64                     mc_mask;
698 699
};

700 701
bool radeon_combios_sideport_present(struct radeon_device *rdev);
bool radeon_atombios_sideport_present(struct radeon_device *rdev);
702 703 704 705 706 707

/*
 * GPU scratch registers structures, functions & helpers
 */
struct radeon_scratch {
	unsigned		num_reg;
708
	uint32_t                reg_base;
709 710 711 712 713 714 715
	bool			free[32];
	uint32_t		reg[32];
};

int radeon_scratch_get(struct radeon_device *rdev, uint32_t *reg);
void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg);

716 717 718
/*
 * GPU doorbell structures, functions & helpers
 */
719 720
#define RADEON_MAX_DOORBELLS 1024	/* Reserve at most 1024 doorbell slots for radeon-owned rings. */

721 722
struct radeon_doorbell {
	/* doorbell mmio */
723 724 725 726
	resource_size_t		base;
	resource_size_t		size;
	u32 __iomem		*ptr;
	u32			num_doorbells;	/* Number of doorbells actually reserved for radeon. */
727
	DECLARE_BITMAP(used, RADEON_MAX_DOORBELLS);
728 729 730 731
};

int radeon_doorbell_get(struct radeon_device *rdev, u32 *page);
void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell);
732 733 734 735
void radeon_doorbell_get_kfd_info(struct radeon_device *rdev,
				  phys_addr_t *aperture_base,
				  size_t *aperture_size,
				  size_t *start_offset);
736 737 738 739

/*
 * IRQS.
 */
740

741 742 743 744 745
struct radeon_flip_work {
	struct work_struct		flip_work;
	struct work_struct		unpin_work;
	struct radeon_device		*rdev;
	int				crtc_id;
746
	u32				target_vblank;
747
	uint64_t			base;
748
	struct drm_pending_vblank_event *event;
749
	struct radeon_bo		*old_rbo;
750
	struct dma_fence		*fence;
751
	bool				async;
752 753 754 755
};

struct r500_irq_stat_regs {
	u32 disp_int;
756
	u32 hdmi0_status;
757 758 759 760 761 762 763 764
};

struct r600_irq_stat_regs {
	u32 disp_int;
	u32 disp_int_cont;
	u32 disp_int_cont2;
	u32 d1grph_int;
	u32 d2grph_int;
765 766
	u32 hdmi0_status;
	u32 hdmi1_status;
767 768 769 770 771 772 773 774 775 776 777 778 779 780 781
};

struct evergreen_irq_stat_regs {
	u32 disp_int;
	u32 disp_int_cont;
	u32 disp_int_cont2;
	u32 disp_int_cont3;
	u32 disp_int_cont4;
	u32 disp_int_cont5;
	u32 d1grph_int;
	u32 d2grph_int;
	u32 d3grph_int;
	u32 d4grph_int;
	u32 d5grph_int;
	u32 d6grph_int;
782 783 784 785 786 787
	u32 afmt_status1;
	u32 afmt_status2;
	u32 afmt_status3;
	u32 afmt_status4;
	u32 afmt_status5;
	u32 afmt_status6;
788 789
};

790 791 792 793 794 795 796 797
struct cik_irq_stat_regs {
	u32 disp_int;
	u32 disp_int_cont;
	u32 disp_int_cont2;
	u32 disp_int_cont3;
	u32 disp_int_cont4;
	u32 disp_int_cont5;
	u32 disp_int_cont6;
798 799 800 801 802 803
	u32 d1grph_int;
	u32 d2grph_int;
	u32 d3grph_int;
	u32 d4grph_int;
	u32 d5grph_int;
	u32 d6grph_int;
804 805
};

806 807 808 809
union radeon_irq_stat_regs {
	struct r500_irq_stat_regs r500;
	struct r600_irq_stat_regs r600;
	struct evergreen_irq_stat_regs evergreen;
810
	struct cik_irq_stat_regs cik;
811 812
};

813
struct radeon_irq {
814 815
	bool				installed;
	spinlock_t			lock;
816
	atomic_t			ring_int[RADEON_NUM_RINGS];
817
	bool				crtc_vblank_int[RADEON_MAX_CRTCS];
818
	atomic_t			pflip[RADEON_MAX_CRTCS];
819 820 821 822
	wait_queue_head_t		vblank_queue;
	bool				hpd[RADEON_MAX_HPD_PINS];
	bool				afmt[RADEON_MAX_AFMT_BLOCKS];
	union radeon_irq_stat_regs	stat_regs;
823
	bool				dpm_thermal;
824 825 826 827
};

int radeon_irq_kms_init(struct radeon_device *rdev);
void radeon_irq_kms_fini(struct radeon_device *rdev);
828
void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev, int ring);
829
bool radeon_irq_kms_sw_irq_get_delayed(struct radeon_device *rdev, int ring);
830
void radeon_irq_kms_sw_irq_put(struct radeon_device *rdev, int ring);
831 832
void radeon_irq_kms_pflip_irq_get(struct radeon_device *rdev, int crtc);
void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc);
833 834 835 836
void radeon_irq_kms_enable_afmt(struct radeon_device *rdev, int block);
void radeon_irq_kms_disable_afmt(struct radeon_device *rdev, int block);
void radeon_irq_kms_enable_hpd(struct radeon_device *rdev, unsigned hpd_mask);
void radeon_irq_kms_disable_hpd(struct radeon_device *rdev, unsigned hpd_mask);
837 838

/*
839
 * CP & rings.
840
 */
841

842
struct radeon_ib {
843 844 845 846
	struct radeon_sa_bo		*sa_bo;
	uint32_t			length_dw;
	uint64_t			gpu_addr;
	uint32_t			*ptr;
847
	int				ring;
848
	struct radeon_fence		*fence;
849
	struct radeon_vm		*vm;
850
	bool				is_const_ib;
851
	struct radeon_sync		sync;
852 853
};

854
struct radeon_ring {
855
	struct radeon_bo	*ring_obj;
856
	volatile uint32_t	*ring;
857
	unsigned		rptr_offs;
858
	unsigned		rptr_save_reg;
859 860
	u64			next_rptr_gpu_addr;
	volatile u32		*next_rptr_cpu_addr;
861 862 863 864 865
	unsigned		wptr;
	unsigned		wptr_old;
	unsigned		ring_size;
	unsigned		ring_free_dw;
	int			count_dw;
866 867
	atomic_t		last_rptr;
	atomic64_t		last_activity;
868 869 870 871
	uint64_t		gpu_addr;
	uint32_t		align_mask;
	uint32_t		ptr_mask;
	bool			ready;
872
	u32			nop;
873
	u32			idx;
874 875
	u64			last_semaphore_signal_addr;
	u64			last_semaphore_wait_addr;
876 877 878 879 880
	/* for CIK queues */
	u32 me;
	u32 pipe;
	u32 queue;
	struct radeon_bo	*mqd_obj;
881
	u32 doorbell_index;
882 883 884 885 886 887 888 889 890
	unsigned		wptr_offs;
};

struct radeon_mec {
	struct radeon_bo	*hpd_eop_obj;
	u64			hpd_eop_gpu_addr;
	u32 num_pipe;
	u32 num_mec;
	u32 num_queue;
891 892
};

893 894 895
/*
 * VM
 */
896

897
/* maximum number of VMIDs */
898 899
#define RADEON_NUM_VM	16

900
/* number of entries in page table */
901
#define RADEON_VM_PTE_COUNT (1 << radeon_vm_block_size)
902

903 904 905 906 907
/* PTBs (Page Table Blocks) need to be aligned to 32K */
#define RADEON_VM_PTB_ALIGN_SIZE   32768
#define RADEON_VM_PTB_ALIGN_MASK (RADEON_VM_PTB_ALIGN_SIZE - 1)
#define RADEON_VM_PTB_ALIGN(a) (((a) + RADEON_VM_PTB_ALIGN_MASK) & ~RADEON_VM_PTB_ALIGN_MASK)

908 909 910 911 912 913
#define R600_PTE_VALID		(1 << 0)
#define R600_PTE_SYSTEM		(1 << 1)
#define R600_PTE_SNOOPED	(1 << 2)
#define R600_PTE_READABLE	(1 << 5)
#define R600_PTE_WRITEABLE	(1 << 6)

914 915 916 917 918
/* PTE (Page Table Entry) fragment field for different page sizes */
#define R600_PTE_FRAG_4KB	(0 << 7)
#define R600_PTE_FRAG_64KB	(4 << 7)
#define R600_PTE_FRAG_256KB	(6 << 7)

919 920 921
/* flags needed to be set so we can copy directly from the GART table */
#define R600_PTE_GART_MASK	( R600_PTE_READABLE | R600_PTE_WRITEABLE | \
				  R600_PTE_SYSTEM | R600_PTE_VALID )
922

923 924 925 926 927
struct radeon_vm_pt {
	struct radeon_bo		*bo;
	uint64_t			addr;
};

928 929 930 931 932 933 934 935 936
struct radeon_vm_id {
	unsigned		id;
	uint64_t		pd_gpu_addr;
	/* last flushed PD/PT update */
	struct radeon_fence	*flushed_updates;
	/* last use of vmid */
	struct radeon_fence	*last_id_use;
};

937
struct radeon_vm {
938 939
	struct mutex		mutex;

940
	struct rb_root		va;
941

942 943 944
	/* protecting invalidated and freed */
	spinlock_t		status_lock;

945
	/* BOs moved, but not yet updated in the PT */
946
	struct list_head	invalidated;
947

948
	/* BOs freed, but not yet updated in the PT */
949
	struct list_head	freed;
950

951 952 953
	/* BOs cleared in the PT */
	struct list_head	cleared;

954
	/* contains the page directory */
955 956
	struct radeon_bo	*page_directory;
	unsigned		max_pde_used;
957 958

	/* array of page tables, one for each page directory entry */
959
	struct radeon_vm_pt	*page_tables;
960

961
	struct radeon_bo_va	*ib_bo_va;
962

963 964
	/* for id and flush management per ring */
	struct radeon_vm_id	ids[RADEON_NUM_RINGS];
965 966 967
};

struct radeon_vm_manager {
968
	struct radeon_fence		*active[RADEON_NUM_VM];
969 970 971 972 973
	uint32_t			max_pfn;
	/* number of VMIDs */
	unsigned			nvm;
	/* vram base address for page table entry  */
	u64				vram_base_offset;
974 975
	/* is vm enabled? */
	bool				enabled;
976 977
	/* for hw to save the PD addr on suspend/resume */
	uint32_t			saved_table_addr[RADEON_NUM_VM];
978 979 980 981 982 983 984 985 986
};

/*
 * file private structure
 */
struct radeon_fpriv {
	struct radeon_vm		vm;
};

987 988 989 990
/*
 * R6xx+ IH ring
 */
struct r600_ih {
991
	struct radeon_bo	*ring_obj;
992 993 994 995 996
	volatile uint32_t	*ring;
	unsigned		rptr;
	unsigned		ring_size;
	uint64_t		gpu_addr;
	uint32_t		ptr_mask;
997
	atomic_t		lock;
998 999 1000
	bool                    enabled;
};

1001
/*
1002
 * RLC stuff
1003
 */
1004 1005 1006
#include "clearstate_defs.h"

struct radeon_rlc {
1007 1008 1009
	/* for power gating */
	struct radeon_bo	*save_restore_obj;
	uint64_t		save_restore_gpu_addr;
1010
	volatile uint32_t	*sr_ptr;
1011
	const u32               *reg_list;
1012
	u32                     reg_list_size;
1013 1014 1015
	/* for clear state */
	struct radeon_bo	*clear_state_obj;
	uint64_t		clear_state_gpu_addr;
1016
	volatile uint32_t	*cs_ptr;
1017
	const struct cs_section_def   *cs_data;
1018 1019 1020 1021 1022 1023
	u32                     clear_state_size;
	/* for cp tables */
	struct radeon_bo	*cp_table_obj;
	uint64_t		cp_table_gpu_addr;
	volatile uint32_t	*cp_table_ptr;
	u32                     cp_table_size;
1024 1025
};

1026
int radeon_ib_get(struct radeon_device *rdev, int ring,
1027 1028
		  struct radeon_ib *ib, struct radeon_vm *vm,
		  unsigned size);
1029
void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib *ib);
1030
int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib,
1031
		       struct radeon_ib *const_ib, bool hdp_flush);
1032 1033
int radeon_ib_pool_init(struct radeon_device *rdev);
void radeon_ib_pool_fini(struct radeon_device *rdev);
1034
int radeon_ib_ring_tests(struct radeon_device *rdev);
1035
/* Ring access between begin & end cannot sleep */
1036 1037
bool radeon_ring_supports_scratch_reg(struct radeon_device *rdev,
				      struct radeon_ring *ring);
1038 1039 1040
void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_ring *cp);
int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_ring *cp, unsigned ndw);
int radeon_ring_lock(struct radeon_device *rdev, struct radeon_ring *cp, unsigned ndw);
1041 1042 1043 1044
void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *cp,
			bool hdp_flush);
void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *cp,
			       bool hdp_flush);
1045
void radeon_ring_undo(struct radeon_ring *ring);
1046 1047
void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *cp);
int radeon_ring_test(struct radeon_device *rdev, struct radeon_ring *cp);
1048 1049
void radeon_ring_lockup_update(struct radeon_device *rdev,
			       struct radeon_ring *ring);
1050
bool radeon_ring_test_lockup(struct radeon_device *rdev, struct radeon_ring *ring);
1051 1052 1053 1054
unsigned radeon_ring_backup(struct radeon_device *rdev, struct radeon_ring *ring,
			    uint32_t **data);
int radeon_ring_restore(struct radeon_device *rdev, struct radeon_ring *ring,
			unsigned size, uint32_t *data);
1055
int radeon_ring_init(struct radeon_device *rdev, struct radeon_ring *cp, unsigned ring_size,
1056
		     unsigned rptr_offs, u32 nop);
1057
void radeon_ring_fini(struct radeon_device *rdev, struct radeon_ring *cp);
1058 1059


1060 1061 1062 1063 1064
/* r600 async dma */
void r600_dma_stop(struct radeon_device *rdev);
int r600_dma_resume(struct radeon_device *rdev);
void r600_dma_fini(struct radeon_device *rdev);

1065 1066 1067 1068
void cayman_dma_stop(struct radeon_device *rdev);
int cayman_dma_resume(struct radeon_device *rdev);
void cayman_dma_fini(struct radeon_device *rdev);

1069 1070 1071 1072 1073 1074
/*
 * CS.
 */
struct radeon_cs_chunk {
	uint32_t		length_dw;
	uint32_t		*kdata;
1075
	void __user		*user_ptr;
1076 1077 1078
};

struct radeon_cs_parser {
1079
	struct device		*dev;
1080 1081 1082 1083 1084 1085 1086 1087 1088 1089
	struct radeon_device	*rdev;
	struct drm_file		*filp;
	/* chunks */
	unsigned		nchunks;
	struct radeon_cs_chunk	*chunks;
	uint64_t		*chunks_array;
	/* IB */
	unsigned		idx;
	/* relocations */
	unsigned		nrelocs;
1090 1091
	struct radeon_bo_list	*relocs;
	struct radeon_bo_list	*vm_bos;
1092
	struct list_head	validated;
1093
	unsigned		dma_reloc_idx;
1094
	/* indices of various chunks */
1095 1096 1097 1098
	struct radeon_cs_chunk  *chunk_ib;
	struct radeon_cs_chunk  *chunk_relocs;
	struct radeon_cs_chunk  *chunk_flags;
	struct radeon_cs_chunk  *chunk_const_ib;
1099 1100
	struct radeon_ib	ib;
	struct radeon_ib	const_ib;
1101
	void			*track;
1102
	unsigned		family;
1103
	int			parser_error;
1104 1105 1106
	u32			cs_flags;
	u32			ring;
	s32			priority;
1107
	struct ww_acquire_ctx	ticket;
1108 1109
};

1110 1111
static inline u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx)
{
1112
	struct radeon_cs_chunk *ibc = p->chunk_ib;
1113 1114 1115 1116 1117 1118

	if (ibc->kdata)
		return ibc->kdata[idx];
	return p->ib.ptr[idx];
}

1119

1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139
struct radeon_cs_packet {
	unsigned	idx;
	unsigned	type;
	unsigned	reg;
	unsigned	opcode;
	int		count;
	unsigned	one_reg_wr;
};

typedef int (*radeon_packet0_check_t)(struct radeon_cs_parser *p,
				      struct radeon_cs_packet *pkt,
				      unsigned idx, unsigned reg);
typedef int (*radeon_packet3_check_t)(struct radeon_cs_parser *p,
				      struct radeon_cs_packet *pkt);


/*
 * AGP
 */
int radeon_agp_init(struct radeon_device *rdev);
1140
void radeon_agp_resume(struct radeon_device *rdev);
1141
void radeon_agp_suspend(struct radeon_device *rdev);
1142 1143 1144 1145 1146 1147 1148
void radeon_agp_fini(struct radeon_device *rdev);


/*
 * Writeback
 */
struct radeon_wb {
1149
	struct radeon_bo	*wb_obj;
1150 1151
	volatile uint32_t	*wb;
	uint64_t		gpu_addr;
1152
	bool                    enabled;
1153
	bool                    use_event;
1154 1155
};

1156
#define RADEON_WB_SCRATCH_OFFSET 0
1157
#define RADEON_WB_RING0_NEXT_RPTR 256
1158
#define RADEON_WB_CP_RPTR_OFFSET 1024
1159 1160
#define RADEON_WB_CP1_RPTR_OFFSET 1280
#define RADEON_WB_CP2_RPTR_OFFSET 1536
1161
#define R600_WB_DMA_RPTR_OFFSET   1792
1162
#define R600_WB_IH_WPTR_OFFSET   2048
1163
#define CAYMAN_WB_DMA1_RPTR_OFFSET   2304
1164
#define R600_WB_EVENT_OFFSET     3072
1165 1166
#define CIK_WB_CP1_WPTR_OFFSET     3328
#define CIK_WB_CP2_WPTR_OFFSET     3584
1167 1168
#define R600_WB_DMA_RING_TEST_OFFSET 3588
#define CAYMAN_WB_DMA1_RING_TEST_OFFSET 3592
1169

1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
/**
 * struct radeon_pm - power management datas
 * @max_bandwidth:      maximum bandwidth the gpu has (MByte/s)
 * @igp_sideport_mclk:  sideport memory clock Mhz (rs690,rs740,rs780,rs880)
 * @igp_system_mclk:    system clock Mhz (rs690,rs740,rs780,rs880)
 * @igp_ht_link_clk:    ht link clock Mhz (rs690,rs740,rs780,rs880)
 * @igp_ht_link_width:  ht link width in bits (rs690,rs740,rs780,rs880)
 * @k8_bandwidth:       k8 bandwidth the gpu has (MByte/s) (IGP)
 * @sideport_bandwidth: sideport bandwidth the gpu has (MByte/s) (IGP)
 * @ht_bandwidth:       ht bandwidth the gpu has (MByte/s) (IGP)
 * @core_bandwidth:     core GPU bandwidth the gpu has (MByte/s) (IGP)
1181
 * @sclk:          	GPU clock Mhz (core bandwidth depends of this clock)
1182 1183 1184
 * @needed_bandwidth:   current bandwidth needs
 *
 * It keeps track of various data needed to take powermanagement decision.
1185
 * Bandwidth need is used to determine minimun clock of the GPU and memory.
1186 1187 1188
 * Equation between gpu/memory clock and available bandwidth is hw dependent
 * (type of memory, bus size, efficiency, ...)
 */
1189 1190 1191 1192

enum radeon_pm_method {
	PM_METHOD_PROFILE,
	PM_METHOD_DYNPM,
1193
	PM_METHOD_DPM,
1194 1195 1196 1197 1198 1199
};

enum radeon_dynpm_state {
	DYNPM_STATE_DISABLED,
	DYNPM_STATE_MINIMUM,
	DYNPM_STATE_PAUSED,
1200 1201
	DYNPM_STATE_ACTIVE,
	DYNPM_STATE_SUSPENDED,
1202
};
1203 1204 1205 1206 1207 1208
enum radeon_dynpm_action {
	DYNPM_ACTION_NONE,
	DYNPM_ACTION_MINIMUM,
	DYNPM_ACTION_DOWNCLOCK,
	DYNPM_ACTION_UPCLOCK,
	DYNPM_ACTION_DEFAULT
1209
};
1210 1211 1212 1213 1214 1215 1216 1217

enum radeon_voltage_type {
	VOLTAGE_NONE = 0,
	VOLTAGE_GPIO,
	VOLTAGE_VDDC,
	VOLTAGE_SW
};

1218
enum radeon_pm_state_type {
1219
	/* not used for dpm */
1220 1221
	POWER_STATE_TYPE_DEFAULT,
	POWER_STATE_TYPE_POWERSAVE,
1222
	/* user selectable states */
1223 1224 1225
	POWER_STATE_TYPE_BATTERY,
	POWER_STATE_TYPE_BALANCED,
	POWER_STATE_TYPE_PERFORMANCE,
1226 1227 1228 1229 1230 1231 1232 1233 1234 1235
	/* internal states */
	POWER_STATE_TYPE_INTERNAL_UVD,
	POWER_STATE_TYPE_INTERNAL_UVD_SD,
	POWER_STATE_TYPE_INTERNAL_UVD_HD,
	POWER_STATE_TYPE_INTERNAL_UVD_HD2,
	POWER_STATE_TYPE_INTERNAL_UVD_MVC,
	POWER_STATE_TYPE_INTERNAL_BOOT,
	POWER_STATE_TYPE_INTERNAL_THERMAL,
	POWER_STATE_TYPE_INTERNAL_ACPI,
	POWER_STATE_TYPE_INTERNAL_ULV,
1236
	POWER_STATE_TYPE_INTERNAL_3DPERF,
1237 1238
};

1239 1240 1241 1242
enum radeon_pm_profile_type {
	PM_PROFILE_DEFAULT,
	PM_PROFILE_AUTO,
	PM_PROFILE_LOW,
1243
	PM_PROFILE_MID,
1244 1245 1246 1247 1248
	PM_PROFILE_HIGH,
};

#define PM_PROFILE_DEFAULT_IDX 0
#define PM_PROFILE_LOW_SH_IDX  1
1249 1250 1251 1252 1253 1254
#define PM_PROFILE_MID_SH_IDX  2
#define PM_PROFILE_HIGH_SH_IDX 3
#define PM_PROFILE_LOW_MH_IDX  4
#define PM_PROFILE_MID_MH_IDX  5
#define PM_PROFILE_HIGH_MH_IDX 6
#define PM_PROFILE_MAX         7
1255 1256 1257 1258 1259 1260

struct radeon_pm_profile {
	int dpms_off_ps_idx;
	int dpms_on_ps_idx;
	int dpms_off_cm_idx;
	int dpms_on_cm_idx;
1261 1262
};

1263 1264
enum radeon_int_thermal_type {
	THERMAL_TYPE_NONE,
1265 1266
	THERMAL_TYPE_EXTERNAL,
	THERMAL_TYPE_EXTERNAL_GPIO,
1267 1268
	THERMAL_TYPE_RV6XX,
	THERMAL_TYPE_RV770,
1269
	THERMAL_TYPE_ADT7473_WITH_INTERNAL,
1270
	THERMAL_TYPE_EVERGREEN,
1271
	THERMAL_TYPE_SUMO,
1272
	THERMAL_TYPE_NI,
1273
	THERMAL_TYPE_SI,
1274
	THERMAL_TYPE_EMC2103_WITH_INTERNAL,
1275
	THERMAL_TYPE_CI,
1276
	THERMAL_TYPE_KV,
1277 1278
};

1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289
struct radeon_voltage {
	enum radeon_voltage_type type;
	/* gpio voltage */
	struct radeon_gpio_rec gpio;
	u32 delay; /* delay in usec from voltage drop to sclk change */
	bool active_high; /* voltage drop is active when bit is high */
	/* VDDC voltage */
	u8 vddc_id; /* index into vddc voltage table */
	u8 vddci_id; /* index into vddci voltage table */
	bool vddci_enabled;
	/* r6xx+ sw */
1290 1291 1292
	u16 voltage;
	/* evergreen+ vddci */
	u16 vddci;
1293 1294
};

1295 1296 1297
/* clock mode flags */
#define RADEON_PM_MODE_NO_DISPLAY          (1 << 0)

1298 1299 1300 1301 1302 1303 1304
struct radeon_pm_clock_info {
	/* memory clock */
	u32 mclk;
	/* engine clock */
	u32 sclk;
	/* voltage info */
	struct radeon_voltage voltage;
1305
	/* standardized clock flags */
1306 1307 1308
	u32 flags;
};

1309
/* state flags */
1310
#define RADEON_PM_STATE_SINGLE_DISPLAY_ONLY (1 << 0)
1311

1312
struct radeon_power_state {
1313
	enum radeon_pm_state_type type;
1314
	struct radeon_pm_clock_info *clock_info;
1315 1316 1317
	/* number of valid clock modes in this power state */
	int num_clock_modes;
	struct radeon_pm_clock_info *default_clock_mode;
1318 1319
	/* standardized state flags */
	u32 flags;
1320 1321 1322
	u32 misc; /* vbios specific flags */
	u32 misc2; /* vbios specific flags */
	int pcie_lanes; /* pcie lanes */
1323 1324
};

1325 1326 1327 1328 1329
/*
 * Some modes are overclocked by very low value, accept them
 */
#define RADEON_MODE_OVERCLOCK_MARGIN 500 /* 5 MHz */

1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342
enum radeon_dpm_auto_throttle_src {
	RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL,
	RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL
};

enum radeon_dpm_event_src {
	RADEON_DPM_EVENT_SRC_ANALOG = 0,
	RADEON_DPM_EVENT_SRC_EXTERNAL = 1,
	RADEON_DPM_EVENT_SRC_DIGITAL = 2,
	RADEON_DPM_EVENT_SRC_ANALOG_OR_EXTERNAL = 3,
	RADEON_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL = 4
};

1343 1344
#define RADEON_MAX_VCE_LEVELS 6

1345 1346 1347 1348 1349 1350 1351 1352 1353
enum radeon_vce_level {
	RADEON_VCE_LEVEL_AC_ALL = 0,     /* AC, All cases */
	RADEON_VCE_LEVEL_DC_EE = 1,      /* DC, entropy encoding */
	RADEON_VCE_LEVEL_DC_LL_LOW = 2,  /* DC, low latency queue, res <= 720 */
	RADEON_VCE_LEVEL_DC_LL_HIGH = 3, /* DC, low latency queue, 1080 >= res > 720 */
	RADEON_VCE_LEVEL_DC_GP_LOW = 4,  /* DC, general purpose queue, res <= 720 */
	RADEON_VCE_LEVEL_DC_GP_HIGH = 5, /* DC, general purpose queue, 1080 >= res > 720 */
};

1354 1355 1356 1357 1358 1359 1360
struct radeon_ps {
	u32 caps; /* vbios flags */
	u32 class; /* vbios flags */
	u32 class2; /* vbios flags */
	/* UVD clocks */
	u32 vclk;
	u32 dclk;
1361 1362 1363
	/* VCE clocks */
	u32 evclk;
	u32 ecclk;
1364 1365
	bool vce_active;
	enum radeon_vce_level vce_level;
1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380
	/* asic priv */
	void *ps_priv;
};

struct radeon_dpm_thermal {
	/* thermal interrupt work */
	struct work_struct work;
	/* low temperature threshold */
	int                min_temp;
	/* high temperature threshold */
	int                max_temp;
	/* was interrupt low to high or high to low */
	bool               high_to_low;
};

1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393
enum radeon_clk_action
{
	RADEON_SCLK_UP = 1,
	RADEON_SCLK_DOWN
};

struct radeon_blacklist_clocks
{
	u32 sclk;
	u32 mclk;
	enum radeon_clk_action action;
};

1394 1395 1396
struct radeon_clock_and_voltage_limits {
	u32 sclk;
	u32 mclk;
1397 1398
	u16 vddc;
	u16 vddci;
1399 1400 1401 1402 1403 1404 1405