x86_pkg_temp_thermal.c 14.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57
/*
 * x86_pkg_temp_thermal driver
 * Copyright (c) 2013, Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc.
 *
 */
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/init.h>
#include <linux/err.h>
#include <linux/param.h>
#include <linux/device.h>
#include <linux/platform_device.h>
#include <linux/cpu.h>
#include <linux/smp.h>
#include <linux/slab.h>
#include <linux/pm.h>
#include <linux/thermal.h>
#include <linux/debugfs.h>
#include <asm/cpu_device_id.h>
#include <asm/mce.h>

/*
* Rate control delay: Idea is to introduce denounce effect
* This should be long enough to avoid reduce events, when
* threshold is set to a temperature, which is constantly
* violated, but at the short enough to take any action.
* The action can be remove threshold or change it to next
* interesting setting. Based on experiments, in around
* every 5 seconds under load will give us a significant
* temperature change.
*/
#define PKG_TEMP_THERMAL_NOTIFY_DELAY	5000
static int notify_delay_ms = PKG_TEMP_THERMAL_NOTIFY_DELAY;
module_param(notify_delay_ms, int, 0644);
MODULE_PARM_DESC(notify_delay_ms,
	"User space notification delay in milli seconds.");

/* Number of trip points in thermal zone. Currently it can't
* be more than 2. MSR can allow setting and getting notifications
* for only 2 thresholds. This define enforces this, if there
* is some wrong values returned by cpuid for number of thresholds.
*/
#define MAX_NUMBER_OF_TRIPS	2

58
struct pkg_device {
59
	int				cpu;
60
	bool				work_scheduled;
61 62 63
	u32				tj_max;
	u32				msr_pkg_therm_low;
	u32				msr_pkg_therm_high;
64
	struct delayed_work		work;
65
	struct thermal_zone_device	*tzone;
66
	struct cpumask			cpumask;
67 68
};

69
static struct thermal_zone_params pkg_temp_tz_params = {
70 71 72
	.no_hwmon	= true,
};

73 74 75 76
/* Keep track of how many package pointers we allocated in init() */
static int max_packages __read_mostly;
/* Array of package pointers */
static struct pkg_device **packages;
77 78 79 80
/* Serializes interrupt notification, work and hotplug */
static DEFINE_SPINLOCK(pkg_temp_lock);
/* Protects zone operation in the work function against hotplug removal */
static DEFINE_MUTEX(thermal_zone_mutex);
81

82 83 84
/* The dynamically assigned cpu hotplug state for module_exit() */
static enum cpuhp_state pkg_thermal_hp_state __read_mostly;

85 86 87 88 89 90 91 92 93 94 95 96 97 98
/* Debug counters to show using debugfs */
static struct dentry *debugfs;
static unsigned int pkg_interrupt_cnt;
static unsigned int pkg_work_cnt;

static int pkg_temp_debugfs_init(void)
{
	struct dentry *d;

	debugfs = debugfs_create_dir("pkg_temp_thermal", NULL);
	if (!debugfs)
		return -ENOENT;

	d = debugfs_create_u32("pkg_thres_interrupt", S_IRUGO, debugfs,
99
			       &pkg_interrupt_cnt);
100 101 102 103
	if (!d)
		goto err_out;

	d = debugfs_create_u32("pkg_thres_work", S_IRUGO, debugfs,
104
			       &pkg_work_cnt);
105 106 107 108 109 110 111 112 113 114
	if (!d)
		goto err_out;

	return 0;

err_out:
	debugfs_remove_recursive(debugfs);
	return -ENOENT;
}

115 116 117 118 119 120 121 122
/*
 * Protection:
 *
 * - cpu hotplug: Read serialized by cpu hotplug lock
 *		  Write must hold pkg_temp_lock
 *
 * - Other callsites: Must hold pkg_temp_lock
 */
123
static struct pkg_device *pkg_temp_thermal_get_dev(unsigned int cpu)
124
{
125
	int pkgid = topology_logical_package_id(cpu);
126

127 128
	if (pkgid >= 0 && pkgid < max_packages)
		return packages[pkgid];
129 130 131 132 133 134 135 136 137
	return NULL;
}

/*
* tj-max is is interesting because threshold is set relative to this
* temperature.
*/
static int get_tj_max(int cpu, u32 *tj_max)
{
138
	u32 eax, edx, val;
139 140 141 142
	int err;

	err = rdmsr_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, &eax, &edx);
	if (err)
143
		return err;
144

145 146 147 148
	val = (eax >> 16) & 0xff;
	*tj_max = val * 1000;

	return val ? 0 : -EINVAL;
149 150
}

151
static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp)
152
{
153
	struct pkg_device *pkgdev = tzd->devdata;
154 155
	u32 eax, edx;

156
	rdmsr_on_cpu(pkgdev->cpu, MSR_IA32_PACKAGE_THERM_STATUS, &eax, &edx);
157
	if (eax & 0x80000000) {
158
		*temp = pkgdev->tj_max - ((eax >> 16) & 0x7f) * 1000;
159
		pr_debug("sys_get_curr_temp %d\n", *temp);
160 161 162 163 164 165
		return 0;
	}
	return -EINVAL;
}

static int sys_get_trip_temp(struct thermal_zone_device *tzd,
166
			     int trip, int *temp)
167
{
168
	struct pkg_device *pkgdev = tzd->devdata;
169
	unsigned long thres_reg_value;
170
	u32 mask, shift, eax, edx;
171 172 173 174 175 176 177 178 179 180 181 182 183
	int ret;

	if (trip >= MAX_NUMBER_OF_TRIPS)
		return -EINVAL;

	if (trip) {
		mask = THERM_MASK_THRESHOLD1;
		shift = THERM_SHIFT_THRESHOLD1;
	} else {
		mask = THERM_MASK_THRESHOLD0;
		shift = THERM_SHIFT_THRESHOLD0;
	}

184
	ret = rdmsr_on_cpu(pkgdev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
185
			   &eax, &edx);
186
	if (ret < 0)
187
		return ret;
188 189 190

	thres_reg_value = (eax & mask) >> shift;
	if (thres_reg_value)
191
		*temp = pkgdev->tj_max - thres_reg_value * 1000;
192 193
	else
		*temp = 0;
194
	pr_debug("sys_get_trip_temp %d\n", *temp);
195 196 197 198

	return 0;
}

199 200
static int
sys_set_trip_temp(struct thermal_zone_device *tzd, int trip, int temp)
201
{
202 203
	struct pkg_device *pkgdev = tzd->devdata;
	u32 l, h, mask, shift, intr;
204 205
	int ret;

206
	if (trip >= MAX_NUMBER_OF_TRIPS || temp >= pkgdev->tj_max)
207 208
		return -EINVAL;

209
	ret = rdmsr_on_cpu(pkgdev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
210
			   &l, &h);
211
	if (ret < 0)
212
		return ret;
213 214 215 216 217 218 219 220 221 222 223 224 225 226 227

	if (trip) {
		mask = THERM_MASK_THRESHOLD1;
		shift = THERM_SHIFT_THRESHOLD1;
		intr = THERM_INT_THRESHOLD1_ENABLE;
	} else {
		mask = THERM_MASK_THRESHOLD0;
		shift = THERM_SHIFT_THRESHOLD0;
		intr = THERM_INT_THRESHOLD0_ENABLE;
	}
	l &= ~mask;
	/*
	* When users space sets a trip temperature == 0, which is indication
	* that, it is no longer interested in receiving notifications.
	*/
228
	if (!temp) {
229
		l &= ~intr;
230 231
	} else {
		l |= (pkgdev->tj_max - temp)/1000 << shift;
232 233 234
		l |= intr;
	}

235
	return wrmsr_on_cpu(pkgdev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
236 237
}

238 239
static int sys_get_trip_type(struct thermal_zone_device *thermal, int trip,
			     enum thermal_trip_type *type)
240 241 242 243 244 245 246 247 248 249 250 251 252
{
	*type = THERMAL_TRIP_PASSIVE;
	return 0;
}

/* Thermal zone callback registry */
static struct thermal_zone_device_ops tzone_ops = {
	.get_temp = sys_get_curr_temp,
	.get_trip_temp = sys_get_trip_temp,
	.get_trip_type = sys_get_trip_type,
	.set_trip_temp = sys_set_trip_temp,
};

253
static bool pkg_thermal_rate_control(void)
254 255 256 257 258 259 260 261
{
	return true;
}

/* Enable threshold interrupt on local package/cpu */
static inline void enable_pkg_thres_interrupt(void)
{
	u8 thres_0, thres_1;
262
	u32 l, h;
263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278

	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
	/* only enable/disable if it had valid threshold value */
	thres_0 = (l & THERM_MASK_THRESHOLD0) >> THERM_SHIFT_THRESHOLD0;
	thres_1 = (l & THERM_MASK_THRESHOLD1) >> THERM_SHIFT_THRESHOLD1;
	if (thres_0)
		l |= THERM_INT_THRESHOLD0_ENABLE;
	if (thres_1)
		l |= THERM_INT_THRESHOLD1_ENABLE;
	wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
}

/* Disable threshold interrupt on local package/cpu */
static inline void disable_pkg_thres_interrupt(void)
{
	u32 l, h;
279

280
	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
281 282 283

	l &= ~(THERM_INT_THRESHOLD0_ENABLE | THERM_INT_THRESHOLD1_ENABLE);
	wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
284 285 286 287
}

static void pkg_temp_thermal_threshold_work_fn(struct work_struct *work)
{
288
	struct thermal_zone_device *tzone = NULL;
289
	int cpu = smp_processor_id();
290
	struct pkg_device *pkgdev;
291
	u64 msr_val, wr_val;
292

293 294
	mutex_lock(&thermal_zone_mutex);
	spin_lock_irq(&pkg_temp_lock);
295
	++pkg_work_cnt;
296 297 298 299 300

	pkgdev = pkg_temp_thermal_get_dev(cpu);
	if (!pkgdev) {
		spin_unlock_irq(&pkg_temp_lock);
		mutex_unlock(&thermal_zone_mutex);
301 302
		return;
	}
303
	pkgdev->work_scheduled = false;
304 305

	rdmsrl(MSR_IA32_PACKAGE_THERM_STATUS, msr_val);
306 307 308
	wr_val = msr_val & ~(THERM_LOG_THRESHOLD0 | THERM_LOG_THRESHOLD1);
	if (wr_val != msr_val) {
		wrmsrl(MSR_IA32_PACKAGE_THERM_STATUS, wr_val);
309
		tzone = pkgdev->tzone;
310
	}
311 312

	enable_pkg_thres_interrupt();
313
	spin_unlock_irq(&pkg_temp_lock);
314

315 316 317 318 319 320 321 322
	/*
	 * If tzone is not NULL, then thermal_zone_mutex will prevent the
	 * concurrent removal in the cpu offline callback.
	 */
	if (tzone)
		thermal_zone_device_update(tzone, THERMAL_EVENT_UNSPECIFIED);

	mutex_unlock(&thermal_zone_mutex);
323 324
}

325 326 327 328 329 330 331
static void pkg_thermal_schedule_work(int cpu, struct delayed_work *work)
{
	unsigned long ms = msecs_to_jiffies(notify_delay_ms);

	schedule_delayed_work_on(cpu, work, ms);
}

332
static int pkg_thermal_notify(u64 msr_val)
333 334
{
	int cpu = smp_processor_id();
335
	struct pkg_device *pkgdev;
336
	unsigned long flags;
337

338
	spin_lock_irqsave(&pkg_temp_lock, flags);
339 340 341
	++pkg_interrupt_cnt;

	disable_pkg_thres_interrupt();
342 343 344

	/* Work is per package, so scheduling it once is enough. */
	pkgdev = pkg_temp_thermal_get_dev(cpu);
345 346
	if (pkgdev && !pkgdev->work_scheduled) {
		pkgdev->work_scheduled = true;
347
		pkg_thermal_schedule_work(pkgdev->cpu, &pkgdev->work);
348 349 350
	}

	spin_unlock_irqrestore(&pkg_temp_lock, flags);
351 352 353 354 355
	return 0;
}

static int pkg_temp_thermal_device_add(unsigned int cpu)
{
356
	int pkgid = topology_logical_package_id(cpu);
357 358 359
	u32 tj_max, eax, ebx, ecx, edx;
	struct pkg_device *pkgdev;
	int thres_count, err;
360

361 362 363
	if (pkgid >= max_packages)
		return -ENOMEM;

364 365 366 367 368 369 370 371 372
	cpuid(6, &eax, &ebx, &ecx, &edx);
	thres_count = ebx & 0x07;
	if (!thres_count)
		return -ENODEV;

	thres_count = clamp_val(thres_count, 0, MAX_NUMBER_OF_TRIPS);

	err = get_tj_max(cpu, &tj_max);
	if (err)
373
		return err;
374

375
	pkgdev = kzalloc(sizeof(*pkgdev), GFP_KERNEL);
376 377
	if (!pkgdev)
		return -ENOMEM;
378

379
	INIT_DELAYED_WORK(&pkgdev->work, pkg_temp_thermal_threshold_work_fn);
380 381 382
	pkgdev->cpu = cpu;
	pkgdev->tj_max = tj_max;
	pkgdev->tzone = thermal_zone_device_register("x86_pkg_temp",
383
			thres_count,
384 385 386 387
			(thres_count == MAX_NUMBER_OF_TRIPS) ? 0x03 : 0x01,
			pkgdev, &tzone_ops, &pkg_temp_tz_params, 0, 0);
	if (IS_ERR(pkgdev->tzone)) {
		err = PTR_ERR(pkgdev->tzone);
388 389
		kfree(pkgdev);
		return err;
390 391
	}
	/* Store MSR value for package thermal interrupt, to restore at exit */
392 393
	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, pkgdev->msr_pkg_therm_low,
	      pkgdev->msr_pkg_therm_high);
394

395 396
	cpumask_set_cpu(cpu, &pkgdev->cpumask);
	spin_lock_irq(&pkg_temp_lock);
397
	packages[pkgid] = pkgdev;
398
	spin_unlock_irq(&pkg_temp_lock);
399 400 401
	return 0;
}

402
static int pkg_thermal_cpu_offline(unsigned int cpu)
403
{
404
	struct pkg_device *pkgdev = pkg_temp_thermal_get_dev(cpu);
405
	bool lastcpu, was_target;
406
	int target;
407

408
	if (!pkgdev)
409
		return 0;
410 411 412 413 414 415 416 417 418 419

	target = cpumask_any_but(&pkgdev->cpumask, cpu);
	cpumask_clear_cpu(cpu, &pkgdev->cpumask);
	lastcpu = target >= nr_cpu_ids;
	/*
	 * Remove the sysfs files, if this is the last cpu in the package
	 * before doing further cleanups.
	 */
	if (lastcpu) {
		struct thermal_zone_device *tzone = pkgdev->tzone;
420

421 422 423 424 425 426 427 428 429
		/*
		 * We must protect against a work function calling
		 * thermal_zone_update, after/while unregister. We null out
		 * the pointer under the zone mutex, so the worker function
		 * won't try to call.
		 */
		mutex_lock(&thermal_zone_mutex);
		pkgdev->tzone = NULL;
		mutex_unlock(&thermal_zone_mutex);
430

431 432 433
		thermal_zone_device_unregister(tzone);
	}

434 435 436
	/* Protect against work and interrupts */
	spin_lock_irq(&pkg_temp_lock);

437
	/*
438 439 440 441 442 443 444 445 446
	 * Check whether this cpu was the current target and store the new
	 * one. When we drop the lock, then the interrupt notify function
	 * will see the new target.
	 */
	was_target = pkgdev->cpu == cpu;
	pkgdev->cpu = target;

	/*
	 * If this is the last CPU in the package remove the package
447
	 * reference from the array and restore the interrupt MSR. When we
448 449
	 * drop the lock neither the interrupt notify function nor the
	 * worker will see the package anymore.
450 451
	 */
	if (lastcpu) {
452
		packages[topology_logical_package_id(cpu)] = NULL;
453 454 455
		/* After this point nothing touches the MSR anymore. */
		wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
		      pkgdev->msr_pkg_therm_low, pkgdev->msr_pkg_therm_high);
456
	}
457

458
	/*
459 460
	 * Check whether there is work scheduled and whether the work is
	 * targeted at the outgoing CPU.
461
	 */
462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484
	if (pkgdev->work_scheduled && was_target) {
		/*
		 * To cancel the work we need to drop the lock, otherwise
		 * we might deadlock if the work needs to be flushed.
		 */
		spin_unlock_irq(&pkg_temp_lock);
		cancel_delayed_work_sync(&pkgdev->work);
		spin_lock_irq(&pkg_temp_lock);
		/*
		 * If this is not the last cpu in the package and the work
		 * did not run after we dropped the lock above, then we
		 * need to reschedule the work, otherwise the interrupt
		 * stays disabled forever.
		 */
		if (!lastcpu && pkgdev->work_scheduled)
			pkg_thermal_schedule_work(target, &pkgdev->work);
	}

	spin_unlock_irq(&pkg_temp_lock);

	/* Final cleanup if this is the last cpu */
	if (lastcpu)
		kfree(pkgdev);
485
	return 0;
486 487
}

488
static int pkg_thermal_cpu_online(unsigned int cpu)
489
{
490
	struct pkg_device *pkgdev = pkg_temp_thermal_get_dev(cpu);
491 492
	struct cpuinfo_x86 *c = &cpu_data(cpu);

493 494 495
	/* Paranoia check */
	if (!cpu_has(c, X86_FEATURE_DTHERM) || !cpu_has(c, X86_FEATURE_PTS))
		return -ENODEV;
496

497
	/* If the package exists, nothing to do */
498 499
	if (pkgdev) {
		cpumask_set_cpu(cpu, &pkgdev->cpumask);
500
		return 0;
501
	}
502
	return pkg_temp_thermal_device_add(cpu);
503 504 505
}

static const struct x86_cpu_id __initconst pkg_temp_thermal_ids[] = {
506
	{ X86_VENDOR_INTEL, X86_FAMILY_ANY, X86_MODEL_ANY, X86_FEATURE_PTS },
507 508 509 510 511 512
	{}
};
MODULE_DEVICE_TABLE(x86cpu, pkg_temp_thermal_ids);

static int __init pkg_temp_thermal_init(void)
{
513
	int ret;
514 515 516 517

	if (!x86_match_cpu(pkg_temp_thermal_ids))
		return -ENODEV;

518
	max_packages = topology_max_packages();
519 520
	packages = kcalloc(max_packages, sizeof(struct pkg_device *),
			   GFP_KERNEL);
521 522 523
	if (!packages)
		return -ENOMEM;

524 525 526 527 528 529 530
	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "thermal/x86_pkg:online",
				pkg_thermal_cpu_online,	pkg_thermal_cpu_offline);
	if (ret < 0)
		goto err;

	/* Store the state for module exit */
	pkg_thermal_hp_state = ret;
531

532 533 534
	platform_thermal_package_notify = pkg_thermal_notify;
	platform_thermal_package_rate_control = pkg_thermal_rate_control;

535 536
	 /* Don't care if it fails */
	pkg_temp_debugfs_init();
537 538
	return 0;

539
err:
540
	kfree(packages);
541
	return ret;
542
}
543
module_init(pkg_temp_thermal_init)
544 545 546

static void __exit pkg_temp_thermal_exit(void)
{
547 548 549
	platform_thermal_package_notify = NULL;
	platform_thermal_package_rate_control = NULL;

550
	cpuhp_remove_state(pkg_thermal_hp_state);
551
	debugfs_remove_recursive(debugfs);
552
	kfree(packages);
553 554 555 556 557 558
}
module_exit(pkg_temp_thermal_exit)

MODULE_DESCRIPTION("X86 PKG TEMP Thermal Driver");
MODULE_AUTHOR("Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>");
MODULE_LICENSE("GPL v2");