edac_mc.c 31.4 KB
Newer Older
1 2
/*
 * edac_mc kernel module
3
 * (C) 2005, 2006 Linux Networx (http://lnxi.com)
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
 * This file may be distributed under the terms of the
 * GNU General Public License.
 *
 * Written by Thayne Harbaugh
 * Based on work by Dan Hollis <goemon at anime dot net> and others.
 *	http://www.anime.net/~goemon/linux-ecc/
 *
 * Modified by Dave Peterson and Doug Thompson
 *
 */

#include <linux/module.h>
#include <linux/proc_fs.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/smp.h>
#include <linux/init.h>
#include <linux/sysctl.h>
#include <linux/highmem.h>
#include <linux/timer.h>
#include <linux/slab.h>
#include <linux/jiffies.h>
#include <linux/spinlock.h>
#include <linux/list.h>
#include <linux/ctype.h>
29
#include <linux/edac.h>
30
#include <linux/bitops.h>
31
#include <linux/uaccess.h>
32
#include <asm/page.h>
33
#include "edac_mc.h"
34
#include "edac_module.h"
35 36
#include <ras/ras_event.h>

37 38 39 40 41 42
#ifdef CONFIG_EDAC_ATOMIC_SCRUB
#include <asm/edac.h>
#else
#define edac_atomic_scrub(va, size) do { } while (0)
#endif

43 44 45
int edac_op_state = EDAC_OPSTATE_INVAL;
EXPORT_SYMBOL_GPL(edac_op_state);

46 47
static int edac_report = EDAC_REPORTING_ENABLED;

48
/* lock to memory controller's control array */
49
static DEFINE_MUTEX(mem_ctls_mutex);
50
static LIST_HEAD(mc_devices);
51

52 53 54 55
/*
 * Used to lock EDAC MC to just one module, avoiding two drivers e. g.
 *	apei/ghes and i7core_edac to be used at the same time.
 */
56
static const char *edac_mc_owner;
57

58 59
static struct bus_type mc_bus[EDAC_MAX_MCS];

60
int edac_get_report_status(void)
61 62 63
{
	return edac_report;
}
64
EXPORT_SYMBOL_GPL(edac_get_report_status);
65

66
void edac_set_report_status(int new)
67 68 69 70 71 72
{
	if (new == EDAC_REPORTING_ENABLED ||
	    new == EDAC_REPORTING_DISABLED ||
	    new == EDAC_REPORTING_FORCE)
		edac_report = new;
}
73
EXPORT_SYMBOL_GPL(edac_set_report_status);
74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118

static int edac_report_set(const char *str, const struct kernel_param *kp)
{
	if (!str)
		return -EINVAL;

	if (!strncmp(str, "on", 2))
		edac_report = EDAC_REPORTING_ENABLED;
	else if (!strncmp(str, "off", 3))
		edac_report = EDAC_REPORTING_DISABLED;
	else if (!strncmp(str, "force", 5))
		edac_report = EDAC_REPORTING_FORCE;

	return 0;
}

static int edac_report_get(char *buffer, const struct kernel_param *kp)
{
	int ret = 0;

	switch (edac_report) {
	case EDAC_REPORTING_ENABLED:
		ret = sprintf(buffer, "on");
		break;
	case EDAC_REPORTING_DISABLED:
		ret = sprintf(buffer, "off");
		break;
	case EDAC_REPORTING_FORCE:
		ret = sprintf(buffer, "force");
		break;
	default:
		ret = -EINVAL;
		break;
	}

	return ret;
}

static const struct kernel_param_ops edac_report_ops = {
	.set = edac_report_set,
	.get = edac_report_get,
};

module_param_cb(edac_report, &edac_report_ops, &edac_report, 0644);

119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139
unsigned edac_dimm_info_location(struct dimm_info *dimm, char *buf,
			         unsigned len)
{
	struct mem_ctl_info *mci = dimm->mci;
	int i, n, count = 0;
	char *p = buf;

	for (i = 0; i < mci->n_layers; i++) {
		n = snprintf(p, len, "%s %d ",
			      edac_layer_name[mci->layers[i].type],
			      dimm->location[i]);
		p += n;
		len -= n;
		count += n;
		if (!len)
			break;
	}

	return count;
}

140 141
#ifdef CONFIG_EDAC_DEBUG

142
static void edac_mc_dump_channel(struct rank_info *chan)
143
{
144 145 146 147
	edac_dbg(4, "  channel->chan_idx = %d\n", chan->chan_idx);
	edac_dbg(4, "    channel = %p\n", chan);
	edac_dbg(4, "    channel->csrow = %p\n", chan->csrow);
	edac_dbg(4, "    channel->dimm = %p\n", chan->dimm);
148 149
}

150
static void edac_mc_dump_dimm(struct dimm_info *dimm, int number)
151
{
152 153 154 155 156
	char location[80];

	edac_dimm_info_location(dimm, location, sizeof(location));

	edac_dbg(4, "%s%i: %smapped as virtual row %d, chan %d\n",
157
		 dimm->mci->csbased ? "rank" : "dimm",
158 159 160 161 162 163
		 number, location, dimm->csrow, dimm->cschannel);
	edac_dbg(4, "  dimm = %p\n", dimm);
	edac_dbg(4, "  dimm->label = '%s'\n", dimm->label);
	edac_dbg(4, "  dimm->nr_pages = 0x%x\n", dimm->nr_pages);
	edac_dbg(4, "  dimm->grain = %d\n", dimm->grain);
	edac_dbg(4, "  dimm->nr_pages = 0x%x\n", dimm->nr_pages);
164 165
}

166
static void edac_mc_dump_csrow(struct csrow_info *csrow)
167
{
168 169 170 171 172 173 174 175
	edac_dbg(4, "csrow->csrow_idx = %d\n", csrow->csrow_idx);
	edac_dbg(4, "  csrow = %p\n", csrow);
	edac_dbg(4, "  csrow->first_page = 0x%lx\n", csrow->first_page);
	edac_dbg(4, "  csrow->last_page = 0x%lx\n", csrow->last_page);
	edac_dbg(4, "  csrow->page_mask = 0x%lx\n", csrow->page_mask);
	edac_dbg(4, "  csrow->nr_channels = %d\n", csrow->nr_channels);
	edac_dbg(4, "  csrow->channels = %p\n", csrow->channels);
	edac_dbg(4, "  csrow->mci = %p\n", csrow->mci);
176 177
}

178
static void edac_mc_dump_mci(struct mem_ctl_info *mci)
179
{
180 181 182 183 184 185 186 187 188 189 190 191 192
	edac_dbg(3, "\tmci = %p\n", mci);
	edac_dbg(3, "\tmci->mtype_cap = %lx\n", mci->mtype_cap);
	edac_dbg(3, "\tmci->edac_ctl_cap = %lx\n", mci->edac_ctl_cap);
	edac_dbg(3, "\tmci->edac_cap = %lx\n", mci->edac_cap);
	edac_dbg(4, "\tmci->edac_check = %p\n", mci->edac_check);
	edac_dbg(3, "\tmci->nr_csrows = %d, csrows = %p\n",
		 mci->nr_csrows, mci->csrows);
	edac_dbg(3, "\tmci->nr_dimms = %d, dimms = %p\n",
		 mci->tot_dimms, mci->dimms);
	edac_dbg(3, "\tdev = %p\n", mci->pdev);
	edac_dbg(3, "\tmod_name:ctl_name = %s:%s\n",
		 mci->mod_name, mci->ctl_name);
	edac_dbg(3, "\tpvt_info = %p\n\n", mci->pvt_info);
193 194
}

195 196
#endif				/* CONFIG_EDAC_DEBUG */

197
const char * const edac_mem_types[] = {
198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216
	[MEM_EMPTY]	= "Empty",
	[MEM_RESERVED]	= "Reserved",
	[MEM_UNKNOWN]	= "Unknown",
	[MEM_FPM]	= "FPM",
	[MEM_EDO]	= "EDO",
	[MEM_BEDO]	= "BEDO",
	[MEM_SDR]	= "Unbuffered-SDR",
	[MEM_RDR]	= "Registered-SDR",
	[MEM_DDR]	= "Unbuffered-DDR",
	[MEM_RDDR]	= "Registered-DDR",
	[MEM_RMBS]	= "RMBS",
	[MEM_DDR2]	= "Unbuffered-DDR2",
	[MEM_FB_DDR2]	= "FullyBuffered-DDR2",
	[MEM_RDDR2]	= "Registered-DDR2",
	[MEM_XDR]	= "XDR",
	[MEM_DDR3]	= "Unbuffered-DDR3",
	[MEM_RDDR3]	= "Registered-DDR3",
	[MEM_LRDDR3]	= "Load-Reduced-DDR3-RAM",
	[MEM_DDR4]	= "Unbuffered-DDR4",
217
	[MEM_RDDR4]	= "Registered-DDR4",
218
	[MEM_LRDDR4]	= "Load-Reduced-DDR4-RAM",
219
	[MEM_NVDIMM]	= "Non-volatile-RAM",
220 221 222
};
EXPORT_SYMBOL_GPL(edac_mem_types);

223 224 225 226 227 228
/**
 * edac_align_ptr - Prepares the pointer offsets for a single-shot allocation
 * @p:		pointer to a pointer with the memory offset to be used. At
 *		return, this will be incremented to point to the next offset
 * @size:	Size of the data structure to be reserved
 * @n_elems:	Number of elements that should be reserved
229 230
 *
 * If 'size' is a constant, the compiler will optimize this whole function
231 232 233 234 235 236 237 238 239
 * down to either a no-op or the addition of a constant to the value of '*p'.
 *
 * The 'p' pointer is absolutely needed to keep the proper advancing
 * further in memory to the proper offsets when allocating the struct along
 * with its embedded structs, as edac_device_alloc_ctl_info() does it
 * above, for example.
 *
 * At return, the pointer 'p' will be incremented to be used on a next call
 * to this function.
240
 */
241
void *edac_align_ptr(void **p, unsigned size, int n_elems)
242 243
{
	unsigned align, r;
244
	void *ptr = *p;
245

246 247 248 249 250 251 252 253
	*p += size * n_elems;

	/*
	 * 'p' can possibly be an unaligned item X such that sizeof(X) is
	 * 'size'.  Adjust 'p' so that its alignment is at least as
	 * stringent as what the compiler would provide for X and return
	 * the aligned result.
	 * Here we assume that the alignment of a "long long" is the most
254 255 256 257 258 259 260 261 262 263 264 265
	 * stringent alignment that the compiler will ever provide by default.
	 * As far as I know, this is a reasonable assumption.
	 */
	if (size > sizeof(long))
		align = sizeof(long long);
	else if (size > sizeof(int))
		align = sizeof(long);
	else if (size > sizeof(short))
		align = sizeof(int);
	else if (size > sizeof(char))
		align = sizeof(short);
	else
266
		return (char *)ptr;
267

268
	r = (unsigned long)p % align;
269 270

	if (r == 0)
271
		return (char *)ptr;
272

273 274
	*p += align - r;

275
	return (void *)(((unsigned long)ptr) + align - r);
276 277
}

278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307
static void _edac_mc_free(struct mem_ctl_info *mci)
{
	int i, chn, row;
	struct csrow_info *csr;
	const unsigned int tot_dimms = mci->tot_dimms;
	const unsigned int tot_channels = mci->num_cschannel;
	const unsigned int tot_csrows = mci->nr_csrows;

	if (mci->dimms) {
		for (i = 0; i < tot_dimms; i++)
			kfree(mci->dimms[i]);
		kfree(mci->dimms);
	}
	if (mci->csrows) {
		for (row = 0; row < tot_csrows; row++) {
			csr = mci->csrows[row];
			if (csr) {
				if (csr->channels) {
					for (chn = 0; chn < tot_channels; chn++)
						kfree(csr->channels[chn]);
					kfree(csr->channels);
				}
				kfree(csr);
			}
		}
		kfree(mci->csrows);
	}
	kfree(mci);
}

308 309 310 311
struct mem_ctl_info *edac_mc_alloc(unsigned mc_num,
				   unsigned n_layers,
				   struct edac_mc_layer *layers,
				   unsigned sz_pvt)
312 313
{
	struct mem_ctl_info *mci;
314
	struct edac_mc_layer *layer;
315 316
	struct csrow_info *csr;
	struct rank_info *chan;
317
	struct dimm_info *dimm;
318 319 320 321
	u32 *ce_per_layer[EDAC_MAX_LAYERS], *ue_per_layer[EDAC_MAX_LAYERS];
	unsigned pos[EDAC_MAX_LAYERS];
	unsigned size, tot_dimms = 1, count = 1;
	unsigned tot_csrows = 1, tot_channels = 1, tot_errcount = 0;
322
	void *pvt, *p, *ptr = NULL;
323
	int i, j, row, chn, n, len, off;
324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340
	bool per_rank = false;

	BUG_ON(n_layers > EDAC_MAX_LAYERS || n_layers == 0);
	/*
	 * Calculate the total amount of dimms and csrows/cschannels while
	 * in the old API emulation mode
	 */
	for (i = 0; i < n_layers; i++) {
		tot_dimms *= layers[i].size;
		if (layers[i].is_virt_csrow)
			tot_csrows *= layers[i].size;
		else
			tot_channels *= layers[i].size;

		if (layers[i].type == EDAC_MC_LAYER_CHIP_SELECT)
			per_rank = true;
	}
341 342 343 344 345 346

	/* Figure out the offsets of the various items from the start of an mc
	 * structure.  We want the alignment of each item to be at least as
	 * stringent as what the compiler would provide if we could simply
	 * hardcode everything into a single struct.
	 */
347
	mci = edac_align_ptr(&ptr, sizeof(*mci), 1);
348 349 350
	layer = edac_align_ptr(&ptr, sizeof(*layer), n_layers);
	for (i = 0; i < n_layers; i++) {
		count *= layers[i].size;
351
		edac_dbg(4, "errcount layer %d size %d\n", i, count);
352 353 354 355 356
		ce_per_layer[i] = edac_align_ptr(&ptr, sizeof(u32), count);
		ue_per_layer[i] = edac_align_ptr(&ptr, sizeof(u32), count);
		tot_errcount += 2 * count;
	}

357
	edac_dbg(4, "allocating %d error counters\n", tot_errcount);
358
	pvt = edac_align_ptr(&ptr, sz_pvt, 1);
359
	size = ((unsigned long)pvt) + sz_pvt;
360

361 362 363 364 365
	edac_dbg(1, "allocating %u bytes for mci data (%d %s, %d csrows/channels)\n",
		 size,
		 tot_dimms,
		 per_rank ? "ranks" : "dimms",
		 tot_csrows * tot_channels);
366

367 368
	mci = kzalloc(size, GFP_KERNEL);
	if (mci == NULL)
369 370 371 372 373
		return NULL;

	/* Adjust pointers so they point within the memory we just allocated
	 * rather than an imaginary chunk of memory located at address 0.
	 */
374 375 376 377 378
	layer = (struct edac_mc_layer *)(((char *)mci) + ((unsigned long)layer));
	for (i = 0; i < n_layers; i++) {
		mci->ce_per_layer[i] = (u32 *)((char *)mci + ((unsigned long)ce_per_layer[i]));
		mci->ue_per_layer[i] = (u32 *)((char *)mci + ((unsigned long)ue_per_layer[i]));
	}
379
	pvt = sz_pvt ? (((char *)mci) + ((unsigned long)pvt)) : NULL;
380

381
	/* setup index and various internal pointers */
382 383
	mci->mc_idx = mc_num;
	mci->tot_dimms = tot_dimms;
384
	mci->pvt_info = pvt;
385 386 387 388 389
	mci->n_layers = n_layers;
	mci->layers = layer;
	memcpy(mci->layers, layers, sizeof(*layer) * n_layers);
	mci->nr_csrows = tot_csrows;
	mci->num_cschannel = tot_channels;
390
	mci->csbased = per_rank;
391

392
	/*
393
	 * Alocate and fill the csrow/channels structs
394
	 */
395
	mci->csrows = kcalloc(tot_csrows, sizeof(*mci->csrows), GFP_KERNEL);
396 397
	if (!mci->csrows)
		goto error;
398
	for (row = 0; row < tot_csrows; row++) {
399 400 401 402
		csr = kzalloc(sizeof(**mci->csrows), GFP_KERNEL);
		if (!csr)
			goto error;
		mci->csrows[row] = csr;
403 404 405
		csr->csrow_idx = row;
		csr->mci = mci;
		csr->nr_channels = tot_channels;
406
		csr->channels = kcalloc(tot_channels, sizeof(*csr->channels),
407 408 409
					GFP_KERNEL);
		if (!csr->channels)
			goto error;
410 411

		for (chn = 0; chn < tot_channels; chn++) {
412 413 414 415
			chan = kzalloc(sizeof(**csr->channels), GFP_KERNEL);
			if (!chan)
				goto error;
			csr->channels[chn] = chan;
416
			chan->chan_idx = chn;
417 418 419 420 421
			chan->csrow = csr;
		}
	}

	/*
422
	 * Allocate and fill the dimm structs
423
	 */
424
	mci->dimms  = kcalloc(tot_dimms, sizeof(*mci->dimms), GFP_KERNEL);
425 426 427
	if (!mci->dimms)
		goto error;

428 429 430 431
	memset(&pos, 0, sizeof(pos));
	row = 0;
	chn = 0;
	for (i = 0; i < tot_dimms; i++) {
432 433 434 435 436 437
		chan = mci->csrows[row]->channels[chn];
		off = EDAC_DIMM_OFF(layer, n_layers, pos[0], pos[1], pos[2]);
		if (off < 0 || off >= tot_dimms) {
			edac_mc_printk(mci, KERN_ERR, "EDAC core bug: EDAC_DIMM_OFF is trying to do an illegal data access\n");
			goto error;
		}
438

439
		dimm = kzalloc(sizeof(**mci->dimms), GFP_KERNEL);
440 441
		if (!dimm)
			goto error;
442
		mci->dimms[off] = dimm;
443 444
		dimm->mci = mci;

445 446 447 448 449 450 451 452 453 454 455 456 457 458
		/*
		 * Copy DIMM location and initialize it.
		 */
		len = sizeof(dimm->label);
		p = dimm->label;
		n = snprintf(p, len, "mc#%u", mc_num);
		p += n;
		len -= n;
		for (j = 0; j < n_layers; j++) {
			n = snprintf(p, len, "%s#%u",
				     edac_layer_name[layers[j].type],
				     pos[j]);
			p += n;
			len -= n;
459 460
			dimm->location[j] = pos[j];

461 462 463 464
			if (len <= 0)
				break;
		}

465 466 467 468 469 470
		/* Link it to the csrows old API data */
		chan->dimm = dimm;
		dimm->csrow = row;
		dimm->cschannel = chn;

		/* Increment csrow location */
471
		if (layers[0].is_virt_csrow) {
472
			chn++;
473 474 475 476 477 478 479 480 481 482
			if (chn == tot_channels) {
				chn = 0;
				row++;
			}
		} else {
			row++;
			if (row == tot_csrows) {
				row = 0;
				chn++;
			}
483
		}
484

485 486 487 488 489 490
		/* Increment dimm location */
		for (j = n_layers - 1; j >= 0; j--) {
			pos[j]++;
			if (pos[j] < layers[j].size)
				break;
			pos[j] = 0;
491 492 493
		}
	}

494
	mci->op_state = OP_ALLOC;
495

496
	return mci;
497 498

error:
499
	_edac_mc_free(mci);
500 501

	return NULL;
502
}
503
EXPORT_SYMBOL_GPL(edac_mc_alloc);
504 505 506

void edac_mc_free(struct mem_ctl_info *mci)
{
507
	edac_dbg(1, "\n");
508

509 510 511 512 513 514 515 516
	/* If we're not yet registered with sysfs free only what was allocated
	 * in edac_mc_alloc().
	 */
	if (!device_is_registered(&mci->dev)) {
		_edac_mc_free(mci);
		return;
	}

517
	/* the mci instance is freed here, when the sysfs object is dropped */
518
	edac_unregister_sysfs(mci);
519
}
520
EXPORT_SYMBOL_GPL(edac_mc_free);
521

522 523 524 525 526 527 528 529 530 531 532 533 534 535
bool edac_has_mcs(void)
{
	bool ret;

	mutex_lock(&mem_ctls_mutex);

	ret = list_empty(&mc_devices);

	mutex_unlock(&mem_ctls_mutex);

	return !ret;
}
EXPORT_SYMBOL_GPL(edac_has_mcs);

536 537
/* Caller must hold mem_ctls_mutex */
static struct mem_ctl_info *__find_mci_by_dev(struct device *dev)
538 539 540 541
{
	struct mem_ctl_info *mci;
	struct list_head *item;

542
	edac_dbg(3, "\n");
543 544 545 546

	list_for_each(item, &mc_devices) {
		mci = list_entry(item, struct mem_ctl_info, link);

547
		if (mci->pdev == dev)
548 549 550 551 552
			return mci;
	}

	return NULL;
}
553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570

/**
 * find_mci_by_dev
 *
 *	scan list of controllers looking for the one that manages
 *	the 'dev' device
 * @dev: pointer to a struct device related with the MCI
 */
struct mem_ctl_info *find_mci_by_dev(struct device *dev)
{
	struct mem_ctl_info *ret;

	mutex_lock(&mem_ctls_mutex);
	ret = __find_mci_by_dev(dev);
	mutex_unlock(&mem_ctls_mutex);

	return ret;
}
571
EXPORT_SYMBOL_GPL(find_mci_by_dev);
572

573 574 575 576 577 578
/*
 * edac_mc_workq_function
 *	performs the operation scheduled by a workq request
 */
static void edac_mc_workq_function(struct work_struct *work_req)
{
579
	struct delayed_work *d_work = to_delayed_work(work_req);
580 581 582 583
	struct mem_ctl_info *mci = to_edac_mem_ctl_work(d_work);

	mutex_lock(&mem_ctls_mutex);

584
	if (mci->op_state != OP_RUNNING_POLL) {
585 586 587 588
		mutex_unlock(&mem_ctls_mutex);
		return;
	}

589
	if (edac_op_state == EDAC_OPSTATE_POLL)
590 591 592 593
		mci->edac_check(mci);

	mutex_unlock(&mem_ctls_mutex);

594
	/* Queue ourselves again. */
595
	edac_queue_work(&mci->work, msecs_to_jiffies(edac_mc_get_poll_msec()));
596 597 598
}

/*
599 600 601 602
 * edac_mc_reset_delay_period(unsigned long value)
 *
 *	user space has updated our poll period value, need to
 *	reset our workq delays
603
 */
604
void edac_mc_reset_delay_period(unsigned long value)
605
{
606 607 608 609 610 611 612 613
	struct mem_ctl_info *mci;
	struct list_head *item;

	mutex_lock(&mem_ctls_mutex);

	list_for_each(item, &mc_devices) {
		mci = list_entry(item, struct mem_ctl_info, link);

614 615
		if (mci->op_state == OP_RUNNING_POLL)
			edac_mod_work(&mci->work, value);
616
	}
617 618 619
	mutex_unlock(&mem_ctls_mutex);
}

620 621


622 623 624
/* Return 0 on success, 1 on failure.
 * Before calling this function, caller must
 * assign a unique value to mci->mc_idx.
625 626 627 628
 *
 *	locking model:
 *
 *		called with the mem_ctls_mutex lock held
629
 */
630
static int add_mc_to_global_list(struct mem_ctl_info *mci)
631 632 633 634
{
	struct list_head *item, *insert_before;
	struct mem_ctl_info *p;

635
	insert_before = &mc_devices;
636

637
	p = __find_mci_by_dev(mci->pdev);
638
	if (unlikely(p != NULL))
639
		goto fail0;
640

641 642
	list_for_each(item, &mc_devices) {
		p = list_entry(item, struct mem_ctl_info, link);
643

644 645 646
		if (p->mc_idx >= mci->mc_idx) {
			if (unlikely(p->mc_idx == mci->mc_idx))
				goto fail1;
647

648 649
			insert_before = item;
			break;
650 651 652 653 654
		}
	}

	list_add_tail_rcu(&mci->link, insert_before);
	return 0;
655

656
fail0:
657
	edac_printk(KERN_WARNING, EDAC_MC,
658
		"%s (%s) %s %s already assigned %d\n", dev_name(p->pdev),
659
		edac_dev_name(mci), p->mod_name, p->ctl_name, p->mc_idx);
660 661
	return 1;

662
fail1:
663
	edac_printk(KERN_WARNING, EDAC_MC,
664 665
		"bug in low-level driver: attempt to assign\n"
		"    duplicate mc_idx %d in %s()\n", p->mc_idx, __func__);
666
	return 1;
667 668
}

669
static int del_mc_from_global_list(struct mem_ctl_info *mci)
670 671
{
	list_del_rcu(&mci->link);
672 673 674 675 676 677

	/* these are for safe removal of devices from global list while
	 * NMI handlers may be traversing list
	 */
	synchronize_rcu();
	INIT_LIST_HEAD(&mci->link);
678

679
	return list_empty(&mc_devices);
680 681
}

682
struct mem_ctl_info *edac_mc_find(int idx)
683
{
684
	struct mem_ctl_info *mci = NULL;
685
	struct list_head *item;
686 687

	mutex_lock(&mem_ctls_mutex);
688 689 690 691 692

	list_for_each(item, &mc_devices) {
		mci = list_entry(item, struct mem_ctl_info, link);

		if (mci->mc_idx >= idx) {
693 694 695
			if (mci->mc_idx == idx) {
				goto unlock;
			}
696 697 698 699
			break;
		}
	}

700 701 702
unlock:
	mutex_unlock(&mem_ctls_mutex);
	return mci;
703 704 705
}
EXPORT_SYMBOL(edac_mc_find);

706 707 708 709 710
const char *edac_get_owner(void)
{
	return edac_mc_owner;
}
EXPORT_SYMBOL_GPL(edac_get_owner);
711 712

/* FIXME - should a warning be printed if no error detection? correction? */
713 714
int edac_mc_add_mc_with_groups(struct mem_ctl_info *mci,
			       const struct attribute_group **groups)
715
{
716
	int ret = -EINVAL;
717
	edac_dbg(0, "\n");
718

719 720 721 722 723
	if (mci->mc_idx >= EDAC_MAX_MCS) {
		pr_warn_once("Too many memory controllers: %d\n", mci->mc_idx);
		return -ENODEV;
	}

724 725 726
#ifdef CONFIG_EDAC_DEBUG
	if (edac_debug_level >= 3)
		edac_mc_dump_mci(mci);
727

728 729 730 731
	if (edac_debug_level >= 4) {
		int i;

		for (i = 0; i < mci->nr_csrows; i++) {
732 733
			struct csrow_info *csrow = mci->csrows[i];
			u32 nr_pages = 0;
734
			int j;
735

736 737 738 739 740 741 742 743
			for (j = 0; j < csrow->nr_channels; j++)
				nr_pages += csrow->channels[j]->dimm->nr_pages;
			if (!nr_pages)
				continue;
			edac_mc_dump_csrow(csrow);
			for (j = 0; j < csrow->nr_channels; j++)
				if (csrow->channels[j]->dimm->nr_pages)
					edac_mc_dump_channel(csrow->channels[j]);
744
		}
745
		for (i = 0; i < mci->tot_dimms; i++)
746 747
			if (mci->dimms[i]->nr_pages)
				edac_mc_dump_dimm(mci->dimms[i], i);
748 749
	}
#endif
750
	mutex_lock(&mem_ctls_mutex);
751

752 753 754 755 756
	if (edac_mc_owner && edac_mc_owner != mci->mod_name) {
		ret = -EPERM;
		goto fail0;
	}

757
	if (add_mc_to_global_list(mci))
758
		goto fail0;
759 760 761 762

	/* set load time so that error rate can be tracked */
	mci->start_time = jiffies;

763 764
	mci->bus = &mc_bus[mci->mc_idx];

765
	if (edac_create_sysfs_mci_device(mci, groups)) {
766
		edac_mc_printk(mci, KERN_WARNING,
767
			"failed to create sysfs device\n");
768 769
		goto fail1;
	}
770

771
	if (mci->edac_check) {
772 773
		mci->op_state = OP_RUNNING_POLL;

774 775 776
		INIT_DELAYED_WORK(&mci->work, edac_mc_workq_function);
		edac_queue_work(&mci->work, msecs_to_jiffies(edac_mc_get_poll_msec()));

777 778 779 780
	} else {
		mci->op_state = OP_RUNNING_INTERRUPT;
	}

781
	/* Report action taken */
782 783 784 785
	edac_mc_printk(mci, KERN_INFO,
		"Giving out device to module %s controller %s: DEV %s (%s)\n",
		mci->mod_name, mci->ctl_name, mci->dev_name,
		edac_op_state_to_string(mci->op_state));
786

787 788
	edac_mc_owner = mci->mod_name;

789
	mutex_unlock(&mem_ctls_mutex);
790
	return 0;
791

792
fail1:
793 794
	del_mc_from_global_list(mci);

795
fail0:
796
	mutex_unlock(&mem_ctls_mutex);
797
	return ret;
798
}
799
EXPORT_SYMBOL_GPL(edac_mc_add_mc_with_groups);
800

801
struct mem_ctl_info *edac_mc_del_mc(struct device *dev)
802
{
803
	struct mem_ctl_info *mci;
804

805
	edac_dbg(0, "\n");
806

807
	mutex_lock(&mem_ctls_mutex);
808

809
	/* find the requested mci struct in the global list */
810
	mci = __find_mci_by_dev(dev);
811
	if (mci == NULL) {
812
		mutex_unlock(&mem_ctls_mutex);
813 814 815
		return NULL;
	}

816 817 818
	/* mark MCI offline: */
	mci->op_state = OP_OFFLINE;

819
	if (del_mc_from_global_list(mci))
820
		edac_mc_owner = NULL;
821

822
	mutex_unlock(&mem_ctls_mutex);
823

824
	if (mci->edac_check)
825
		edac_stop_work(&mci->work);
826 827

	/* remove from sysfs */
828 829
	edac_remove_sysfs_mci_device(mci);

830
	edac_printk(KERN_INFO, EDAC_MC,
831
		"Removed device %d for %s %s: DEV %s\n", mci->mc_idx,
832
		mci->mod_name, mci->ctl_name, edac_dev_name(mci));
833

834
	return mci;
835
}
836
EXPORT_SYMBOL_GPL(edac_mc_del_mc);
837

838 839
static void edac_mc_scrub_block(unsigned long page, unsigned long offset,
				u32 size)
840 841 842 843 844
{
	struct page *pg;
	void *virt_addr;
	unsigned long flags = 0;

845
	edac_dbg(3, "\n");
846 847

	/* ECC error page was not in our memory. Ignore it. */
848
	if (!pfn_valid(page))
849 850 851 852 853 854 855 856
		return;

	/* Find the actual page structure then map it and fix */
	pg = pfn_to_page(page);

	if (PageHighMem(pg))
		local_irq_save(flags);

857
	virt_addr = kmap_atomic(pg);
858 859

	/* Perform architecture specific atomic scrub operation */
860
	edac_atomic_scrub(virt_addr + offset, size);
861 862

	/* Unmap and complete */
863
	kunmap_atomic(virt_addr);
864 865 866 867 868 869

	if (PageHighMem(pg))
		local_irq_restore(flags);
}

/* FIXME - should return -1 */
870
int edac_mc_find_csrow_by_page(struct mem_ctl_info *mci, unsigned long page)
871
{
872
	struct csrow_info **csrows = mci->csrows;
873
	int row, i, j, n;
874

875
	edac_dbg(1, "MC%d: 0x%lx\n", mci->mc_idx, page);
876 877 878
	row = -1;

	for (i = 0; i < mci->nr_csrows; i++) {
879
		struct csrow_info *csrow = csrows[i];
880 881
		n = 0;
		for (j = 0; j < csrow->nr_channels; j++) {
882
			struct dimm_info *dimm = csrow->channels[j]->dimm;
883 884 885
			n += dimm->nr_pages;
		}
		if (n == 0)
886 887
			continue;

888 889 890 891
		edac_dbg(3, "MC%d: first(0x%lx) page(0x%lx) last(0x%lx) mask(0x%lx)\n",
			 mci->mc_idx,
			 csrow->first_page, page, csrow->last_page,
			 csrow->page_mask);
892 893 894 895 896 897 898 899 900 901 902

		if ((page >= csrow->first_page) &&
		    (page <= csrow->last_page) &&
		    ((page & csrow->page_mask) ==
		     (csrow->first_page & csrow->page_mask))) {
			row = i;
			break;
		}
	}

	if (row == -1)
903
		edac_mc_printk(mci, KERN_ERR,
904 905
			"could not look up page error address %lx\n",
			(unsigned long)page);
906 907 908

	return row;
}
909
EXPORT_SYMBOL_GPL(edac_mc_find_csrow_by_page);
910

911 912 913 914 915
const char *edac_layer_name[] = {
	[EDAC_MC_LAYER_BRANCH] = "branch",
	[EDAC_MC_LAYER_CHANNEL] = "channel",
	[EDAC_MC_LAYER_SLOT] = "slot",
	[EDAC_MC_LAYER_CHIP_SELECT] = "csrow",
916
	[EDAC_MC_LAYER_ALL_MEM] = "memory",
917 918 919 920
};
EXPORT_SYMBOL_GPL(edac_layer_name);

static void edac_inc_ce_error(struct mem_ctl_info *mci,
921 922 923
			      bool enable_per_layer_report,
			      const int pos[EDAC_MAX_LAYERS],
			      const u16 count)
924
{
925
	int i, index = 0;
926

927
	mci->ce_mc += count;
928

929
	if (!enable_per_layer_report) {
930
		mci->ce_noinfo_count += count;
931 932
		return;
	}
933

934 935 936 937
	for (i = 0; i < mci->n_layers; i++) {
		if (pos[i] < 0)
			break;
		index += pos[i];
938
		mci->ce_per_layer[i][index] += count;
939 940 941 942 943 944 945 946

		if (i < mci->n_layers - 1)
			index *= mci->layers[i + 1].size;
	}
}

static void edac_inc_ue_error(struct mem_ctl_info *mci,
				    bool enable_per_layer_report,
947 948
				    const int pos[EDAC_MAX_LAYERS],
				    const u16 count)
949 950 951
{
	int i, index = 0;

952
	mci->ue_mc += count;
953 954

	if (!enable_per_layer_report) {
955
		mci->ue_noinfo_count += count;
956 957 958
		return;
	}

959 960 961 962
	for (i = 0; i < mci->n_layers; i++) {
		if (pos[i] < 0)
			break;
		index += pos[i];
963
		mci->ue_per_layer[i][index] += count;
964

965 966 967 968
		if (i < mci->n_layers - 1)
			index *= mci->layers[i + 1].size;
	}
}
969

970
static void edac_ce_error(struct mem_ctl_info *mci,
971
			  const u16 error_count,
972 973 974 975 976 977 978 979 980
			  const int pos[EDAC_MAX_LAYERS],
			  const char *msg,
			  const char *location,
			  const char *label,
			  const char *detail,
			  const char *other_detail,
			  const bool enable_per_layer_report,
			  const unsigned long page_frame_number,
			  const unsigned long offset_in_page,
981
			  long grain)
982 983
{
	unsigned long remapped_page;
984 985 986 987
	char *msg_aux = "";

	if (*msg)
		msg_aux = " ";
988 989 990 991

	if (edac_mc_get_log_ce()) {
		if (other_detail && *other_detail)
			edac_mc_printk(mci, KERN_WARNING,
992 993 994
				       "%d CE %s%son %s (%s %s - %s)\n",
				       error_count, msg, msg_aux, label,
				       location, detail, other_detail);
995 996
		else
			edac_mc_printk(mci, KERN_WARNING,
997 998 999
				       "%d CE %s%son %s (%s %s)\n",
				       error_count, msg, msg_aux, label,
				       location, detail);
1000
	}
1001
	edac_inc_ce_error(mci, enable_per_layer_report, pos, error_count);
1002

1003
	if (mci->scrub_mode == SCRUB_SW_SRC) {
1004
		/*
1005 1006 1007 1008 1009 1010 1011 1012 1013 1014
			* Some memory controllers (called MCs below) can remap
			* memory so that it is still available at a different
			* address when PCI devices map into memory.
			* MC's that can't do this, lose the memory where PCI
			* devices are mapped. This mapping is MC-dependent
			* and so we call back into the MC driver for it to
			* map the MC page to a physical (CPU) page which can
			* then be mapped to a virtual page - which can then
			* be scrubbed.
			*/
1015
		remapped_page = mci->ctl_page_to_phys ?
1016 1017
			mci->ctl_page_to_phys(mci, page_frame_number) :
			page_frame_number;
1018

1019 1020
		edac_mc_scrub_block(remapped_page,
					offset_in_page, grain);
1021 1022 1023
	}
}

1024
static void edac_ue_error(struct mem_ctl_info *mci,
1025
			  const u16 error_count,
1026 1027 1028 1029 1030 1031 1032
			  const int pos[EDAC_MAX_LAYERS],
			  const char *msg,
			  const char *location,
			  const char *label,
			  const char *detail,
			  const char *other_detail,
			  const bool enable_per_layer_report)
1033
{
1034 1035 1036 1037 1038
	char *msg_aux = "";

	if (*msg)
		msg_aux = " ";

1039 1040 1041
	if (edac_mc_get_log_ue()) {
		if (other_detail && *other_detail)
			edac_mc_printk(mci, KERN_WARNING,
1042 1043 1044
				       "%d UE %s%son %s (%s %s - %s)\n",
				       error_count, msg, msg_aux, label,
				       location, detail, other_detail);
1045 1046
		else
			edac_mc_printk(mci, KERN_WARNING,
1047 1048 1049
				       "%d UE %s%son %s (%s %s)\n",
				       error_count, msg, msg_aux, label,
				       location, detail);
1050
	}
1051

1052 1053
	if (edac_mc_get_panic_on_ue()) {
		if (other_detail && *other_detail)
1054 1055
			panic("UE %s%son %s (%s%s - %s)\n",
			      msg, msg_aux, label, location, detail, other_detail);
1056
		else
1057 1058
			panic("UE %s%son %s (%s%s)\n",
			      msg, msg_aux, label, location, detail);
1059 1060
	}

1061
	edac_inc_ue_error(mci, enable_per_layer_report, pos, error_count);
1062 1063
}

1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091
void edac_raw_mc_handle_error(const enum hw_event_mc_err_type type,
			      struct mem_ctl_info *mci,
			      struct edac_raw_error_desc *e)
{
	char detail[80];
	int pos[EDAC_MAX_LAYERS] = { e->top_layer, e->mid_layer, e->low_layer };

	/* Memory type dependent details about the error */
	if (type == HW_EVENT_ERR_CORRECTED) {
		snprintf(detail, sizeof(detail),
			"page:0x%lx offset:0x%lx grain:%ld syndrome:0x%lx",
			e->page_frame_number, e->offset_in_page,
			e->grain, e->syndrome);
		edac_ce_error(mci, e->error_count, pos, e->msg, e->location, e->label,
			      detail, e->other_detail, e->enable_per_layer_report,
			      e->page_frame_number, e->offset_in_page, e->grain);
	} else {
		snprintf(detail, sizeof(detail),
			"page:0x%lx offset:0x%lx grain:%ld",
			e->page_frame_number, e->offset_in_page, e->grain);

		edac_ue_error(mci, e->error_count, pos, e->msg, e->location, e->label,
			      detail, e->other_detail, e->enable_per_layer_report);
	}


}
EXPORT_SYMBOL_GPL(edac_raw_mc_handle_error);
1092

1093 1094
void edac_mc_handle_error(const enum hw_event_mc_err_type type,
			  struct mem_ctl_info *mci,
1095
			  const u16 error_count,
1096 1097 1098
			  const unsigned long page_frame_number,
			  const unsigned long offset_in_page,
			  const unsigned long syndrome,
1099 1100 1101
			  const int top_layer,
			  const int mid_layer,
			  const int low_layer,
1102
			  const char *msg,
1103
			  const char *other_detail)
1104
{
1105 1106
	char *p;
	int row = -1, chan = -1;
1107
	int pos[EDAC_MAX_LAYERS] = { top_layer, mid_layer, low_layer };
1108
	int i, n_labels = 0;
1109
	u8 grain_bits;
1110
	struct edac_raw_error_desc *e = &mci->error_desc;
1111

1112
	edac_dbg(3, "MC%d\n", mci->mc_idx);
1113

1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125
	/* Fills the error report buffer */
	memset(e, 0, sizeof (*e));
	e->error_count = error_count;
	e->top_layer = top_layer;
	e->mid_layer = mid_layer;
	e->low_layer = low_layer;
	e->page_frame_number = page_frame_number;
	e->offset_in_page = offset_in_page;
	e->syndrome = syndrome;
	e->msg = msg;
	e->other_detail = other_detail;

1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147
	/*
	 * Check if the event report is consistent and if the memory
	 * location is known. If it is known, enable_per_layer_report will be
	 * true, the DIMM(s) label info will be filled and the per-layer
	 * error counters will be incremented.
	 */
	for (i = 0; i < mci->n_layers; i++) {
		if (pos[i] >= (int)mci->layers[i].size) {

			edac_mc_printk(mci, KERN_ERR,
				       "INTERNAL ERROR: %s value is out of range (%d >= %d)\n",
				       edac_layer_name[mci->layers[i].type],
				       pos[i], mci->layers[i].size);
			/*
			 * Instead of just returning it, let's use what's
			 * known about the error. The increment routines and
			 * the DIMM filter logic will do the right thing by
			 * pointing the likely damaged DIMMs.
			 */
			pos[i] = -1;
		}
		if (pos[i] >= 0)
1148
			e->enable_per_layer_report = true;
1149 1150
	}

1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161
	/*
	 * Get the dimm label/grain that applies to the match criteria.
	 * As the error algorithm may not be able to point to just one memory
	 * stick, the logic here will get all possible labels that could
	 * pottentially be affected by the error.
	 * On FB-DIMM memory controllers, for uncorrected errors, it is common
	 * to have only the MC channel and the MC dimm (also called "branch")
	 * but the channel is not known, as the memory is arranged in pairs,
	 * where each memory belongs to a separate channel within the same
	 * branch.
	 */
1162
	p = e->label;
1163
	*p = '\0';
1164

1165
	for (i = 0; i < mci->tot_dimms; i++) {
1166
		struct dimm_info *dimm = mci->dimms[i];
1167

1168
		if (top_layer >= 0 && top_layer != dimm->location[0])
1169
			continue;
1170
		if (mid_layer >= 0 && mid_layer != dimm->location[1])
1171
			continue;
1172
		if (low_layer >= 0 && low_layer != dimm->location[2])
1173
			continue;
1174

1175
		/* get the max grain, over the error match range */
1176 1177
		if (dimm->grain > e->grain)
			e->grain = dimm->grain;
1178

1179 1180 1181 1182 1183 1184
		/*
		 * If the error is memory-controller wide, there's no need to
		 * seek for the affected DIMMs because the whole
		 * channel/memory controller/...  may be affected.
		 * Also, don't show errors for empty DIMM slots.
		 */
1185 1186 1187 1188 1189 1190 1191
		if (e->enable_per_layer_report && dimm->nr_pages) {
			if (n_labels >= EDAC_MAX_LABELS) {
				e->enable_per_layer_report = false;
				break;
			}
			n_labels++;
			if (p != e->label) {
1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202
				strcpy(p, OTHER_LABEL);
				p += strlen(OTHER_LABEL);
			}
			strcpy(p, dimm->label);
			p += strlen(p);
			*p = '\0';

			/*
			 * get csrow/channel of the DIMM, in order to allow
			 * incrementing the compat API counters
			 */
1203
			edac_dbg(4, "%s csrows map: (%d,%d)\n",
1204
				 mci->csbased ? "rank" : "dimm",
1205
				 dimm->csrow, dimm->cschannel);
1206 1207 1208 1209 1210 1211 1212 1213 1214 1215
			if (row == -1)
				row = dimm->csrow;
			else if (row >= 0 && row != dimm->csrow)
				row = -2;

			if (chan == -1)
				chan = dimm->cschannel;
			else if (chan >= 0 && chan != dimm->cschannel)
				chan = -2;
		}
1216 1217
	}

1218 1219
	if (!e->enable_per_layer_report) {
		strcpy(e->label, "any memory");
1220
	} else {
1221
		edac_dbg(4, "csrow/channel to increment: (%d,%d)\n", row, chan);
1222 1223
		if (p == e->label)
			strcpy(e->label, "unknown memory");
1224 1225
		if (type == HW_EVENT_ERR_CORRECTED) {
			if (row >= 0) {
1226
				mci->csrows[row]->ce_count += error_count;
1227
				if (chan >= 0)
1228
					mci->csrows[row]->channels[chan]->ce_count += error_count;
1229 1230 1231
			}
		} else
			if (row >= 0)
1232
				mci->csrows[row]->ue_count += error_count;
1233 1234
	}

1235
	/* Fill the RAM location data */
1236
	p = e->location;
1237

1238 1239 1240
	for (i = 0; i < mci->n_layers; i++) {
		if (pos[i] < 0)
			continue;
1241

1242 1243 1244
		p += sprintf(p, "%s:%d ",
			     edac_layer_name[mci->layers[i].type],
			     pos[i]);
1245
	}
1246
	if (p > e->location)
1247 1248 1249
		*(p - 1) = '\0';

	/* Report the error via the trace interface */
1250
	grain_bits = fls_long(e->grain) + 1;
1251 1252 1253 1254 1255 1256 1257

	if (IS_ENABLED(CONFIG_RAS))
		trace_mc_event(type, e->msg, e->label, e->error_count,
			       mci->mc_idx, e->top_layer, e->mid_layer,
			       e->low_layer,
			       (e->page_frame_number << PAGE_SHIFT) | e->offset_in_page,
			       grain_bits, e->syndrome, e->other_detail);
1258

1259
	edac_raw_mc_handle_error(type, mci, e);
1260
}
1261
EXPORT_SYMBOL_GPL(edac_mc_handle_error);