bcm-sba-raid.c 48.6 KB
Newer Older
1 2 3
/*
 * Copyright (C) 2017 Broadcom
 *
4 5 6 7 8 9 10 11
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation version 2.
 *
 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
 * kind, whether express or implied; without even the implied warranty
 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
 */

/*
 * Broadcom SBA RAID Driver
 *
 * The Broadcom stream buffer accelerator (SBA) provides offloading
 * capabilities for RAID operations. The SBA offload engine is accessible
 * via Broadcom SoC specific ring manager. Two or more offload engines
 * can share same Broadcom SoC specific ring manager due to this Broadcom
 * SoC specific ring manager driver is implemented as a mailbox controller
 * driver and offload engine drivers are implemented as mallbox clients.
 *
 * Typically, Broadcom SoC specific ring manager will implement larger
 * number of hardware rings over one or more SBA hardware devices. By
 * design, the internal buffer size of SBA hardware device is limited
 * but all offload operations supported by SBA can be broken down into
 * multiple small size requests and executed parallely on multiple SBA
 * hardware devices for achieving high through-put.
 *
 * The Broadcom SBA RAID driver does not require any register programming
 * except submitting request to SBA hardware device via mailbox channels.
33 34
 * This driver implements a DMA device with one DMA channel using a single
 * mailbox channel provided by Broadcom SoC specific ring manager driver.
35 36 37 38 39 40
 * For having more SBA DMA channels, we can create more SBA device nodes
 * in Broadcom SoC specific DTS based on number of hardware rings supported
 * by Broadcom SoC ring manager.
 */

#include <linux/bitops.h>
41
#include <linux/debugfs.h>
42 43 44 45 46 47 48 49 50 51 52 53
#include <linux/dma-mapping.h>
#include <linux/dmaengine.h>
#include <linux/list.h>
#include <linux/mailbox_client.h>
#include <linux/mailbox/brcm-message.h>
#include <linux/module.h>
#include <linux/of_device.h>
#include <linux/slab.h>
#include <linux/raid/pq.h>

#include "dmaengine.h"

54 55
/* ====== Driver macros and defines ===== */

56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
#define SBA_TYPE_SHIFT					48
#define SBA_TYPE_MASK					GENMASK(1, 0)
#define SBA_TYPE_A					0x0
#define SBA_TYPE_B					0x2
#define SBA_TYPE_C					0x3
#define SBA_USER_DEF_SHIFT				32
#define SBA_USER_DEF_MASK				GENMASK(15, 0)
#define SBA_R_MDATA_SHIFT				24
#define SBA_R_MDATA_MASK				GENMASK(7, 0)
#define SBA_C_MDATA_MS_SHIFT				18
#define SBA_C_MDATA_MS_MASK				GENMASK(1, 0)
#define SBA_INT_SHIFT					17
#define SBA_INT_MASK					BIT(0)
#define SBA_RESP_SHIFT					16
#define SBA_RESP_MASK					BIT(0)
#define SBA_C_MDATA_SHIFT				8
#define SBA_C_MDATA_MASK				GENMASK(7, 0)
#define SBA_C_MDATA_BNUMx_SHIFT(__bnum)			(2 * (__bnum))
#define SBA_C_MDATA_BNUMx_MASK				GENMASK(1, 0)
#define SBA_C_MDATA_DNUM_SHIFT				5
#define SBA_C_MDATA_DNUM_MASK				GENMASK(4, 0)
#define SBA_C_MDATA_LS(__v)				((__v) & 0xff)
#define SBA_C_MDATA_MS(__v)				(((__v) >> 8) & 0x3)
#define SBA_CMD_SHIFT					0
#define SBA_CMD_MASK					GENMASK(3, 0)
#define SBA_CMD_ZERO_BUFFER				0x4
#define SBA_CMD_ZERO_ALL_BUFFERS			0x8
#define SBA_CMD_LOAD_BUFFER				0x9
#define SBA_CMD_XOR					0xa
#define SBA_CMD_GALOIS_XOR				0xb
#define SBA_CMD_WRITE_BUFFER				0xc
#define SBA_CMD_GALOIS					0xe

89
#define SBA_MAX_REQ_PER_MBOX_CHANNEL			8192
90
#define SBA_MAX_MSG_SEND_PER_MBOX_CHANNEL		8
91

92 93 94 95 96 97
/* Driver helper macros */
#define to_sba_request(tx)		\
	container_of(tx, struct sba_request, tx)
#define to_sba_device(dchan)		\
	container_of(dchan, struct sba_device, dma_chan)

98 99
/* ===== Driver data structures ===== */

100 101 102 103 104
enum sba_request_flags {
	SBA_REQUEST_STATE_FREE		= 0x001,
	SBA_REQUEST_STATE_ALLOCED	= 0x002,
	SBA_REQUEST_STATE_PENDING	= 0x004,
	SBA_REQUEST_STATE_ACTIVE	= 0x008,
105
	SBA_REQUEST_STATE_ABORTED	= 0x010,
106 107
	SBA_REQUEST_STATE_MASK		= 0x0ff,
	SBA_REQUEST_FENCE		= 0x100,
108 109 110 111 112 113
};

struct sba_request {
	/* Global state */
	struct list_head node;
	struct sba_device *sba;
114
	u32 flags;
115 116 117 118 119 120 121
	/* Chained requests management */
	struct sba_request *first;
	struct list_head next;
	atomic_t next_pending_count;
	/* BRCM message data */
	struct brcm_message msg;
	struct dma_async_tx_descriptor tx;
122 123
	/* SBA commands */
	struct brcm_sba_command cmds[0];
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
};

enum sba_version {
	SBA_VER_1 = 0,
	SBA_VER_2
};

struct sba_device {
	/* Underlying device */
	struct device *dev;
	/* DT configuration parameters */
	enum sba_version ver;
	/* Derived configuration parameters */
	u32 max_req;
	u32 hw_buf_size;
	u32 hw_resp_size;
	u32 max_pq_coefs;
	u32 max_pq_srcs;
	u32 max_cmd_per_req;
	u32 max_xor_srcs;
	u32 max_resp_pool_size;
	u32 max_cmds_pool_size;
	/* Maibox client and Mailbox channels */
	struct mbox_client client;
148
	struct mbox_chan *mchan;
149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164
	struct device *mbox_dev;
	/* DMA device and DMA channel */
	struct dma_device dma_dev;
	struct dma_chan dma_chan;
	/* DMA channel resources */
	void *resp_base;
	dma_addr_t resp_dma_base;
	void *cmds_base;
	dma_addr_t cmds_dma_base;
	spinlock_t reqs_lock;
	bool reqs_fence;
	struct list_head reqs_alloc_list;
	struct list_head reqs_pending_list;
	struct list_head reqs_active_list;
	struct list_head reqs_aborted_list;
	struct list_head reqs_free_list;
165 166 167
	/* DebugFS directory entries */
	struct dentry *root;
	struct dentry *stats;
168 169
};

170
/* ====== Command helper routines ===== */
171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201

static inline u64 __pure sba_cmd_enc(u64 cmd, u32 val, u32 shift, u32 mask)
{
	cmd &= ~((u64)mask << shift);
	cmd |= ((u64)(val & mask) << shift);
	return cmd;
}

static inline u32 __pure sba_cmd_load_c_mdata(u32 b0)
{
	return b0 & SBA_C_MDATA_BNUMx_MASK;
}

static inline u32 __pure sba_cmd_write_c_mdata(u32 b0)
{
	return b0 & SBA_C_MDATA_BNUMx_MASK;
}

static inline u32 __pure sba_cmd_xor_c_mdata(u32 b1, u32 b0)
{
	return (b0 & SBA_C_MDATA_BNUMx_MASK) |
	       ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1));
}

static inline u32 __pure sba_cmd_pq_c_mdata(u32 d, u32 b1, u32 b0)
{
	return (b0 & SBA_C_MDATA_BNUMx_MASK) |
	       ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1)) |
	       ((d & SBA_C_MDATA_DNUM_MASK) << SBA_C_MDATA_DNUM_SHIFT);
}

202
/* ====== General helper routines ===== */
203 204 205

static struct sba_request *sba_alloc_request(struct sba_device *sba)
{
206
	bool found = false;
207 208 209 210
	unsigned long flags;
	struct sba_request *req = NULL;

	spin_lock_irqsave(&sba->reqs_lock, flags);
211 212 213 214 215 216 217
	list_for_each_entry(req, &sba->reqs_free_list, node) {
		if (async_tx_test_ack(&req->tx)) {
			list_move_tail(&req->node, &sba->reqs_alloc_list);
			found = true;
			break;
		}
	}
218
	spin_unlock_irqrestore(&sba->reqs_lock, flags);
219

220
	if (!found) {
221 222 223 224 225 226
		/*
		 * We have no more free requests so, we peek
		 * mailbox channels hoping few active requests
		 * would have completed which will create more
		 * room for new requests.
		 */
227
		mbox_client_peek_data(sba->mchan);
228
		return NULL;
229
	}
230

231
	req->flags = SBA_REQUEST_STATE_ALLOCED;
232 233 234 235 236
	req->first = req;
	INIT_LIST_HEAD(&req->next);
	atomic_set(&req->next_pending_count, 1);

	dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan);
237
	async_tx_ack(&req->tx);
238 239 240 241 242 243 244 245 246

	return req;
}

/* Note: Must be called with sba->reqs_lock held */
static void _sba_pending_request(struct sba_device *sba,
				 struct sba_request *req)
{
	lockdep_assert_held(&sba->reqs_lock);
247 248
	req->flags &= ~SBA_REQUEST_STATE_MASK;
	req->flags |= SBA_REQUEST_STATE_PENDING;
249 250 251 252 253 254 255 256 257 258 259 260 261 262
	list_move_tail(&req->node, &sba->reqs_pending_list);
	if (list_empty(&sba->reqs_active_list))
		sba->reqs_fence = false;
}

/* Note: Must be called with sba->reqs_lock held */
static bool _sba_active_request(struct sba_device *sba,
				struct sba_request *req)
{
	lockdep_assert_held(&sba->reqs_lock);
	if (list_empty(&sba->reqs_active_list))
		sba->reqs_fence = false;
	if (sba->reqs_fence)
		return false;
263 264
	req->flags &= ~SBA_REQUEST_STATE_MASK;
	req->flags |= SBA_REQUEST_STATE_ACTIVE;
265
	list_move_tail(&req->node, &sba->reqs_active_list);
266
	if (req->flags & SBA_REQUEST_FENCE)
267 268 269 270 271 272 273 274 275
		sba->reqs_fence = true;
	return true;
}

/* Note: Must be called with sba->reqs_lock held */
static void _sba_abort_request(struct sba_device *sba,
			       struct sba_request *req)
{
	lockdep_assert_held(&sba->reqs_lock);
276 277
	req->flags &= ~SBA_REQUEST_STATE_MASK;
	req->flags |= SBA_REQUEST_STATE_ABORTED;
278 279 280 281 282 283 284 285 286 287
	list_move_tail(&req->node, &sba->reqs_aborted_list);
	if (list_empty(&sba->reqs_active_list))
		sba->reqs_fence = false;
}

/* Note: Must be called with sba->reqs_lock held */
static void _sba_free_request(struct sba_device *sba,
			      struct sba_request *req)
{
	lockdep_assert_held(&sba->reqs_lock);
288 289
	req->flags &= ~SBA_REQUEST_STATE_MASK;
	req->flags |= SBA_REQUEST_STATE_FREE;
290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319
	list_move_tail(&req->node, &sba->reqs_free_list);
	if (list_empty(&sba->reqs_active_list))
		sba->reqs_fence = false;
}

static void sba_free_chained_requests(struct sba_request *req)
{
	unsigned long flags;
	struct sba_request *nreq;
	struct sba_device *sba = req->sba;

	spin_lock_irqsave(&sba->reqs_lock, flags);

	_sba_free_request(sba, req);
	list_for_each_entry(nreq, &req->next, next)
		_sba_free_request(sba, nreq);

	spin_unlock_irqrestore(&sba->reqs_lock, flags);
}

static void sba_chain_request(struct sba_request *first,
			      struct sba_request *req)
{
	unsigned long flags;
	struct sba_device *sba = req->sba;

	spin_lock_irqsave(&sba->reqs_lock, flags);

	list_add_tail(&req->next, &first->next);
	req->first = first;
320
	atomic_inc(&first->next_pending_count);
321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364

	spin_unlock_irqrestore(&sba->reqs_lock, flags);
}

static void sba_cleanup_nonpending_requests(struct sba_device *sba)
{
	unsigned long flags;
	struct sba_request *req, *req1;

	spin_lock_irqsave(&sba->reqs_lock, flags);

	/* Freeup all alloced request */
	list_for_each_entry_safe(req, req1, &sba->reqs_alloc_list, node)
		_sba_free_request(sba, req);

	/* Set all active requests as aborted */
	list_for_each_entry_safe(req, req1, &sba->reqs_active_list, node)
		_sba_abort_request(sba, req);

	/*
	 * Note: We expect that aborted request will be eventually
	 * freed by sba_receive_message()
	 */

	spin_unlock_irqrestore(&sba->reqs_lock, flags);
}

static void sba_cleanup_pending_requests(struct sba_device *sba)
{
	unsigned long flags;
	struct sba_request *req, *req1;

	spin_lock_irqsave(&sba->reqs_lock, flags);

	/* Freeup all pending request */
	list_for_each_entry_safe(req, req1, &sba->reqs_pending_list, node)
		_sba_free_request(sba, req);

	spin_unlock_irqrestore(&sba->reqs_lock, flags);
}

static int sba_send_mbox_request(struct sba_device *sba,
				 struct sba_request *req)
{
365
	int ret = 0;
366 367 368

	/* Send message for the request */
	req->msg.error = 0;
369
	ret = mbox_send_message(sba->mchan, &req->msg);
370 371 372 373
	if (ret < 0) {
		dev_err(sba->dev, "send message failed with error %d", ret);
		return ret;
	}
374 375

	/* Check error returned by mailbox controller */
376 377 378 379 380
	ret = req->msg.error;
	if (ret < 0) {
		dev_err(sba->dev, "message error %d", ret);
	}

381
	/* Signal txdone for mailbox channel */
382
	mbox_client_txdone(sba->mchan, ret);
383 384

	return ret;
385 386
}

387 388
/* Note: Must be called with sba->reqs_lock held */
static void _sba_process_pending_requests(struct sba_device *sba)
389 390
{
	int ret;
391 392 393
	u32 count;
	struct sba_request *req;

394 395
	/* Process few pending requests */
	count = SBA_MAX_MSG_SEND_PER_MBOX_CHANNEL;
396 397 398 399 400
	while (!list_empty(&sba->reqs_pending_list) && count) {
		/* Get the first pending request */
		req = list_first_entry(&sba->reqs_pending_list,
				       struct sba_request, node);

401 402 403 404 405 406 407 408 409 410
		/* Try to make request active */
		if (!_sba_active_request(sba, req))
			break;

		/* Send request to mailbox channel */
		ret = sba_send_mbox_request(sba, req);
		if (ret < 0) {
			_sba_pending_request(sba, req);
			break;
		}
411 412 413

		count--;
	}
414
}
415

416 417 418 419 420 421
static void sba_process_received_request(struct sba_device *sba,
					 struct sba_request *req)
{
	unsigned long flags;
	struct dma_async_tx_descriptor *tx;
	struct sba_request *nreq, *first = req->first;
422

423 424 425
	/* Process only after all chained requests are received */
	if (!atomic_dec_return(&first->next_pending_count)) {
		tx = &first->tx;
426 427 428

		WARN_ON(tx->cookie < 0);
		if (tx->cookie > 0) {
429
			spin_lock_irqsave(&sba->reqs_lock, flags);
430
			dma_cookie_complete(tx);
431
			spin_unlock_irqrestore(&sba->reqs_lock, flags);
432 433 434 435 436 437 438 439 440 441
			dmaengine_desc_get_callback_invoke(tx, NULL);
			dma_descriptor_unmap(tx);
			tx->callback = NULL;
			tx->callback_result = NULL;
		}

		dma_run_dependencies(tx);

		spin_lock_irqsave(&sba->reqs_lock, flags);

442 443 444 445
		/* Free all requests chained to first request */
		list_for_each_entry(nreq, &first->next, next)
			_sba_free_request(sba, nreq);
		INIT_LIST_HEAD(&first->next);
446

447 448
		/* Free the first request */
		_sba_free_request(sba, first);
449

450 451
		/* Process pending requests */
		_sba_process_pending_requests(sba);
452

453 454
		spin_unlock_irqrestore(&sba->reqs_lock, flags);
	}
455 456
}

457 458 459 460 461
static void sba_write_stats_in_seqfile(struct sba_device *sba,
				       struct seq_file *file)
{
	unsigned long flags;
	struct sba_request *req;
462 463
	u32 free_count = 0, alloced_count = 0;
	u32 pending_count = 0, active_count = 0, aborted_count = 0;
464 465 466 467

	spin_lock_irqsave(&sba->reqs_lock, flags);

	list_for_each_entry(req, &sba->reqs_free_list, node)
468 469
		if (async_tx_test_ack(&req->tx))
			free_count++;
470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492

	list_for_each_entry(req, &sba->reqs_alloc_list, node)
		alloced_count++;

	list_for_each_entry(req, &sba->reqs_pending_list, node)
		pending_count++;

	list_for_each_entry(req, &sba->reqs_active_list, node)
		active_count++;

	list_for_each_entry(req, &sba->reqs_aborted_list, node)
		aborted_count++;

	spin_unlock_irqrestore(&sba->reqs_lock, flags);

	seq_printf(file, "maximum requests   = %d\n", sba->max_req);
	seq_printf(file, "free requests      = %d\n", free_count);
	seq_printf(file, "alloced requests   = %d\n", alloced_count);
	seq_printf(file, "pending requests   = %d\n", pending_count);
	seq_printf(file, "active requests    = %d\n", active_count);
	seq_printf(file, "aborted requests   = %d\n", aborted_count);
}

493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514
/* ====== DMAENGINE callbacks ===== */

static void sba_free_chan_resources(struct dma_chan *dchan)
{
	/*
	 * Channel resources are pre-alloced so we just free-up
	 * whatever we can so that we can re-use pre-alloced
	 * channel resources next time.
	 */
	sba_cleanup_nonpending_requests(to_sba_device(dchan));
}

static int sba_device_terminate_all(struct dma_chan *dchan)
{
	/* Cleanup all pending requests */
	sba_cleanup_pending_requests(to_sba_device(dchan));

	return 0;
}

static void sba_issue_pending(struct dma_chan *dchan)
{
515
	unsigned long flags;
516 517
	struct sba_device *sba = to_sba_device(dchan);

518 519 520 521
	/* Process pending requests */
	spin_lock_irqsave(&sba->reqs_lock, flags);
	_sba_process_pending_requests(sba);
	spin_unlock_irqrestore(&sba->reqs_lock, flags);
522 523
}

524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558
static dma_cookie_t sba_tx_submit(struct dma_async_tx_descriptor *tx)
{
	unsigned long flags;
	dma_cookie_t cookie;
	struct sba_device *sba;
	struct sba_request *req, *nreq;

	if (unlikely(!tx))
		return -EINVAL;

	sba = to_sba_device(tx->chan);
	req = to_sba_request(tx);

	/* Assign cookie and mark all chained requests pending */
	spin_lock_irqsave(&sba->reqs_lock, flags);
	cookie = dma_cookie_assign(tx);
	_sba_pending_request(sba, req);
	list_for_each_entry(nreq, &req->next, next)
		_sba_pending_request(sba, nreq);
	spin_unlock_irqrestore(&sba->reqs_lock, flags);

	return cookie;
}

static enum dma_status sba_tx_status(struct dma_chan *dchan,
				     dma_cookie_t cookie,
				     struct dma_tx_state *txstate)
{
	enum dma_status ret;
	struct sba_device *sba = to_sba_device(dchan);

	ret = dma_cookie_status(dchan, cookie, txstate);
	if (ret == DMA_COMPLETE)
		return ret;

559
	mbox_client_peek_data(sba->mchan);
560

561 562 563 564 565 566 567 568 569
	return dma_cookie_status(dchan, cookie, txstate);
}

static void sba_fillup_interrupt_msg(struct sba_request *req,
				     struct brcm_sba_command *cmds,
				     struct brcm_message *msg)
{
	u64 cmd;
	u32 c_mdata;
570
	dma_addr_t resp_dma = req->tx.phys;
571 572 573 574 575 576 577 578 579 580 581 582 583 584 585
	struct brcm_sba_command *cmdsp = cmds;

	/* Type-B command to load dummy data into buf0 */
	cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
			  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
	cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size,
			  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
	c_mdata = sba_cmd_load_c_mdata(0);
	cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
			  SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
	cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
			  SBA_CMD_SHIFT, SBA_CMD_MASK);
	cmdsp->cmd = cmd;
	*cmdsp->cmd_dma = cpu_to_le64(cmd);
	cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
586
	cmdsp->data = resp_dma;
587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606
	cmdsp->data_len = req->sba->hw_resp_size;
	cmdsp++;

	/* Type-A command to write buf0 to dummy location */
	cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
			  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
	cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size,
			  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
	cmd = sba_cmd_enc(cmd, 0x1,
			  SBA_RESP_SHIFT, SBA_RESP_MASK);
	c_mdata = sba_cmd_write_c_mdata(0);
	cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
			  SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
	cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
			  SBA_CMD_SHIFT, SBA_CMD_MASK);
	cmdsp->cmd = cmd;
	*cmdsp->cmd_dma = cpu_to_le64(cmd);
	cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
	if (req->sba->hw_resp_size) {
		cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
607
		cmdsp->resp = resp_dma;
608 609 610
		cmdsp->resp_len = req->sba->hw_resp_size;
	}
	cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
611
	cmdsp->data = resp_dma;
612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637
	cmdsp->data_len = req->sba->hw_resp_size;
	cmdsp++;

	/* Fillup brcm_message */
	msg->type = BRCM_MESSAGE_SBA;
	msg->sba.cmds = cmds;
	msg->sba.cmds_count = cmdsp - cmds;
	msg->ctx = req;
	msg->error = 0;
}

static struct dma_async_tx_descriptor *
sba_prep_dma_interrupt(struct dma_chan *dchan, unsigned long flags)
{
	struct sba_request *req = NULL;
	struct sba_device *sba = to_sba_device(dchan);

	/* Alloc new request */
	req = sba_alloc_request(sba);
	if (!req)
		return NULL;

	/*
	 * Force fence so that no requests are submitted
	 * until DMA callback for this request is invoked.
	 */
638
	req->flags |= SBA_REQUEST_FENCE;
639 640 641 642 643 644 645 646

	/* Fillup request message */
	sba_fillup_interrupt_msg(req, req->cmds, &req->msg);

	/* Init async_tx descriptor */
	req->tx.flags = flags;
	req->tx.cookie = -EBUSY;

647
	return &req->tx;
648 649 650 651 652 653 654 655 656 657
}

static void sba_fillup_memcpy_msg(struct sba_request *req,
				  struct brcm_sba_command *cmds,
				  struct brcm_message *msg,
				  dma_addr_t msg_offset, size_t msg_len,
				  dma_addr_t dst, dma_addr_t src)
{
	u64 cmd;
	u32 c_mdata;
658
	dma_addr_t resp_dma = req->tx.phys;
659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694
	struct brcm_sba_command *cmdsp = cmds;

	/* Type-B command to load data into buf0 */
	cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
			  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
	cmd = sba_cmd_enc(cmd, msg_len,
			  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
	c_mdata = sba_cmd_load_c_mdata(0);
	cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
			  SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
	cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
			  SBA_CMD_SHIFT, SBA_CMD_MASK);
	cmdsp->cmd = cmd;
	*cmdsp->cmd_dma = cpu_to_le64(cmd);
	cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
	cmdsp->data = src + msg_offset;
	cmdsp->data_len = msg_len;
	cmdsp++;

	/* Type-A command to write buf0 */
	cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
			  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
	cmd = sba_cmd_enc(cmd, msg_len,
			  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
	cmd = sba_cmd_enc(cmd, 0x1,
			  SBA_RESP_SHIFT, SBA_RESP_MASK);
	c_mdata = sba_cmd_write_c_mdata(0);
	cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
			  SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
	cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
			  SBA_CMD_SHIFT, SBA_CMD_MASK);
	cmdsp->cmd = cmd;
	*cmdsp->cmd_dma = cpu_to_le64(cmd);
	cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
	if (req->sba->hw_resp_size) {
		cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
695
		cmdsp->resp = resp_dma;
696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721
		cmdsp->resp_len = req->sba->hw_resp_size;
	}
	cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
	cmdsp->data = dst + msg_offset;
	cmdsp->data_len = msg_len;
	cmdsp++;

	/* Fillup brcm_message */
	msg->type = BRCM_MESSAGE_SBA;
	msg->sba.cmds = cmds;
	msg->sba.cmds_count = cmdsp - cmds;
	msg->ctx = req;
	msg->error = 0;
}

static struct sba_request *
sba_prep_dma_memcpy_req(struct sba_device *sba,
			dma_addr_t off, dma_addr_t dst, dma_addr_t src,
			size_t len, unsigned long flags)
{
	struct sba_request *req = NULL;

	/* Alloc new request */
	req = sba_alloc_request(sba);
	if (!req)
		return NULL;
722 723
	if (flags & DMA_PREP_FENCE)
		req->flags |= SBA_REQUEST_FENCE;
724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777

	/* Fillup request message */
	sba_fillup_memcpy_msg(req, req->cmds, &req->msg,
			      off, len, dst, src);

	/* Init async_tx descriptor */
	req->tx.flags = flags;
	req->tx.cookie = -EBUSY;

	return req;
}

static struct dma_async_tx_descriptor *
sba_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t src,
		    size_t len, unsigned long flags)
{
	size_t req_len;
	dma_addr_t off = 0;
	struct sba_device *sba = to_sba_device(dchan);
	struct sba_request *first = NULL, *req;

	/* Create chained requests where each request is upto hw_buf_size */
	while (len) {
		req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;

		req = sba_prep_dma_memcpy_req(sba, off, dst, src,
					      req_len, flags);
		if (!req) {
			if (first)
				sba_free_chained_requests(first);
			return NULL;
		}

		if (first)
			sba_chain_request(first, req);
		else
			first = req;

		off += req_len;
		len -= req_len;
	}

	return (first) ? &first->tx : NULL;
}

static void sba_fillup_xor_msg(struct sba_request *req,
				struct brcm_sba_command *cmds,
				struct brcm_message *msg,
				dma_addr_t msg_offset, size_t msg_len,
				dma_addr_t dst, dma_addr_t *src, u32 src_cnt)
{
	u64 cmd;
	u32 c_mdata;
	unsigned int i;
778
	dma_addr_t resp_dma = req->tx.phys;
779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833
	struct brcm_sba_command *cmdsp = cmds;

	/* Type-B command to load data into buf0 */
	cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
			  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
	cmd = sba_cmd_enc(cmd, msg_len,
			  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
	c_mdata = sba_cmd_load_c_mdata(0);
	cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
			  SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
	cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
			  SBA_CMD_SHIFT, SBA_CMD_MASK);
	cmdsp->cmd = cmd;
	*cmdsp->cmd_dma = cpu_to_le64(cmd);
	cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
	cmdsp->data = src[0] + msg_offset;
	cmdsp->data_len = msg_len;
	cmdsp++;

	/* Type-B commands to xor data with buf0 and put it back in buf0 */
	for (i = 1; i < src_cnt; i++) {
		cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
				  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
		cmd = sba_cmd_enc(cmd, msg_len,
				  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
		c_mdata = sba_cmd_xor_c_mdata(0, 0);
		cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
				  SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
		cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
				  SBA_CMD_SHIFT, SBA_CMD_MASK);
		cmdsp->cmd = cmd;
		*cmdsp->cmd_dma = cpu_to_le64(cmd);
		cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
		cmdsp->data = src[i] + msg_offset;
		cmdsp->data_len = msg_len;
		cmdsp++;
	}

	/* Type-A command to write buf0 */
	cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
			  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
	cmd = sba_cmd_enc(cmd, msg_len,
			  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
	cmd = sba_cmd_enc(cmd, 0x1,
			  SBA_RESP_SHIFT, SBA_RESP_MASK);
	c_mdata = sba_cmd_write_c_mdata(0);
	cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
			  SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
	cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
			  SBA_CMD_SHIFT, SBA_CMD_MASK);
	cmdsp->cmd = cmd;
	*cmdsp->cmd_dma = cpu_to_le64(cmd);
	cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
	if (req->sba->hw_resp_size) {
		cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
834
		cmdsp->resp = resp_dma;
835 836 837 838 839 840 841 842 843 844 845 846 847 848 849
		cmdsp->resp_len = req->sba->hw_resp_size;
	}
	cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
	cmdsp->data = dst + msg_offset;
	cmdsp->data_len = msg_len;
	cmdsp++;

	/* Fillup brcm_message */
	msg->type = BRCM_MESSAGE_SBA;
	msg->sba.cmds = cmds;
	msg->sba.cmds_count = cmdsp - cmds;
	msg->ctx = req;
	msg->error = 0;
}

850
static struct sba_request *
851 852 853 854 855 856 857 858 859 860
sba_prep_dma_xor_req(struct sba_device *sba,
		     dma_addr_t off, dma_addr_t dst, dma_addr_t *src,
		     u32 src_cnt, size_t len, unsigned long flags)
{
	struct sba_request *req = NULL;

	/* Alloc new request */
	req = sba_alloc_request(sba);
	if (!req)
		return NULL;
861 862
	if (flags & DMA_PREP_FENCE)
		req->flags |= SBA_REQUEST_FENCE;
863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922

	/* Fillup request message */
	sba_fillup_xor_msg(req, req->cmds, &req->msg,
			   off, len, dst, src, src_cnt);

	/* Init async_tx descriptor */
	req->tx.flags = flags;
	req->tx.cookie = -EBUSY;

	return req;
}

static struct dma_async_tx_descriptor *
sba_prep_dma_xor(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t *src,
		 u32 src_cnt, size_t len, unsigned long flags)
{
	size_t req_len;
	dma_addr_t off = 0;
	struct sba_device *sba = to_sba_device(dchan);
	struct sba_request *first = NULL, *req;

	/* Sanity checks */
	if (unlikely(src_cnt > sba->max_xor_srcs))
		return NULL;

	/* Create chained requests where each request is upto hw_buf_size */
	while (len) {
		req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;

		req = sba_prep_dma_xor_req(sba, off, dst, src, src_cnt,
					   req_len, flags);
		if (!req) {
			if (first)
				sba_free_chained_requests(first);
			return NULL;
		}

		if (first)
			sba_chain_request(first, req);
		else
			first = req;

		off += req_len;
		len -= req_len;
	}

	return (first) ? &first->tx : NULL;
}

static void sba_fillup_pq_msg(struct sba_request *req,
				bool pq_continue,
				struct brcm_sba_command *cmds,
				struct brcm_message *msg,
				dma_addr_t msg_offset, size_t msg_len,
				dma_addr_t *dst_p, dma_addr_t *dst_q,
				const u8 *scf, dma_addr_t *src, u32 src_cnt)
{
	u64 cmd;
	u32 c_mdata;
	unsigned int i;
923
	dma_addr_t resp_dma = req->tx.phys;
924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016
	struct brcm_sba_command *cmdsp = cmds;

	if (pq_continue) {
		/* Type-B command to load old P into buf0 */
		if (dst_p) {
			cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
				SBA_TYPE_SHIFT, SBA_TYPE_MASK);
			cmd = sba_cmd_enc(cmd, msg_len,
				SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
			c_mdata = sba_cmd_load_c_mdata(0);
			cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
				SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
			cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
				SBA_CMD_SHIFT, SBA_CMD_MASK);
			cmdsp->cmd = cmd;
			*cmdsp->cmd_dma = cpu_to_le64(cmd);
			cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
			cmdsp->data = *dst_p + msg_offset;
			cmdsp->data_len = msg_len;
			cmdsp++;
		}

		/* Type-B command to load old Q into buf1 */
		if (dst_q) {
			cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
				SBA_TYPE_SHIFT, SBA_TYPE_MASK);
			cmd = sba_cmd_enc(cmd, msg_len,
				SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
			c_mdata = sba_cmd_load_c_mdata(1);
			cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
				SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
			cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
				SBA_CMD_SHIFT, SBA_CMD_MASK);
			cmdsp->cmd = cmd;
			*cmdsp->cmd_dma = cpu_to_le64(cmd);
			cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
			cmdsp->data = *dst_q + msg_offset;
			cmdsp->data_len = msg_len;
			cmdsp++;
		}
	} else {
		/* Type-A command to zero all buffers */
		cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
				  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
		cmd = sba_cmd_enc(cmd, msg_len,
				  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
		cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS,
				  SBA_CMD_SHIFT, SBA_CMD_MASK);
		cmdsp->cmd = cmd;
		*cmdsp->cmd_dma = cpu_to_le64(cmd);
		cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
		cmdsp++;
	}

	/* Type-B commands for generate P onto buf0 and Q onto buf1 */
	for (i = 0; i < src_cnt; i++) {
		cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
				  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
		cmd = sba_cmd_enc(cmd, msg_len,
				  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
		c_mdata = sba_cmd_pq_c_mdata(raid6_gflog[scf[i]], 1, 0);
		cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
				  SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
		cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
				  SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
		cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS_XOR,
				  SBA_CMD_SHIFT, SBA_CMD_MASK);
		cmdsp->cmd = cmd;
		*cmdsp->cmd_dma = cpu_to_le64(cmd);
		cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
		cmdsp->data = src[i] + msg_offset;
		cmdsp->data_len = msg_len;
		cmdsp++;
	}

	/* Type-A command to write buf0 */
	if (dst_p) {
		cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
				  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
		cmd = sba_cmd_enc(cmd, msg_len,
				  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
		cmd = sba_cmd_enc(cmd, 0x1,
				  SBA_RESP_SHIFT, SBA_RESP_MASK);
		c_mdata = sba_cmd_write_c_mdata(0);
		cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
				  SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
		cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
				  SBA_CMD_SHIFT, SBA_CMD_MASK);
		cmdsp->cmd = cmd;
		*cmdsp->cmd_dma = cpu_to_le64(cmd);
		cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
		if (req->sba->hw_resp_size) {
			cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
1017
			cmdsp->resp = resp_dma;
1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043
			cmdsp->resp_len = req->sba->hw_resp_size;
		}
		cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
		cmdsp->data = *dst_p + msg_offset;
		cmdsp->data_len = msg_len;
		cmdsp++;
	}

	/* Type-A command to write buf1 */
	if (dst_q) {
		cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
				  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
		cmd = sba_cmd_enc(cmd, msg_len,
				  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
		cmd = sba_cmd_enc(cmd, 0x1,
				  SBA_RESP_SHIFT, SBA_RESP_MASK);
		c_mdata = sba_cmd_write_c_mdata(1);
		cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
				  SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
		cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
				  SBA_CMD_SHIFT, SBA_CMD_MASK);
		cmdsp->cmd = cmd;
		*cmdsp->cmd_dma = cpu_to_le64(cmd);
		cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
		if (req->sba->hw_resp_size) {
			cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
1044
			cmdsp->resp = resp_dma;
1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060
			cmdsp->resp_len = req->sba->hw_resp_size;
		}
		cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
		cmdsp->data = *dst_q + msg_offset;
		cmdsp->data_len = msg_len;
		cmdsp++;
	}

	/* Fillup brcm_message */
	msg->type = BRCM_MESSAGE_SBA;
	msg->sba.cmds = cmds;
	msg->sba.cmds_count = cmdsp - cmds;
	msg->ctx = req;
	msg->error = 0;
}

1061
static struct sba_request *
1062 1063 1064 1065 1066 1067 1068 1069 1070 1071
sba_prep_dma_pq_req(struct sba_device *sba, dma_addr_t off,
		    dma_addr_t *dst_p, dma_addr_t *dst_q, dma_addr_t *src,
		    u32 src_cnt, const u8 *scf, size_t len, unsigned long flags)
{
	struct sba_request *req = NULL;

	/* Alloc new request */
	req = sba_alloc_request(sba);
	if (!req)
		return NULL;
1072 1073
	if (flags & DMA_PREP_FENCE)
		req->flags |= SBA_REQUEST_FENCE;
1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097

	/* Fillup request messages */
	sba_fillup_pq_msg(req, dmaf_continue(flags),
			  req->cmds, &req->msg,
			  off, len, dst_p, dst_q, scf, src, src_cnt);

	/* Init async_tx descriptor */
	req->tx.flags = flags;
	req->tx.cookie = -EBUSY;

	return req;
}

static void sba_fillup_pq_single_msg(struct sba_request *req,
				bool pq_continue,
				struct brcm_sba_command *cmds,
				struct brcm_message *msg,
				dma_addr_t msg_offset, size_t msg_len,
				dma_addr_t *dst_p, dma_addr_t *dst_q,
				dma_addr_t src, u8 scf)
{
	u64 cmd;
	u32 c_mdata;
	u8 pos, dpos = raid6_gflog[scf];
1098
	dma_addr_t resp_dma = req->tx.phys;
1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176
	struct brcm_sba_command *cmdsp = cmds;

	if (!dst_p)
		goto skip_p;

	if (pq_continue) {
		/* Type-B command to load old P into buf0 */
		cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
				  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
		cmd = sba_cmd_enc(cmd, msg_len,
				  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
		c_mdata = sba_cmd_load_c_mdata(0);
		cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
				  SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
		cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
				  SBA_CMD_SHIFT, SBA_CMD_MASK);
		cmdsp->cmd = cmd;
		*cmdsp->cmd_dma = cpu_to_le64(cmd);
		cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
		cmdsp->data = *dst_p + msg_offset;
		cmdsp->data_len = msg_len;
		cmdsp++;

		/*
		 * Type-B commands to xor data with buf0 and put it
		 * back in buf0
		 */
		cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
				  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
		cmd = sba_cmd_enc(cmd, msg_len,
				  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
		c_mdata = sba_cmd_xor_c_mdata(0, 0);
		cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
				  SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
		cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
				  SBA_CMD_SHIFT, SBA_CMD_MASK);
		cmdsp->cmd = cmd;
		*cmdsp->cmd_dma = cpu_to_le64(cmd);
		cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
		cmdsp->data = src + msg_offset;
		cmdsp->data_len = msg_len;
		cmdsp++;
	} else {
		/* Type-B command to load old P into buf0 */
		cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
				  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
		cmd = sba_cmd_enc(cmd, msg_len,
				  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
		c_mdata = sba_cmd_load_c_mdata(0);
		cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
				  SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
		cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
				  SBA_CMD_SHIFT, SBA_CMD_MASK);
		cmdsp->cmd = cmd;
		*cmdsp->cmd_dma = cpu_to_le64(cmd);
		cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
		cmdsp->data = src + msg_offset;
		cmdsp->data_len = msg_len;
		cmdsp++;
	}

	/* Type-A command to write buf0 */
	cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
			  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
	cmd = sba_cmd_enc(cmd, msg_len,
			  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
	cmd = sba_cmd_enc(cmd, 0x1,
			  SBA_RESP_SHIFT, SBA_RESP_MASK);
	c_mdata = sba_cmd_write_c_mdata(0);
	cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
			  SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
	cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
			  SBA_CMD_SHIFT, SBA_CMD_MASK);
	cmdsp->cmd = cmd;
	*cmdsp->cmd_dma = cpu_to_le64(cmd);
	cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
	if (req->sba->hw_resp_size) {
		cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
1177
		cmdsp->resp = resp_dma;
1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297
		cmdsp->resp_len = req->sba->hw_resp_size;
	}
	cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
	cmdsp->data = *dst_p + msg_offset;
	cmdsp->data_len = msg_len;
	cmdsp++;

skip_p:
	if (!dst_q)
		goto skip_q;

	/* Type-A command to zero all buffers */
	cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
			  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
	cmd = sba_cmd_enc(cmd, msg_len,
			  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
	cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS,
			  SBA_CMD_SHIFT, SBA_CMD_MASK);
	cmdsp->cmd = cmd;
	*cmdsp->cmd_dma = cpu_to_le64(cmd);
	cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
	cmdsp++;

	if (dpos == 255)
		goto skip_q_computation;
	pos = (dpos < req->sba->max_pq_coefs) ?
		dpos : (req->sba->max_pq_coefs - 1);

	/*
	 * Type-B command to generate initial Q from data
	 * and store output into buf0
	 */
	cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
			  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
	cmd = sba_cmd_enc(cmd, msg_len,
			  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
	c_mdata = sba_cmd_pq_c_mdata(pos, 0, 0);
	cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
			  SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
	cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
			  SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
	cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS,
			  SBA_CMD_SHIFT, SBA_CMD_MASK);
	cmdsp->cmd = cmd;
	*cmdsp->cmd_dma = cpu_to_le64(cmd);
	cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
	cmdsp->data = src + msg_offset;
	cmdsp->data_len = msg_len;
	cmdsp++;

	dpos -= pos;

	/* Multiple Type-A command to generate final Q */
	while (dpos) {
		pos = (dpos < req->sba->max_pq_coefs) ?
			dpos : (req->sba->max_pq_coefs - 1);

		/*
		 * Type-A command to generate Q with buf0 and
		 * buf1 store result in buf0
		 */
		cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
				  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
		cmd = sba_cmd_enc(cmd, msg_len,
				  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
		c_mdata = sba_cmd_pq_c_mdata(pos, 0, 1);
		cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
				  SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
		cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
				  SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
		cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS,
				  SBA_CMD_SHIFT, SBA_CMD_MASK);
		cmdsp->cmd = cmd;
		*cmdsp->cmd_dma = cpu_to_le64(cmd);
		cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
		cmdsp++;

		dpos -= pos;
	}

skip_q_computation:
	if (pq_continue) {
		/*
		 * Type-B command to XOR previous output with
		 * buf0 and write it into buf0
		 */
		cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
				  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
		cmd = sba_cmd_enc(cmd, msg_len,
				  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
		c_mdata = sba_cmd_xor_c_mdata(0, 0);
		cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
				  SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
		cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
				  SBA_CMD_SHIFT, SBA_CMD_MASK);
		cmdsp->cmd = cmd;
		*cmdsp->cmd_dma = cpu_to_le64(cmd);
		cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
		cmdsp->data = *dst_q + msg_offset;
		cmdsp->data_len = msg_len;
		cmdsp++;
	}

	/* Type-A command to write buf0 */
	cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
			  SBA_TYPE_SHIFT, SBA_TYPE_MASK);
	cmd = sba_cmd_enc(cmd, msg_len,
			  SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
	cmd = sba_cmd_enc(cmd, 0x1,
			  SBA_RESP_SHIFT, SBA_RESP_MASK);
	c_mdata = sba_cmd_write_c_mdata(0);
	cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
			  SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
	cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
			  SBA_CMD_SHIFT, SBA_CMD_MASK);
	cmdsp->cmd = cmd;
	*cmdsp->cmd_dma = cpu_to_le64(cmd);
	cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
	if (req->sba->hw_resp_size) {
		cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
1298
		cmdsp->resp = resp_dma;
1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314
		cmdsp->resp_len = req->sba->hw_resp_size;
	}
	cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
	cmdsp->data = *dst_q + msg_offset;
	cmdsp->data_len = msg_len;
	cmdsp++;

skip_q:
	/* Fillup brcm_message */
	msg->type = BRCM_MESSAGE_SBA;
	msg->sba.cmds = cmds;
	msg->sba.cmds_count = cmdsp - cmds;
	msg->ctx = req;
	msg->error = 0;
}

1315
static struct sba_request *
1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326
sba_prep_dma_pq_single_req(struct sba_device *sba, dma_addr_t off,
			   dma_addr_t *dst_p, dma_addr_t *dst_q,
			   dma_addr_t src, u8 scf, size_t len,
			   unsigned long flags)
{
	struct sba_request *req = NULL;

	/* Alloc new request */
	req = sba_alloc_request(sba);
	if (!req)
		return NULL;
1327 1328
	if (flags & DMA_PREP_FENCE)
		req->flags |= SBA_REQUEST_FENCE;
1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445

	/* Fillup request messages */
	sba_fillup_pq_single_msg(req,  dmaf_continue(flags),
				 req->cmds, &req->msg, off, len,
				 dst_p, dst_q, src, scf);

	/* Init async_tx descriptor */
	req->tx.flags = flags;
	req->tx.cookie = -EBUSY;

	return req;
}

static struct dma_async_tx_descriptor *
sba_prep_dma_pq(struct dma_chan *dchan, dma_addr_t *dst, dma_addr_t *src,
		u32 src_cnt, const u8 *scf, size_t len, unsigned long flags)
{
	u32 i, dst_q_index;
	size_t req_len;
	bool slow = false;
	dma_addr_t off = 0;
	dma_addr_t *dst_p = NULL, *dst_q = NULL;
	struct sba_device *sba = to_sba_device(dchan);
	struct sba_request *first = NULL, *req;

	/* Sanity checks */
	if (unlikely(src_cnt > sba->max_pq_srcs))
		return NULL;
	for (i = 0; i < src_cnt; i++)
		if (sba->max_pq_coefs <= raid6_gflog[scf[i]])
			slow = true;

	/* Figure-out P and Q destination addresses */
	if (!(flags & DMA_PREP_PQ_DISABLE_P))
		dst_p = &dst[0];
	if (!(flags & DMA_PREP_PQ_DISABLE_Q))
		dst_q = &dst[1];

	/* Create chained requests where each request is upto hw_buf_size */
	while (len) {
		req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;

		if (slow) {
			dst_q_index = src_cnt;

			if (dst_q) {
				for (i = 0; i < src_cnt; i++) {
					if (*dst_q == src[i]) {
						dst_q_index = i;
						break;
					}
				}
			}

			if (dst_q_index < src_cnt) {
				i = dst_q_index;
				req = sba_prep_dma_pq_single_req(sba,
					off, dst_p, dst_q, src[i], scf[i],
					req_len, flags | DMA_PREP_FENCE);
				if (!req)
					goto fail;

				if (first)
					sba_chain_request(first, req);
				else
					first = req;

				flags |= DMA_PREP_CONTINUE;
			}

			for (i = 0; i < src_cnt; i++) {
				if (dst_q_index == i)
					continue;

				req = sba_prep_dma_pq_single_req(sba,
					off, dst_p, dst_q, src[i], scf[i],
					req_len, flags | DMA_PREP_FENCE);
				if (!req)
					goto fail;

				if (first)
					sba_chain_request(first, req);
				else
					first = req;

				flags |= DMA_PREP_CONTINUE;
			}
		} else {
			req = sba_prep_dma_pq_req(sba, off,
						  dst_p, dst_q, src, src_cnt,
						  scf, req_len, flags);
			if (!req)
				goto fail;

			if (first)
				sba_chain_request(first, req);
			else
				first = req;
		}

		off += req_len;
		len -= req_len;
	}

	return (first) ? &first->tx : NULL;

fail:
	if (first)
		sba_free_chained_requests(first);
	return NULL;
}

/* ====== Mailbox callbacks ===== */

static void sba_receive_message(struct mbox_client *cl, void *msg)
{
	struct brcm_message *m = msg;
1446
	struct sba_request *req = m->ctx;
1447 1448 1449 1450 1451 1452 1453
	struct sba_device *sba = req->sba;

	/* Error count if message has error */
	if (m->error < 0)
		dev_err(sba->dev, "%s got message with error %d",
			dma_chan_name(&sba->dma_chan), m->error);

1454 1455
	/* Process received request */
	sba_process_received_request(sba, req);
1456 1457
}

1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470
/* ====== Debugfs callbacks ====== */

static int sba_debugfs_stats_show(struct seq_file *file, void *offset)
{
	struct platform_device *pdev = to_platform_device(file->private);
	struct sba_device *sba = platform_get_drvdata(pdev);

	/* Write stats in file */
	sba_write_stats_in_seqfile(sba, file);

	return 0;
}

1471 1472 1473 1474
/* ====== Platform driver routines ===== */

static int sba_prealloc_channel_resources(struct sba_device *sba)
{
1475
	int i, j, ret = 0;
1476 1477
	struct sba_request *req = NULL;

1478
	sba->resp_base = dma_alloc_coherent(sba->mbox_dev,
1479 1480 1481 1482 1483
					    sba->max_resp_pool_size,
					    &sba->resp_dma_base, GFP_KERNEL);
	if (!sba->resp_base)
		return -ENOMEM;

1484
	sba->cmds_base = dma_alloc_coherent(sba->mbox_dev,
1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499
					    sba->max_cmds_pool_size,
					    &sba->cmds_dma_base, GFP_KERNEL);
	if (!sba->cmds_base) {
		ret = -ENOMEM;
		goto fail_free_resp_pool;
	}

	spin_lock_init(&sba->reqs_lock);
	sba->reqs_fence = false;
	INIT_LIST_HEAD(&sba->reqs_alloc_list);
	INIT_LIST_HEAD(&sba->reqs_pending_list);
	INIT_LIST_HEAD(&sba->reqs_active_list);
	INIT_LIST_HEAD(&sba->reqs_aborted_list);
	INIT_LIST_HEAD(&sba->reqs_free_list);

1500
	for (i = 0; i < sba->max_req; i++) {
1501
		req = devm_kzalloc(sba->dev,
1502 1503
				   struct_size(req, cmds, sba->max_cmd_per_req),
				   GFP_KERNEL);
1504 1505 1506 1507
		if (!req) {
			ret = -ENOMEM;
			goto fail_free_cmds_pool;
		}
1508 1509
		INIT_LIST_HEAD(&req->node);
		req->sba = sba;
1510
		req->flags = SBA_REQUEST_STATE_FREE;
1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522
		INIT_LIST_HEAD(&req->next);
		atomic_set(&req->next_pending_count, 0);
		for (j = 0; j < sba->max_cmd_per_req; j++) {
			req->cmds[j].cmd = 0;
			req->cmds[j].cmd_dma = sba->cmds_base +
				(i * sba->max_cmd_per_req + j) * sizeof(u64);
			req->cmds[j].cmd_dma_addr = sba->cmds_dma_base +
				(i * sba->max_cmd_per_req + j) * sizeof(u64);
			req->cmds[j].flags = 0;
		}
		memset(&req->msg, 0, sizeof(req->msg));
		dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan);
1523
		async_tx_ack(&req->tx);
1524
		req->tx.tx_submit = sba_tx_submit;
1525
		req->tx.phys = sba->resp_dma_base + i * sba->hw_resp_size;
1526 1527 1528 1529 1530 1531
		list_add_tail(&req->node, &sba->reqs_free_list);
	}

	return 0;

fail_free_cmds_pool:
1532
	dma_free_coherent(sba->mbox_dev,
1533 1534 1535
			  sba->max_cmds_pool_size,
			  sba->cmds_base, sba->cmds_dma_base);
fail_free_resp_pool:
1536
	dma_free_coherent(sba->mbox_dev,
1537 1538 1539 1540 1541 1542 1543 1544
			  sba->max_resp_pool_size,
			  sba->resp_base, sba->resp_dma_base);
	return ret;
}

static void sba_freeup_channel_resources(struct sba_device *sba)
{
	dmaengine_terminate_all(&sba->dma_chan);
1545
	dma_free_coherent(sba->mbox_dev, sba->max_cmds_pool_size,
1546
			  sba->cmds_base, sba->cmds_dma_base);
1547
	dma_free_coherent(sba->mbox_dev, sba->max_resp_pool_size,
1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624
			  sba->resp_base, sba->resp_dma_base);
	sba->resp_base = NULL;
	sba->resp_dma_base = 0;
}

static int sba_async_register(struct sba_device *sba)
{
	int ret;
	struct dma_device *dma_dev = &sba->dma_dev;

	/* Initialize DMA channel cookie */
	sba->dma_chan.device = dma_dev;
	dma_cookie_init(&sba->dma_chan);

	/* Initialize DMA device capability mask */
	dma_cap_zero(dma_dev->cap_mask);
	dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
	dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
	dma_cap_set(DMA_XOR, dma_dev->cap_mask);
	dma_cap_set(DMA_PQ, dma_dev->cap_mask);

	/*
	 * Set mailbox channel device as the base device of
	 * our dma_device because the actual memory accesses
	 * will be done by mailbox controller
	 */
	dma_dev->dev = sba->mbox_dev;

	/* Set base prep routines */
	dma_dev->device_free_chan_resources = sba_free_chan_resources;
	dma_dev->device_terminate_all = sba_device_terminate_all;
	dma_dev->device_issue_pending = sba_issue_pending;
	dma_dev->device_tx_status = sba_tx_status;

	/* Set interrupt routine */
	if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
		dma_dev->device_prep_dma_interrupt = sba_prep_dma_interrupt;

	/* Set memcpy routine */
	if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
		dma_dev->device_prep_dma_memcpy = sba_prep_dma_memcpy;

	/* Set xor routine and capability */
	if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
		dma_dev->device_prep_dma_xor = sba_prep_dma_xor;
		dma_dev->max_xor = sba->max_xor_srcs;
	}

	/* Set pq routine and capability */
	if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
		dma_dev->device_prep_dma_pq = sba_prep_dma_pq;
		dma_set_maxpq(dma_dev, sba->max_pq_srcs, 0);
	}

	/* Initialize DMA device channel list */
	INIT_LIST_HEAD(&dma_dev->channels);
	list_add_tail(&sba->dma_chan.device_node, &dma_dev->channels);

	/* Register with Linux async DMA framework*/
	ret = dma_async_device_register(dma_dev);
	if (ret) {
		dev_err(sba->dev, "async device register error %d", ret);
		return ret;
	}

	dev_info(sba->dev, "%s capabilities: %s%s%s%s\n",
	dma_chan_name(&sba->dma_chan),
	dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "interrupt " : "",
	dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "memcpy " : "",
	dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
	dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "");

	return 0;
}

static int sba_probe(struct platform_device *pdev)
{
1625
	int ret = 0;
1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637
	struct sba_device *sba;
	struct platform_device *mbox_pdev;
	struct of_phandle_args args;

	/* Allocate main SBA struct */
	sba = devm_kzalloc(&pdev->dev, sizeof(*sba), GFP_KERNEL);
	if (!sba)
		return -ENOMEM;

	sba->dev = &pdev->dev;
	platform_set_drvdata(pdev, sba);

1638
	/* Number of mailbox channels should be atleast 1 */
1639 1640 1641 1642 1643
	ret = of_count_phandle_with_args(pdev->dev.of_node,
					 "mboxes", "#mbox-cells");
	if (ret <= 0)
		return -ENODEV;

1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674
	/* Determine SBA version from DT compatible string */
	if (of_device_is_compatible(sba->dev->of_node, "brcm,iproc-sba"))
		sba->ver = SBA_VER_1;
	else if (of_device_is_compatible(sba->dev->of_node,
					 "brcm,iproc-sba-v2"))
		sba->ver = SBA_VER_2;
	else
		return -ENODEV;

	/* Derived Configuration parameters */
	switch (sba->ver) {
	case SBA_VER_1:
		sba->hw_buf_size = 4096;
		sba->hw_resp_size = 8;
		sba->max_pq_coefs = 6;
		sba->max_pq_srcs = 6;
		break;
	case SBA_VER_2:
		sba->hw_buf_size = 4096;
		sba->hw_resp_size = 8;
		sba->max_pq_coefs = 30;
		/*
		 * We can support max_pq_srcs == max_pq_coefs because
		 * we are limited by number of SBA commands that we can
		 * fit in one message for underlying ring manager HW.
		 */
		sba->max_pq_srcs = 12;
		break;
	default:
		return -EINVAL;
	}
1675
	sba->max_req = SBA_MAX_REQ_PER_MBOX_CHANNEL;
1676 1677 1678 1679 1680 1681 1682 1683 1684 1685
	sba->max_cmd_per_req = sba->max_pq_srcs + 3;
	sba->max_xor_srcs = sba->max_cmd_per_req - 1;
	sba->max_resp_pool_size = sba->max_req * sba->hw_resp_size;
	sba->max_cmds_pool_size = sba->max_req *
				  sba->max_cmd_per_req * sizeof(u64);

	/* Setup mailbox client */
	sba->client.dev			= &pdev->dev;
	sba->client.rx_callback		= sba_receive_message;
	sba->client.tx_block		= false;
1686
	sba->client.knows_txdone	= true;
1687 1688
	sba->client.tx_tout		= 0;

1689 1690 1691 1692 1693
	/* Request mailbox channel */
	sba->mchan = mbox_request_channel(&sba->client, 0);
	if (IS_ERR(sba->mchan)) {
		ret = PTR_ERR(sba->mchan);
		goto fail_free_mchan;
1694 1695 1696 1697 1698 1699
	}

	/* Find-out underlying mailbox device */
	ret = of_parse_phandle_with_args(pdev->dev.of_node,
					 "mboxes", "#mbox-cells", 0, &args);
	if (ret)
1700
		goto fail_free_mchan;
1701 1702 1703 1704
	mbox_pdev = of_find_device_by_node(args.np);
	of_node_put(args.np);
	if (!mbox_pdev) {
		ret = -ENODEV;
1705
		goto fail_free_mchan;
1706 1707 1708
	}
	sba->mbox_dev = &mbox_pdev->dev;

1709 1710
	/* Prealloc channel resource */
	ret = sba_prealloc_channel_resources(sba);
1711
	if (ret)
1712
		goto fail_free_mchan;
1713

1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732
	/* Check availability of debugfs */
	if (!debugfs_initialized())
		goto skip_debugfs;

	/* Create debugfs root entry */
	sba->root = debugfs_create_dir(dev_name(sba->dev), NULL);
	if (IS_ERR_OR_NULL(sba->root)) {
		dev_err(sba->dev, "failed to create debugfs root entry\n");
		sba->root = NULL;
		goto skip_debugfs;
	}

	/* Create debugfs stats entry */
	sba->stats = debugfs_create_devm_seqfile(sba->dev, "stats", sba->root,
						 sba_debugfs_stats_show);
	if (IS_ERR_OR_NULL(sba->stats))
		dev_err(sba->dev, "failed to create debugfs stats file\n");
skip_debugfs:

1733 1734
	/* Register DMA device with Linux async framework */
	ret = sba_async_register(sba);
1735
	if (ret)
1736
		goto fail_free_resources;
1737 1738

	/* Print device info */
1739
	dev_info(sba->dev, "%s using SBAv%d mailbox channel from %s",
1740
		 dma_chan_name(&sba->dma_chan), sba->ver+1,
1741
		 dev_name(sba->mbox_dev));
1742 1743 1744

	return 0;

1745
fail_free_resources:
1746
	debugfs_remove_recursive(sba->root);
1747
	sba_freeup_channel_resources(sba);
1748 1749
fail_free_mchan:
	mbox_free_channel(sba->mchan);
1750 1751 1752 1753 1754 1755 1756 1757 1758
	return ret;
}

static int sba_remove(struct platform_device *pdev)
{
	struct sba_device *sba = platform_get_drvdata(pdev);

	dma_async_device_unregister(&sba->dma_dev);

1759 1760
	debugfs_remove_recursive(sba->root);

1761 1762
	sba_freeup_channel_resources(sba);

1763
	mbox_free_channel(sba->mchan);
1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787

	return 0;
}

static const struct of_device_id sba_of_match[] = {
	{ .compatible = "brcm,iproc-sba", },
	{ .compatible = "brcm,iproc-sba-v2", },
	{},
};
MODULE_DEVICE_TABLE(of, sba_of_match);

static struct platform_driver sba_driver = {
	.probe = sba_probe,
	.remove = sba_remove,
	.driver = {
		.name = "bcm-sba-raid",
		.of_match_table = sba_of_match,
	},
};
module_platform_driver(sba_driver);

MODULE_DESCRIPTION("Broadcom SBA RAID driver");
MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>");
MODULE_LICENSE("GPL v2");