gcm.c 33.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
 * GCM: Galois/Counter Mode.
 *
 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 */

#include <crypto/gf128mul.h>
12
#include <crypto/internal/aead.h>
13
#include <crypto/internal/skcipher.h>
14
#include <crypto/internal/hash.h>
15
#include <crypto/null.h>
16
#include <crypto/scatterwalk.h>
17 18
#include <crypto/hash.h>
#include "internal.h"
19
#include <linux/completion.h>
20 21 22 23 24 25 26
#include <linux/err.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>

struct gcm_instance_ctx {
27
	struct crypto_skcipher_spawn ctr;
28
	struct crypto_ahash_spawn ghash;
29 30 31
};

struct crypto_gcm_ctx {
Herbert Xu's avatar
Herbert Xu committed
32
	struct crypto_skcipher *ctr;
33
	struct crypto_ahash *ghash;
34 35
};

36 37 38 39 40
struct crypto_rfc4106_ctx {
	struct crypto_aead *child;
	u8 nonce[4];
};

41 42 43 44 45 46
struct crypto_rfc4106_req_ctx {
	struct scatterlist src[3];
	struct scatterlist dst[3];
	struct aead_request subreq;
};

47 48 49 50
struct crypto_rfc4543_instance_ctx {
	struct crypto_aead_spawn aead;
};

51 52
struct crypto_rfc4543_ctx {
	struct crypto_aead *child;
Herbert Xu's avatar
Herbert Xu committed
53
	struct crypto_skcipher *null;
54 55 56 57 58 59 60
	u8 nonce[4];
};

struct crypto_rfc4543_req_ctx {
	struct aead_request subreq;
};

61
struct crypto_gcm_ghash_ctx {
62 63
	unsigned int cryptlen;
	struct scatterlist *src;
64
	int (*complete)(struct aead_request *req, u32 flags);
65 66 67
};

struct crypto_gcm_req_priv_ctx {
68
	u8 iv[16];
69
	u8 auth_tag[16];
70
	u8 iauth_tag[16];
71 72 73
	struct scatterlist src[3];
	struct scatterlist dst[3];
	struct scatterlist sg;
74 75 76
	struct crypto_gcm_ghash_ctx ghash_ctx;
	union {
		struct ahash_request ahreq;
Herbert Xu's avatar
Herbert Xu committed
77
		struct skcipher_request skreq;
78
	} u;
79 80
};

81 82 83 84 85
struct crypto_gcm_setkey_result {
	int err;
	struct completion completion;
};

86 87 88 89 90 91
static struct {
	u8 buf[16];
	struct scatterlist sg;
} *gcm_zeroes;

static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc);
92

93 94 95 96 97 98 99 100
static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx(
	struct aead_request *req)
{
	unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));

	return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);
}

101
static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err)
102
{
103
	struct crypto_gcm_setkey_result *result = req->data;
104

105 106 107 108 109
	if (err == -EINPROGRESS)
		return;

	result->err = err;
	complete(&result->completion);
110 111 112 113 114 115
}

static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key,
			     unsigned int keylen)
{
	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
116
	struct crypto_ahash *ghash = ctx->ghash;
Herbert Xu's avatar
Herbert Xu committed
117
	struct crypto_skcipher *ctr = ctx->ctr;
118 119
	struct {
		be128 hash;
120
		u8 iv[16];
121 122 123 124

		struct crypto_gcm_setkey_result result;

		struct scatterlist sg[1];
Herbert Xu's avatar
Herbert Xu committed
125
		struct skcipher_request req;
126 127
	} *data;
	int err;
128

Herbert Xu's avatar
Herbert Xu committed
129 130 131 132 133
	crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
	crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
				       CRYPTO_TFM_REQ_MASK);
	err = crypto_skcipher_setkey(ctr, key, keylen);
	crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctr) &
134
				    CRYPTO_TFM_RES_MASK);
135
	if (err)
136
		return err;
137

Herbert Xu's avatar
Herbert Xu committed
138
	data = kzalloc(sizeof(*data) + crypto_skcipher_reqsize(ctr),
139 140 141 142 143 144
		       GFP_KERNEL);
	if (!data)
		return -ENOMEM;

	init_completion(&data->result.completion);
	sg_init_one(data->sg, &data->hash, sizeof(data->hash));
Herbert Xu's avatar
Herbert Xu committed
145 146 147 148 149 150 151 152 153
	skcipher_request_set_tfm(&data->req, ctr);
	skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
						  CRYPTO_TFM_REQ_MAY_BACKLOG,
				      crypto_gcm_setkey_done,
				      &data->result);
	skcipher_request_set_crypt(&data->req, data->sg, data->sg,
				   sizeof(data->hash), data->iv);

	err = crypto_skcipher_encrypt(&data->req);
154 155 156 157 158 159 160
	if (err == -EINPROGRESS || err == -EBUSY) {
		err = wait_for_completion_interruptible(
			&data->result.completion);
		if (!err)
			err = data->result.err;
	}

161 162 163
	if (err)
		goto out;

164 165 166 167 168 169
	crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK);
	crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) &
			       CRYPTO_TFM_REQ_MASK);
	err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128));
	crypto_aead_set_flags(aead, crypto_ahash_get_flags(ghash) &
			      CRYPTO_TFM_RES_MASK);
170

171
out:
172
	kzfree(data);
173 174 175
	return err;
}

176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194
static int crypto_gcm_setauthsize(struct crypto_aead *tfm,
				  unsigned int authsize)
{
	switch (authsize) {
	case 4:
	case 8:
	case 12:
	case 13:
	case 14:
	case 15:
	case 16:
		break;
	default:
		return -EINVAL;
	}

	return 0;
}

195
static void crypto_gcm_init_common(struct aead_request *req)
196
{
197
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
198
	__be32 counter = cpu_to_be32(1);
199
	struct scatterlist *sg;
200 201

	memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag));
202 203
	memcpy(pctx->iv, req->iv, 12);
	memcpy(pctx->iv + 12, &counter, 4);
204

205
	sg_init_table(pctx->src, 3);
206
	sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag));
207 208
	sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen);
	if (sg != pctx->src + 1)
209
		sg_chain(pctx->src, 2, sg);
210 211

	if (req->src != req->dst) {
212
		sg_init_table(pctx->dst, 3);
213
		sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag));
214 215
		sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen);
		if (sg != pctx->dst + 1)
216
			sg_chain(pctx->dst, 2, sg);
217
	}
218 219 220 221 222 223 224 225
}

static void crypto_gcm_init_crypt(struct aead_request *req,
				  unsigned int cryptlen)
{
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
Herbert Xu's avatar
Herbert Xu committed
226
	struct skcipher_request *skreq = &pctx->u.skreq;
227 228 229
	struct scatterlist *dst;

	dst = req->src == req->dst ? pctx->src : pctx->dst;
230

Herbert Xu's avatar
Herbert Xu committed
231 232
	skcipher_request_set_tfm(skreq, ctx->ctr);
	skcipher_request_set_crypt(skreq, pctx->src, dst,
233
				     cryptlen + sizeof(pctx->auth_tag),
234
				     pctx->iv);
235 236 237 238 239 240 241 242 243
}

static inline unsigned int gcm_remain(unsigned int len)
{
	len &= 0xfU;
	return len ? 16 - len : 0;
}

static void gcm_hash_len_done(struct crypto_async_request *areq, int err);
244

245
static int gcm_hash_update(struct aead_request *req,
246
			   crypto_completion_t compl,
247
			   struct scatterlist *src,
248
			   unsigned int len, u32 flags)
249
{
250
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
251
	struct ahash_request *ahreq = &pctx->u.ahreq;
252

253
	ahash_request_set_callback(ahreq, flags, compl, req);
254 255 256
	ahash_request_set_crypt(ahreq, src, NULL, len);

	return crypto_ahash_update(ahreq);
257 258
}

259 260
static int gcm_hash_remain(struct aead_request *req,
			   unsigned int remain,
261
			   crypto_completion_t compl, u32 flags)
262
{
263
	return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags);
264 265
}

266
static int gcm_hash_len(struct aead_request *req, u32 flags)
267
{
268
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
269 270 271 272 273 274 275
	struct ahash_request *ahreq = &pctx->u.ahreq;
	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
	u128 lengths;

	lengths.a = cpu_to_be64(req->assoclen * 8);
	lengths.b = cpu_to_be64(gctx->cryptlen * 8);
	memcpy(pctx->iauth_tag, &lengths, 16);
276 277 278 279
	sg_init_one(&pctx->sg, pctx->iauth_tag, 16);
	ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req);
	ahash_request_set_crypt(ahreq, &pctx->sg,
				pctx->iauth_tag, sizeof(lengths));
280

281
	return crypto_ahash_finup(ahreq);
282 283
}

284
static int gcm_hash_len_continue(struct aead_request *req, u32 flags)
285
{
286
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
287 288
	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;

289
	return gctx->complete(req, flags);
290 291
}

292
static void gcm_hash_len_done(struct crypto_async_request *areq, int err)
293 294
{
	struct aead_request *req = areq->data;
295

296 297
	if (err)
		goto out;
298

299 300 301
	err = gcm_hash_len_continue(req, 0);
	if (err == -EINPROGRESS)
		return;
302

303 304
out:
	aead_request_complete(req, err);
305 306
}

307
static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags)
308
{
309 310
	return gcm_hash_len(req, flags) ?:
	       gcm_hash_len_continue(req, flags);
311 312
}

313 314
static void gcm_hash_crypt_remain_done(struct crypto_async_request *areq,
				       int err)
315 316
{
	struct aead_request *req = areq->data;
317

318 319 320 321 322 323 324 325 326
	if (err)
		goto out;

	err = gcm_hash_crypt_remain_continue(req, 0);
	if (err == -EINPROGRESS)
		return;

out:
	aead_request_complete(req, err);
327 328
}

329
static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags)
330
{
331 332 333 334
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
	unsigned int remain;

335 336 337 338 339
	remain = gcm_remain(gctx->cryptlen);
	if (remain)
		return gcm_hash_remain(req, remain,
				       gcm_hash_crypt_remain_done, flags) ?:
		       gcm_hash_crypt_remain_continue(req, flags);
340

341
	return gcm_hash_crypt_remain_continue(req, flags);
342 343
}

344
static void gcm_hash_crypt_done(struct crypto_async_request *areq, int err)
345 346
{
	struct aead_request *req = areq->data;
347

348 349 350 351 352 353 354 355 356
	if (err)
		goto out;

	err = gcm_hash_crypt_continue(req, 0);
	if (err == -EINPROGRESS)
		return;

out:
	aead_request_complete(req, err);
357 358
}

359
static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags)
360
{
361 362 363
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;

364 365 366 367 368 369
	if (gctx->cryptlen)
		return gcm_hash_update(req, gcm_hash_crypt_done,
				       gctx->src, gctx->cryptlen, flags) ?:
		       gcm_hash_crypt_continue(req, flags);

	return gcm_hash_crypt_remain_continue(req, flags);
370 371
}

372 373
static void gcm_hash_assoc_remain_done(struct crypto_async_request *areq,
				       int err)
374 375
{
	struct aead_request *req = areq->data;
376

377 378 379 380 381 382 383 384 385
	if (err)
		goto out;

	err = gcm_hash_assoc_remain_continue(req, 0);
	if (err == -EINPROGRESS)
		return;

out:
	aead_request_complete(req, err);
386 387
}

388
static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags)
389
{
390 391
	unsigned int remain;

392 393 394 395 396
	remain = gcm_remain(req->assoclen);
	if (remain)
		return gcm_hash_remain(req, remain,
				       gcm_hash_assoc_remain_done, flags) ?:
		       gcm_hash_assoc_remain_continue(req, flags);
397

398
	return gcm_hash_assoc_remain_continue(req, flags);
399 400
}

401
static void gcm_hash_assoc_done(struct crypto_async_request *areq, int err)
402 403
{
	struct aead_request *req = areq->data;
404

405 406 407 408 409 410 411 412 413
	if (err)
		goto out;

	err = gcm_hash_assoc_continue(req, 0);
	if (err == -EINPROGRESS)
		return;

out:
	aead_request_complete(req, err);
414 415
}

416
static int gcm_hash_init_continue(struct aead_request *req, u32 flags)
417
{
418 419 420 421
	if (req->assoclen)
		return gcm_hash_update(req, gcm_hash_assoc_done,
				       req->src, req->assoclen, flags) ?:
		       gcm_hash_assoc_continue(req, flags);
422

423
	return gcm_hash_assoc_remain_continue(req, flags);
424 425 426 427 428 429
}

static void gcm_hash_init_done(struct crypto_async_request *areq, int err)
{
	struct aead_request *req = areq->data;

430 431 432 433 434 435 436 437 438
	if (err)
		goto out;

	err = gcm_hash_init_continue(req, 0);
	if (err == -EINPROGRESS)
		return;

out:
	aead_request_complete(req, err);
439 440
}

441
static int gcm_hash(struct aead_request *req, u32 flags)
442
{
443
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
444
	struct ahash_request *ahreq = &pctx->u.ahreq;
445
	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
446 447 448

	ahash_request_set_tfm(ahreq, ctx->ghash);

449 450 451
	ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req);
	return crypto_ahash_init(ahreq) ?:
	       gcm_hash_init_continue(req, flags);
452 453
}

454
static int gcm_enc_copy_hash(struct aead_request *req, u32 flags)
455
{
456
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
457 458
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	u8 *auth_tag = pctx->auth_tag;
459

460 461 462
	crypto_xor(auth_tag, pctx->iauth_tag, 16);
	scatterwalk_map_and_copy(auth_tag, req->dst,
				 req->assoclen + req->cryptlen,
463
				 crypto_aead_authsize(aead), 1);
464
	return 0;
465 466
}

467
static int gcm_encrypt_continue(struct aead_request *req, u32 flags)
468
{
469
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
470
	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
471

472 473 474
	gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst);
	gctx->cryptlen = req->cryptlen;
	gctx->complete = gcm_enc_copy_hash;
475

476
	return gcm_hash(req, flags);
477 478
}

479
static void gcm_encrypt_done(struct crypto_async_request *areq, int err)
480 481 482
{
	struct aead_request *req = areq->data;

483 484 485 486 487 488
	if (err)
		goto out;

	err = gcm_encrypt_continue(req, 0);
	if (err == -EINPROGRESS)
		return;
489

490
out:
491
	aead_request_complete(req, err);
492 493
}

494 495
static int crypto_gcm_encrypt(struct aead_request *req)
{
496
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
Herbert Xu's avatar
Herbert Xu committed
497
	struct skcipher_request *skreq = &pctx->u.skreq;
498
	u32 flags = aead_request_flags(req);
499

500 501
	crypto_gcm_init_common(req);
	crypto_gcm_init_crypt(req, req->cryptlen);
Herbert Xu's avatar
Herbert Xu committed
502
	skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req);
503

Herbert Xu's avatar
Herbert Xu committed
504
	return crypto_skcipher_encrypt(skreq) ?:
505
	       gcm_encrypt_continue(req, flags);
506 507
}

508
static int crypto_gcm_verify(struct aead_request *req)
509
{
510
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
511 512 513 514 515 516
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	u8 *auth_tag = pctx->auth_tag;
	u8 *iauth_tag = pctx->iauth_tag;
	unsigned int authsize = crypto_aead_authsize(aead);
	unsigned int cryptlen = req->cryptlen - authsize;

517
	crypto_xor(auth_tag, iauth_tag, 16);
518 519
	scatterwalk_map_and_copy(iauth_tag, req->src,
				 req->assoclen + cryptlen, authsize, 0);
520
	return crypto_memneq(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0;
521 522
}

523
static void gcm_decrypt_done(struct crypto_async_request *areq, int err)
524
{
525 526 527
	struct aead_request *req = areq->data;

	if (!err)
528
		err = crypto_gcm_verify(req);
529 530

	aead_request_complete(req, err);
531 532
}

533
static int gcm_dec_hash_continue(struct aead_request *req, u32 flags)
534 535
{
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
Herbert Xu's avatar
Herbert Xu committed
536
	struct skcipher_request *skreq = &pctx->u.skreq;
537 538
	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;

539
	crypto_gcm_init_crypt(req, gctx->cryptlen);
Herbert Xu's avatar
Herbert Xu committed
540 541
	skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req);
	return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req);
542 543
}

544 545
static int crypto_gcm_decrypt(struct aead_request *req)
{
546
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
547
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
548
	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
549
	unsigned int authsize = crypto_aead_authsize(aead);
550
	unsigned int cryptlen = req->cryptlen;
551
	u32 flags = aead_request_flags(req);
552

553
	cryptlen -= authsize;
554

555
	crypto_gcm_init_common(req);
556

557 558 559
	gctx->src = sg_next(pctx->src);
	gctx->cryptlen = cryptlen;
	gctx->complete = gcm_dec_hash_continue;
560

561
	return gcm_hash(req, flags);
562 563
}

564
static int crypto_gcm_init_tfm(struct crypto_aead *tfm)
565
{
566 567 568
	struct aead_instance *inst = aead_alg_instance(tfm);
	struct gcm_instance_ctx *ictx = aead_instance_ctx(inst);
	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm);
Herbert Xu's avatar
Herbert Xu committed
569
	struct crypto_skcipher *ctr;
570
	struct crypto_ahash *ghash;
571 572 573
	unsigned long align;
	int err;

574 575 576 577
	ghash = crypto_spawn_ahash(&ictx->ghash);
	if (IS_ERR(ghash))
		return PTR_ERR(ghash);

578
	ctr = crypto_spawn_skcipher(&ictx->ctr);
579 580
	err = PTR_ERR(ctr);
	if (IS_ERR(ctr))
581
		goto err_free_hash;
582 583

	ctx->ctr = ctr;
584
	ctx->ghash = ghash;
585

586
	align = crypto_aead_alignmask(tfm);
587
	align &= ~(crypto_tfm_ctx_alignment() - 1);
588
	crypto_aead_set_reqsize(tfm,
589
		align + offsetof(struct crypto_gcm_req_priv_ctx, u) +
Herbert Xu's avatar
Herbert Xu committed
590 591
		max(sizeof(struct skcipher_request) +
		    crypto_skcipher_reqsize(ctr),
592
		    sizeof(struct ahash_request) +
593
		    crypto_ahash_reqsize(ghash)));
594 595

	return 0;
596 597 598 599

err_free_hash:
	crypto_free_ahash(ghash);
	return err;
600 601
}

602
static void crypto_gcm_exit_tfm(struct crypto_aead *tfm)
603
{
604
	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm);
605

606
	crypto_free_ahash(ctx->ghash);
Herbert Xu's avatar
Herbert Xu committed
607
	crypto_free_skcipher(ctx->ctr);
608 609
}

610 611 612 613 614 615 616 617 618
static void crypto_gcm_free(struct aead_instance *inst)
{
	struct gcm_instance_ctx *ctx = aead_instance_ctx(inst);

	crypto_drop_skcipher(&ctx->ctr);
	crypto_drop_ahash(&ctx->ghash);
	kfree(inst);
}

619 620 621 622 623
static int crypto_gcm_create_common(struct crypto_template *tmpl,
				    struct rtattr **tb,
				    const char *full_name,
				    const char *ctr_name,
				    const char *ghash_name)
624
{
625
	struct crypto_attr_type *algt;
626
	struct aead_instance *inst;
Herbert Xu's avatar
Herbert Xu committed
627
	struct skcipher_alg *ctr;
628
	struct crypto_alg *ghash_alg;
629
	struct hash_alg_common *ghash;
630 631 632
	struct gcm_instance_ctx *ctx;
	int err;

633 634
	algt = crypto_get_attr_type(tb);
	if (IS_ERR(algt))
635
		return PTR_ERR(algt);
636

637
	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
638
		return -EINVAL;
639

640 641
	ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type,
				    CRYPTO_ALG_TYPE_HASH,
642 643 644
				    CRYPTO_ALG_TYPE_AHASH_MASK |
				    crypto_requires_sync(algt->type,
							 algt->mask));
645
	if (IS_ERR(ghash_alg))
646 647 648
		return PTR_ERR(ghash_alg);

	ghash = __crypto_hash_alg_common(ghash_alg);
649 650

	err = -ENOMEM;
651 652
	inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
	if (!inst)
653
		goto out_put_ghash;
654

655 656 657
	ctx = aead_instance_ctx(inst);
	err = crypto_init_ahash_spawn(&ctx->ghash, ghash,
				      aead_crypto_instance(inst));
658 659 660
	if (err)
		goto err_free_inst;

661 662 663 664 665
	err = -EINVAL;
	if (ghash->digestsize != 16)
		goto err_drop_ghash;

	crypto_set_skcipher_spawn(&ctx->ctr, aead_crypto_instance(inst));
666 667 668
	err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0,
				   crypto_requires_sync(algt->type,
							algt->mask));
669
	if (err)
670
		goto err_drop_ghash;
671

Herbert Xu's avatar
Herbert Xu committed
672
	ctr = crypto_spawn_skcipher_alg(&ctx->ctr);
673

674
	/* We only support 16-byte blocks. */
675
	err = -EINVAL;
Herbert Xu's avatar
Herbert Xu committed
676
	if (crypto_skcipher_alg_ivsize(ctr) != 16)
677 678 679
		goto out_put_ctr;

	/* Not a stream cipher? */
Herbert Xu's avatar
Herbert Xu committed
680
	if (ctr->base.cra_blocksize != 1)
681 682 683
		goto out_put_ctr;

	err = -ENAMETOOLONG;
684
	if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
Herbert Xu's avatar
Herbert Xu committed
685
		     "gcm_base(%s,%s)", ctr->base.cra_driver_name,
686
		     ghash_alg->cra_driver_name) >=
687
	    CRYPTO_MAX_ALG_NAME)
688
		goto out_put_ctr;
689

690 691
	memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME);

Herbert Xu's avatar
Herbert Xu committed
692 693
	inst->alg.base.cra_flags = (ghash->base.cra_flags |
				    ctr->base.cra_flags) & CRYPTO_ALG_ASYNC;
694
	inst->alg.base.cra_priority = (ghash->base.cra_priority +
Herbert Xu's avatar
Herbert Xu committed
695
				       ctr->base.cra_priority) / 2;
696 697
	inst->alg.base.cra_blocksize = 1;
	inst->alg.base.cra_alignmask = ghash->base.cra_alignmask |
Herbert Xu's avatar
Herbert Xu committed
698
				       ctr->base.cra_alignmask;
699 700
	inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx);
	inst->alg.ivsize = 12;
Herbert Xu's avatar
Herbert Xu committed
701
	inst->alg.chunksize = crypto_skcipher_alg_chunksize(ctr);
702 703 704 705 706 707 708 709
	inst->alg.maxauthsize = 16;
	inst->alg.init = crypto_gcm_init_tfm;
	inst->alg.exit = crypto_gcm_exit_tfm;
	inst->alg.setkey = crypto_gcm_setkey;
	inst->alg.setauthsize = crypto_gcm_setauthsize;
	inst->alg.encrypt = crypto_gcm_encrypt;
	inst->alg.decrypt = crypto_gcm_decrypt;

710 711
	inst->free = crypto_gcm_free;

712 713 714
	err = aead_register_instance(tmpl, inst);
	if (err)
		goto out_put_ctr;
715

716
out_put_ghash:
717
	crypto_mod_put(ghash_alg);
718
	return err;
719 720 721

out_put_ctr:
	crypto_drop_skcipher(&ctx->ctr);
722 723
err_drop_ghash:
	crypto_drop_ahash(&ctx->ghash);
724 725
err_free_inst:
	kfree(inst);
726
	goto out_put_ghash;
727 728
}

729
static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb)
730 731 732 733 734 735 736
{
	const char *cipher_name;
	char ctr_name[CRYPTO_MAX_ALG_NAME];
	char full_name[CRYPTO_MAX_ALG_NAME];

	cipher_name = crypto_attr_alg_name(tb[1]);
	if (IS_ERR(cipher_name))
737
		return PTR_ERR(cipher_name);
738 739 740

	if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >=
	    CRYPTO_MAX_ALG_NAME)
741
		return -ENAMETOOLONG;
742 743 744

	if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm(%s)", cipher_name) >=
	    CRYPTO_MAX_ALG_NAME)
745
		return -ENAMETOOLONG;
746

747 748
	return crypto_gcm_create_common(tmpl, tb, full_name,
					ctr_name, "ghash");
749 750
}

751 752
static struct crypto_template crypto_gcm_tmpl = {
	.name = "gcm",
753
	.create = crypto_gcm_create,
754 755 756
	.module = THIS_MODULE,
};

757 758
static int crypto_gcm_base_create(struct crypto_template *tmpl,
				  struct rtattr **tb)
759 760
{
	const char *ctr_name;
761
	const char *ghash_name;
762 763 764 765
	char full_name[CRYPTO_MAX_ALG_NAME];

	ctr_name = crypto_attr_alg_name(tb[1]);
	if (IS_ERR(ctr_name))
766
		return PTR_ERR(ctr_name);
767

768 769
	ghash_name = crypto_attr_alg_name(tb[2]);
	if (IS_ERR(ghash_name))
770
		return PTR_ERR(ghash_name);
771 772 773

	if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s,%s)",
		     ctr_name, ghash_name) >= CRYPTO_MAX_ALG_NAME)
774
		return -ENAMETOOLONG;
775

776 777
	return crypto_gcm_create_common(tmpl, tb, full_name,
					ctr_name, ghash_name);
778 779 780 781
}

static struct crypto_template crypto_gcm_base_tmpl = {
	.name = "gcm_base",
782
	.create = crypto_gcm_base_create,
783 784 785
	.module = THIS_MODULE,
};

786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827
static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key,
				 unsigned int keylen)
{
	struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent);
	struct crypto_aead *child = ctx->child;
	int err;

	if (keylen < 4)
		return -EINVAL;

	keylen -= 4;
	memcpy(ctx->nonce, key + keylen, 4);

	crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);
	crypto_aead_set_flags(child, crypto_aead_get_flags(parent) &
				     CRYPTO_TFM_REQ_MASK);
	err = crypto_aead_setkey(child, key, keylen);
	crypto_aead_set_flags(parent, crypto_aead_get_flags(child) &
				      CRYPTO_TFM_RES_MASK);

	return err;
}

static int crypto_rfc4106_setauthsize(struct crypto_aead *parent,
				      unsigned int authsize)
{
	struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent);

	switch (authsize) {
	case 8:
	case 12:
	case 16:
		break;
	default:
		return -EINVAL;
	}

	return crypto_aead_setauthsize(ctx->child, authsize);
}

static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req)
{
828
	struct crypto_rfc4106_req_ctx *rctx = aead_request_ctx(req);
829 830
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(aead);
831
	struct aead_request *subreq = &rctx->subreq;
832
	struct crypto_aead *child = ctx->child;
833
	struct scatterlist *sg;
834 835 836
	u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child),
			   crypto_aead_alignmask(child) + 1);

837 838
	scatterwalk_map_and_copy(iv + 12, req->src, 0, req->assoclen - 8, 0);

839 840 841
	memcpy(iv, ctx->nonce, 4);
	memcpy(iv + 4, req->iv, 8);

842 843 844 845 846 847 848 849 850 851 852 853 854 855
	sg_init_table(rctx->src, 3);
	sg_set_buf(rctx->src, iv + 12, req->assoclen - 8);
	sg = scatterwalk_ffwd(rctx->src + 1, req->src, req->assoclen);
	if (sg != rctx->src + 1)
		sg_chain(rctx->src, 2, sg);

	if (req->src != req->dst) {
		sg_init_table(rctx->dst, 3);
		sg_set_buf(rctx->dst, iv + 12, req->assoclen - 8);
		sg = scatterwalk_ffwd(rctx->dst + 1, req->dst, req->assoclen);
		if (sg != rctx->dst + 1)
			sg_chain(rctx->dst, 2, sg);
	}

856 857 858
	aead_request_set_tfm(subreq, child);
	aead_request_set_callback(subreq, req->base.flags, req->base.complete,
				  req->base.data);
859 860 861 862
	aead_request_set_crypt(subreq, rctx->src,
			       req->src == req->dst ? rctx->src : rctx->dst,
			       req->cryptlen, iv);
	aead_request_set_ad(subreq, req->assoclen - 8);
863 864 865 866 867 868

	return subreq;
}

static int crypto_rfc4106_encrypt(struct aead_request *req)
{
869 870 871
	if (req->assoclen != 16 && req->assoclen != 20)
		return -EINVAL;

872 873 874 875 876 877 878
	req = crypto_rfc4106_crypt(req);

	return crypto_aead_encrypt(req);
}

static int crypto_rfc4106_decrypt(struct aead_request *req)
{
879 880 881
	if (req->assoclen != 16 && req->assoclen != 20)
		return -EINVAL;

882 883 884 885 886
	req = crypto_rfc4106_crypt(req);

	return crypto_aead_decrypt(req);
}

887
static int crypto_rfc4106_init_tfm(struct crypto_aead *tfm)
888
{
889 890 891
	struct aead_instance *inst = aead_alg_instance(tfm);
	struct crypto_aead_spawn *spawn = aead_instance_ctx(inst);
	struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm);
892 893 894 895 896 897 898 899 900 901 902
	struct crypto_aead *aead;
	unsigned long align;

	aead = crypto_spawn_aead(spawn);
	if (IS_ERR(aead))
		return PTR_ERR(aead);

	ctx->child = aead;

	align = crypto_aead_alignmask(aead);
	align &= ~(crypto_tfm_ctx_alignment() - 1);
903 904
	crypto_aead_set_reqsize(
		tfm,
905
		sizeof(struct crypto_rfc4106_req_ctx) +
906
		ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) +
907
		align + 24);
908 909 910 911

	return 0;
}

912
static void crypto_rfc4106_exit_tfm(struct crypto_aead *tfm)
913
{
914
	struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm);
915 916 917 918

	crypto_free_aead(ctx->child);
}

919 920 921 922 923 924
static void crypto_rfc4106_free(struct aead_instance *inst)
{
	crypto_drop_aead(aead_instance_ctx(inst));
	kfree(inst);
}

925 926
static int crypto_rfc4106_create(struct crypto_template *tmpl,
				 struct rtattr **tb)
927 928
{
	struct crypto_attr_type *algt;
929
	struct aead_instance *inst;
930
	struct crypto_aead_spawn *spawn;
931
	struct aead_alg *alg;
932 933 934 935 936
	const char *ccm_name;
	int err;

	algt = crypto_get_attr_type(tb);
	if (IS_ERR(algt))
937
		return PTR_ERR(algt);
938

939
	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
940
		return -EINVAL;
941 942 943

	ccm_name = crypto_attr_alg_name(tb[1]);
	if (IS_ERR(ccm_name))
944
		return PTR_ERR(ccm_name);
945 946 947

	inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
	if (!inst)
948
		return -ENOMEM;
949

950 951
	spawn = aead_instance_ctx(inst);
	crypto_set_aead_spawn(spawn, aead_crypto_instance(inst));
952 953 954 955 956
	err = crypto_grab_aead(spawn, ccm_name, 0,
			       crypto_requires_sync(algt->type, algt->mask));
	if (err)
		goto out_free_inst;

957
	alg = crypto_spawn_aead_alg(spawn);
958 959 960

	err = -EINVAL;

961 962
	/* Underlying IV size must be 12. */
	if (crypto_aead_alg_ivsize(alg) != 12)
963 964 965
		goto out_drop_alg;

	/* Not a stream cipher? */
966
	if (alg->base.cra_blocksize != 1)
967 968 969
		goto out_drop_alg;

	err = -ENAMETOOLONG;
970 971 972 973 974
	if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
		     "rfc4106(%s)", alg->base.cra_name) >=
	    CRYPTO_MAX_ALG_NAME ||
	    snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
		     "rfc4106(%s)", alg->base.cra_driver_name) >=
975 976 977
	    CRYPTO_MAX_ALG_NAME)
		goto out_drop_alg;

978
	inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC;
979 980 981
	inst->alg.base.cra_priority = alg->base.cra_priority;
	inst->alg.base.cra_blocksize = 1;
	inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
982

983
	inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx);
984

985
	inst->alg.ivsize = 8;
Herbert Xu's avatar
Herbert Xu committed
986
	inst->alg.chunksize = crypto_aead_alg_chunksize(alg);
987
	inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
988

989 990
	inst->alg.init = crypto_rfc4106_init_tfm;
	inst->alg.exit = crypto_rfc4106_exit_tfm;
991

992 993 994 995
	inst->alg.setkey = crypto_rfc4106_setkey;
	inst->alg.setauthsize = crypto_rfc4106_setauthsize;
	inst->alg.encrypt = crypto_rfc4106_encrypt;
	inst->alg.decrypt = crypto_rfc4106_decrypt;
996

997 998
	inst->free = crypto_rfc4106_free;

999 1000 1001
	err = aead_register_instance(tmpl, inst);
	if (err)
		goto out_drop_alg;
1002 1003

out:
1004
	return err;
1005 1006 1007 1008 1009 1010 1011 1012 1013 1014

out_drop_alg:
	crypto_drop_aead(spawn);
out_free_inst:
	kfree(inst);
	goto out;
}

static struct crypto_template crypto_rfc4106_tmpl = {
	.name = "rfc4106",
1015
	.create = crypto_rfc4106_create,
1016 1017 1018
	.module = THIS_MODULE,
};

1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052
static int crypto_rfc4543_setkey(struct crypto_aead *parent, const u8 *key,
				 unsigned int keylen)
{
	struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent);
	struct crypto_aead *child = ctx->child;
	int err;

	if (keylen < 4)
		return -EINVAL;

	keylen -= 4;
	memcpy(ctx->nonce, key + keylen, 4);

	crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);
	crypto_aead_set_flags(child, crypto_aead_get_flags(parent) &
				     CRYPTO_TFM_REQ_MASK);
	err = crypto_aead_setkey(child, key, keylen);
	crypto_aead_set_flags(parent, crypto_aead_get_flags(child) &
				      CRYPTO_TFM_RES_MASK);

	return err;
}

static int crypto_rfc4543_setauthsize(struct crypto_aead *parent,
				      unsigned int authsize)
{
	struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent);

	if (authsize != 16)
		return -EINVAL;

	return crypto_aead_setauthsize(ctx->child, authsize);
}

1053
static int crypto_rfc4543_crypt(struct aead_request *req, bool enc)
1054 1055 1056
{
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead);
1057
	struct crypto_rfc4543_req_ctx *rctx = aead_request_ctx(req);
1058 1059 1060 1061
	struct aead_request *subreq = &rctx->subreq;
	unsigned int authsize = crypto_aead_authsize(aead);
	u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child),
			   crypto_aead_alignmask(ctx->child) + 1);
1062 1063 1064 1065 1066 1067 1068
	int err;

	if (req->src != req->dst) {
		err = crypto_rfc4543_copy_src_to_dst(req, enc);
		if (err)
			return err;
	}
1069 1070 1071 1072 1073

	memcpy(iv, ctx->nonce, 4);
	memcpy(iv + 4, req->iv, 8);

	aead_request_set_tfm(subreq, ctx->child);
1074 1075 1076 1077 1078 1079 1080 1081
	aead_request_set_callback(subreq, req->base.flags,
				  req->base.complete, req->base.data);
	aead_request_set_crypt(subreq, req->src, req->dst,
			       enc ? 0 : authsize, iv);
	aead_request_set_ad(subreq, req->assoclen + req->cryptlen -
				    subreq->cryptlen);

	return enc ? crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq);
1082 1083
}

1084 1085 1086 1087 1088
static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc)
{
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead);
	unsigned int authsize = crypto_aead_authsize(aead);
1089 1090
	unsigned int nbytes = req->assoclen + req->cryptlen -
			      (enc ? 0 : authsize);
Herbert Xu's avatar
Herbert Xu committed
1091 1092 1093 1094 1095
	SKCIPHER_REQUEST_ON_STACK(nreq, ctx->null);

	skcipher_request_set_tfm(nreq, ctx->null);
	skcipher_request_set_callback(nreq, req->base.flags, NULL, NULL);
	skcipher_request_set_crypt(nreq, req->src, req->dst, nbytes, NULL);
1096

Herbert Xu's avatar
Herbert Xu committed
1097
	return crypto_skcipher_encrypt(nreq);
1098 1099
}

1100 1101
static int crypto_rfc4543_encrypt(struct aead_request *req)
{
1102
	return crypto_rfc4543_crypt(req, true);
1103 1104 1105 1106
}

static int crypto_rfc4543_decrypt(struct aead_request *req)
{
1107
	return crypto_rfc4543_crypt(req, false);
1108 1109
}

1110
static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm)
1111
{
1112 1113
	struct aead_instance *inst = aead_alg_instance(tfm);
	struct crypto_rfc4543_instance_ctx *ictx = aead_instance_ctx(inst);
1114
	struct crypto_aead_spawn *spawn = &ictx->aead;
1115
	struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);
1116
	struct crypto_aead *aead;
Herbert Xu's avatar
Herbert Xu committed
1117
	struct crypto_skcipher *null;
1118
	unsigned long align;
1119
	int err = 0;
1120 1121 1122 1123 1124

	aead = crypto_spawn_aead(spawn);
	if (IS_ERR(aead))
		return PTR_ERR(aead);

Herbert Xu's avatar
Herbert Xu committed
1125
	null = crypto_get_default_null_skcipher2();
1126 1127 1128 1129
	err = PTR_ERR(null);
	if (IS_ERR(null))
		goto err_free_aead;

1130
	ctx->child = aead;
1131
	ctx->null = null;
1132 1133 1134

	align = crypto_aead_alignmask(aead);
	align &= ~(crypto_tfm_ctx_alignment() - 1);
1135 1136
	crypto_aead_set_reqsize(
		tfm,
1137 1138
		sizeof(struct crypto_rfc4543_req_ctx) +
		ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) +
1139
		align + 12);
1140 1141

	return 0;
1142 1143 1144 1145

err_free_aead:
	crypto_free_aead(aead);
	return err;
1146 1147
}

1148
static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm)
1149
{
1150
	struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);
1151 1152

	crypto_free_aead(ctx->child);
Herbert Xu's avatar
Herbert Xu committed
1153
	crypto_put_default_null_skcipher2();
1154 1155
}

1156 1157 1158 1159 1160 1161 1162 1163 1164
static void crypto_rfc4543_free(struct aead_instance *inst)
{
	struct crypto_rfc4543_instance_ctx *ctx = aead_instance_ctx(inst);

	crypto_drop_aead(&ctx->aead);

	kfree(inst);
}

1165 1166
static int crypto_rfc4543_create(struct crypto_template *tmpl,
				struct rtattr **tb)
1167 1168
{
	struct crypto_attr_type *algt;
1169
	struct aead_instance *inst;
1170
	struct crypto_aead_spawn *spawn;
1171
	struct aead_alg *alg;
1172
	struct crypto_rfc4543_instance_ctx *ctx;
1173 1174 1175 1176 1177
	const char *ccm_name;
	int err;

	algt = crypto_get_attr_type(tb);
	if (IS_ERR(algt))
1178
		return PTR_ERR(algt);
1179

1180
	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
1181
		return -EINVAL;
1182 1183 1184

	ccm_name = crypto_attr_alg_name(tb[1]);
	if (IS_ERR(ccm_name))
1185
		return PTR_ERR(ccm_name);
1186

1187
	inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
1188
	if (!inst)
1189
		return -ENOMEM;
1190

1191
	ctx = aead_instance_ctx(inst);
1192
	spawn = &ctx->aead;
1193
	crypto_set_aead_spawn(spawn, aead_crypto_instance(inst));
1194 1195 1196 1197 1198
	err = crypto_grab_aead(spawn, ccm_name, 0,
			       crypto_requires_sync(algt->type, algt->mask));
	if (err)
		goto out_free_inst;

1199
	alg = crypto_spawn_aead_alg(spawn);