gnutls_hash_copy() fails on SHA384 after gnutls_hash_output().
In order to support TLSv1.3 with TPM-based keys (cf. #1235) I implemented RSA-PSS padding in my application: openconnect/openconnect@ff367965
After generating the initial hash M'
from ( 00 | 00 | 00 | 00 | 00 | 00 | 00 | 00 | mHash | Salt )
we then repeatedly generate hashes of ( M' | C )
where C is an incrementing 32-bit big-endian counter.
Instead of repeatedly feeding M'
into the hash function, I chose to use gnutls_hash_copy()
from a context where M'
had already been hashed. Then each time round the loop it's just gnutls_hash_copy()
, gnutls_hash()
to add the four bytes of C
, and gnutls_hash_deinit()
of the copy.
This appears to work for SHA256 and SHA512, but fails for SHA384. It seems to have something to do with the fact that the hash context has already been used (to create M'
). I thought that gnutls_hash_output()
was supposed to reset it and leave it ready for re-use, but apparently not; calling gnutls_hash_deinit()
and gnutls_hash_init()
again does seem to reset it harder and make it work.
Tested with gnutls-3.6.16-1.fc33.x86_64
Test case:
#
#include <gnutls/gnutls.h>
#include <gnutls/crypto.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <arpa/inet.h>
#define SHA512_SIZE 64
static unsigned char mHash[SHA512_SIZE] = {
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f
};
static unsigned char mPrime[SHA512_SIZE];
static unsigned char h1[SHA512_SIZE];
static unsigned char h2[SHA512_SIZE];
static unsigned char h1p[SHA512_SIZE];
static unsigned char h2p[SHA512_SIZE];
static int mgf1(gnutls_hash_hd_t hctx, int count, unsigned char *buf, int diglen)
{
gnutls_hash_hd_t ctx2 = gnutls_hash_copy(hctx);
if (!ctx2)
return GNUTLS_E_PK_SIGN_FAILED;
uint32_t be_count = htonl(count);
int err = gnutls_hash(ctx2, &be_count, sizeof(be_count));
if (err) {
gnutls_hash_deinit(ctx2, NULL);
return err;
}
gnutls_hash_deinit(ctx2, buf);
int i;
for (i = 0; i < diglen; i++) {
if (!(i & 15)) printf("\n(%d) %04x:", count, i);
printf(" %02x", buf[i]);
}
printf("\n");
return 0;
}
int psstest(int dig)
{
gnutls_hash_hd_t hashctx = NULL;
int diglen = gnutls_hash_get_len(dig);
int err = 0;
if ((err = gnutls_hash_init(&hashctx, dig)) ||
(err = gnutls_hash(hashctx, "\0\0\0\0\0\0\0\0", 8)) ||
(err = gnutls_hash(hashctx, mHash, diglen)))
goto out;
gnutls_hash_output(hashctx, mPrime);
/* gnutls_hash_output() is supposed to reset the state. For SHA256 at least
* it *does* seem to work, in
* http://git.infradead.org/users/dwmw2/openconnect.git/commitdiff/ff367965f
*/
if (err = gnutls_hash(hashctx, mPrime, diglen))
goto out;
if (err = mgf1(hashctx, 0, h1, diglen))
goto out;
if (err = mgf1(hashctx, 1, h2, diglen))
goto out;
/* This one really *does* reset it, and makes things work. But *should*
* be identical to the above? */
gnutls_hash_deinit(hashctx, NULL);
if (err = gnutls_hash_init(&hashctx, dig))
goto out;
if (err = gnutls_hash(hashctx, mPrime, diglen))
goto out;
if (err = mgf1(hashctx, 0, h1p, diglen))
goto out;
if (err = mgf1(hashctx, 1, h2p, diglen))
goto out;
if (memcmp(h1, h1p, diglen) || memcmp(h2, h2p, diglen)) {
printf("Error: mismatch\n");
return 1;
}
return 0;
out:
printf("Error: %s\n", gnutls_strerror(err));
return 1;
}
int main(void)
{
//psstest(GNUTLS_DIG_SHA256);
psstest(GNUTLS_DIG_SHA384);
//psstest(GNUTLS_DIG_SHA512);
}