Source code
Revision control
Copy as Markdown
Other Tools
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
/*
* RSA PKCS#1 v2.1 (RFC 3447) operations
*/
#ifdef FREEBL_NO_DEPEND
#include "stubs.h"
#endif
#include "secerr.h"
#include "blapi.h"
#include "secitem.h"
#include "blapii.h"
#define RSA_BLOCK_MIN_PAD_LEN 8
#define RSA_BLOCK_FIRST_OCTET 0x00
#define RSA_BLOCK_PRIVATE_PAD_OCTET 0xff
#define RSA_BLOCK_AFTER_PAD_OCTET 0x00
/*
* RSA block types
*
* The values of RSA_BlockPrivate and RSA_BlockPublic are fixed.
* The value of RSA_BlockRaw isn't fixed by definition, but we are keeping
* the value that NSS has been using in the past.
*/
typedef enum {
RSA_BlockPrivate = 1, /* pad for a private-key operation */
RSA_BlockPublic = 2, /* pad for a public-key operation */
RSA_BlockRaw = 4 /* simply justify the block appropriately */
} RSA_BlockType;
/* Needed for RSA-PSS functions */
static const unsigned char eightZeros[] = { 0, 0, 0, 0, 0, 0, 0, 0 };
/* Constant time comparison of a single byte.
* Returns 1 iff a == b, otherwise returns 0.
* Note: For ranges of bytes, use constantTimeCompare.
*/
static unsigned char
constantTimeEQ8(unsigned char a, unsigned char b)
{
unsigned char c = ~((a - b) | (b - a));
c >>= 7;
return c;
}
/* Constant time comparison of a range of bytes.
* Returns 1 iff len bytes of a are identical to len bytes of b, otherwise
* returns 0.
*/
static unsigned char
constantTimeCompare(const unsigned char *a,
const unsigned char *b,
unsigned int len)
{
unsigned char tmp = 0;
unsigned int i;
for (i = 0; i < len; ++i, ++a, ++b)
tmp |= *a ^ *b;
return constantTimeEQ8(0x00, tmp);
}
/* Constant time conditional.
* Returns a if c is 1, or b if c is 0. The result is undefined if c is
* not 0 or 1.
*/
static unsigned int
constantTimeCondition(unsigned int c,
unsigned int a,
unsigned int b)
{
return (~(c - 1) & a) | ((c - 1) & b);
}
static unsigned int
rsa_modulusLen(SECItem *modulus)
{
if (modulus->len == 0) {
return 0;
}
unsigned char byteZero = modulus->data[0];
unsigned int modLen = modulus->len - !byteZero;
return modLen;
}
static unsigned int
rsa_modulusBits(SECItem *modulus)
{
if (modulus->len == 0) {
return 0;
}
unsigned char byteZero = modulus->data[0];
unsigned int numBits = (modulus->len - 1) * 8;
if (byteZero == 0 && modulus->len == 1) {
return 0;
}
if (byteZero == 0) {
numBits -= 8;
byteZero = modulus->data[1];
}
while (byteZero > 0) {
numBits++;
byteZero >>= 1;
}
return numBits;
}
/*
* Format one block of data for public/private key encryption using
* the rules defined in PKCS #1.
*/
static unsigned char *
rsa_FormatOneBlock(unsigned modulusLen,
RSA_BlockType blockType,
SECItem *data)
{
unsigned char *block;
unsigned char *bp;
unsigned int padLen;
unsigned int i, j;
SECStatus rv;
block = (unsigned char *)PORT_Alloc(modulusLen);
if (block == NULL)
return NULL;
bp = block;
/*
* All RSA blocks start with two octets:
* 0x00 || BlockType
*/
*bp++ = RSA_BLOCK_FIRST_OCTET;
*bp++ = (unsigned char)blockType;
switch (blockType) {
/*
* Blocks intended for private-key operation.
*/
case RSA_BlockPrivate: /* preferred method */
/*
* 0x00 || BT || Pad || 0x00 || ActualData
* 1 1 padLen 1 data->len
* padLen must be at least RSA_BLOCK_MIN_PAD_LEN (8) bytes.
* Pad is either all 0x00 or all 0xff bytes, depending on blockType.
*/
padLen = modulusLen - data->len - 3;
PORT_Assert(padLen >= RSA_BLOCK_MIN_PAD_LEN);
if (padLen < RSA_BLOCK_MIN_PAD_LEN) {
PORT_ZFree(block, modulusLen);
return NULL;
}
PORT_Memset(bp, RSA_BLOCK_PRIVATE_PAD_OCTET, padLen);
bp += padLen;
*bp++ = RSA_BLOCK_AFTER_PAD_OCTET;
PORT_Memcpy(bp, data->data, data->len);
break;
/*
* Blocks intended for public-key operation.
*/
case RSA_BlockPublic:
/*
* 0x00 || BT || Pad || 0x00 || ActualData
* 1 1 padLen 1 data->len
* Pad is 8 or more non-zero random bytes.
*
* Build the block left to right.
* Fill the entire block from Pad to the end with random bytes.
* Use the bytes after Pad as a supply of extra random bytes from
* which to find replacements for the zero bytes in Pad.
* If we need more than that, refill the bytes after Pad with
* new random bytes as necessary.
*/
padLen = modulusLen - (data->len + 3);
PORT_Assert(padLen >= RSA_BLOCK_MIN_PAD_LEN);
if (padLen < RSA_BLOCK_MIN_PAD_LEN) {
PORT_ZFree(block, modulusLen);
return NULL;
}
j = modulusLen - 2;
rv = RNG_GenerateGlobalRandomBytes(bp, j);
if (rv == SECSuccess) {
for (i = 0; i < padLen;) {
unsigned char repl;
/* Pad with non-zero random data. */
if (bp[i] != RSA_BLOCK_AFTER_PAD_OCTET) {
++i;
continue;
}
if (j <= padLen) {
rv = RNG_GenerateGlobalRandomBytes(bp + padLen,
modulusLen - (2 + padLen));
if (rv != SECSuccess)
break;
j = modulusLen - 2;
}
do {
repl = bp[--j];
} while (repl == RSA_BLOCK_AFTER_PAD_OCTET && j > padLen);
if (repl != RSA_BLOCK_AFTER_PAD_OCTET) {
bp[i++] = repl;
}
}
}
if (rv != SECSuccess) {
PORT_ZFree(block, modulusLen);
PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
return NULL;
}
bp += padLen;
*bp++ = RSA_BLOCK_AFTER_PAD_OCTET;
PORT_Memcpy(bp, data->data, data->len);
break;
default:
PORT_Assert(0);
PORT_ZFree(block, modulusLen);
return NULL;
}
return block;
}
/* modulusLen has to be larger than RSA_BLOCK_MIN_PAD_LEN + 3, and data has to be smaller than modulus - (RSA_BLOCK_MIN_PAD_LEN + 3) */
static SECStatus
rsa_FormatBlock(SECItem *result,
unsigned modulusLen,
RSA_BlockType blockType,
SECItem *data)
{
switch (blockType) {
case RSA_BlockPrivate:
case RSA_BlockPublic:
/*
* 0x00 || BT || Pad || 0x00 || ActualData
*
* The "3" below is the first octet + the second octet + the 0x00
* octet that always comes just before the ActualData.
*/
if (modulusLen < (3 + RSA_BLOCK_MIN_PAD_LEN) || data->len > (modulusLen - (3 + RSA_BLOCK_MIN_PAD_LEN))) {
return SECFailure;
}
result->data = rsa_FormatOneBlock(modulusLen, blockType, data);
if (result->data == NULL) {
result->len = 0;
return SECFailure;
}
result->len = modulusLen;
break;
case RSA_BlockRaw:
/*
* Pad || ActualData
* Pad is zeros. The application is responsible for recovering
* the actual data.
*/
if (data->len > modulusLen) {
return SECFailure;
}
result->data = (unsigned char *)PORT_ZAlloc(modulusLen);
result->len = modulusLen;
PORT_Memcpy(result->data + (modulusLen - data->len),
data->data, data->len);
break;
default:
PORT_Assert(0);
result->data = NULL;
result->len = 0;
return SECFailure;
}
return SECSuccess;
}
/*
* Mask generation function MGF1 as defined in PKCS #1 v2.1 / RFC 3447.
*/
static SECStatus
MGF1(HASH_HashType hashAlg,
unsigned char *mask,
unsigned int maskLen,
const unsigned char *mgfSeed,
unsigned int mgfSeedLen)
{
unsigned int digestLen;
PRUint32 counter;
PRUint32 rounds;
unsigned char *tempHash;
unsigned char *temp;
const SECHashObject *hash;
void *hashContext;
unsigned char C[4];
SECStatus rv = SECSuccess;
hash = HASH_GetRawHashObject(hashAlg);
if (hash == NULL) {
return SECFailure;
}
hashContext = (*hash->create)();
rounds = (maskLen + hash->length - 1) / hash->length;
for (counter = 0; counter < rounds; counter++) {
C[0] = (unsigned char)((counter >> 24) & 0xff);
C[1] = (unsigned char)((counter >> 16) & 0xff);
C[2] = (unsigned char)((counter >> 8) & 0xff);
C[3] = (unsigned char)(counter & 0xff);
/* This could be optimized when the clone functions in
* rawhash.c are implemented. */
(*hash->begin)(hashContext);
(*hash->update)(hashContext, mgfSeed, mgfSeedLen);
(*hash->update)(hashContext, C, sizeof C);
tempHash = mask + counter * hash->length;
if (counter != (rounds - 1)) {
(*hash->end)(hashContext, tempHash, &digestLen, hash->length);
} else { /* we're in the last round and need to cut the hash */
temp = (unsigned char *)PORT_Alloc(hash->length);
if (!temp) {
rv = SECFailure;
goto done;
}
(*hash->end)(hashContext, temp, &digestLen, hash->length);
PORT_Memcpy(tempHash, temp, maskLen - counter * hash->length);
PORT_Free(temp);
}
}
done:
(*hash->destroy)(hashContext, PR_TRUE);
return rv;
}
/* XXX Doesn't set error code */
SECStatus
RSA_SignRaw(RSAPrivateKey *key,
unsigned char *output,
unsigned int *outputLen,
unsigned int maxOutputLen,
const unsigned char *data,
unsigned int dataLen)
{
SECStatus rv = SECSuccess;
unsigned int modulusLen = rsa_modulusLen(&key->modulus);
SECItem formatted;
SECItem unformatted;
if (maxOutputLen < modulusLen)
return SECFailure;
unformatted.len = dataLen;
unformatted.data = (unsigned char *)data;
formatted.data = NULL;
rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockRaw, &unformatted);
if (rv != SECSuccess)
goto done;
rv = RSA_PrivateKeyOpDoubleChecked(key, output, formatted.data);
*outputLen = modulusLen;
done:
if (formatted.data != NULL)
PORT_ZFree(formatted.data, modulusLen);
return rv;
}
/* XXX Doesn't set error code */
SECStatus
RSA_CheckSignRaw(RSAPublicKey *key,
const unsigned char *sig,
unsigned int sigLen,
const unsigned char *hash,
unsigned int hashLen)
{
SECStatus rv;
unsigned int modulusLen = rsa_modulusLen(&key->modulus);
unsigned char *buffer;
if (sigLen != modulusLen)
goto failure;
if (hashLen > modulusLen)
goto failure;
buffer = (unsigned char *)PORT_Alloc(modulusLen + 1);
if (!buffer)
goto failure;
rv = RSA_PublicKeyOp(key, buffer, sig);
if (rv != SECSuccess)
goto loser;
/*
* make sure we get the same results
*/
/* XXX(rsleevi): Constant time */
/* NOTE: should we verify the leading zeros? */
if (PORT_Memcmp(buffer + (modulusLen - hashLen), hash, hashLen) != 0)
goto loser;
PORT_Free(buffer);
return SECSuccess;
loser:
PORT_Free(buffer);
failure:
return SECFailure;
}
/* XXX Doesn't set error code */
SECStatus
RSA_CheckSignRecoverRaw(RSAPublicKey *key,
unsigned char *data,
unsigned int *dataLen,
unsigned int maxDataLen,
const unsigned char *sig,
unsigned int sigLen)
{
SECStatus rv;
unsigned int modulusLen = rsa_modulusLen(&key->modulus);
if (sigLen != modulusLen)
goto failure;
if (maxDataLen < modulusLen)
goto failure;
rv = RSA_PublicKeyOp(key, data, sig);
if (rv != SECSuccess)
goto failure;
*dataLen = modulusLen;
return SECSuccess;
failure:
return SECFailure;
}
/* XXX Doesn't set error code */
SECStatus
RSA_EncryptRaw(RSAPublicKey *key,
unsigned char *output,
unsigned int *outputLen,
unsigned int maxOutputLen,
const unsigned char *input,
unsigned int inputLen)
{
SECStatus rv;
unsigned int modulusLen = rsa_modulusLen(&key->modulus);
SECItem formatted;
SECItem unformatted;
formatted.data = NULL;
if (maxOutputLen < modulusLen)
goto failure;
unformatted.len = inputLen;
unformatted.data = (unsigned char *)input;
formatted.data = NULL;
rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockRaw, &unformatted);
if (rv != SECSuccess)
goto failure;
rv = RSA_PublicKeyOp(key, output, formatted.data);
if (rv != SECSuccess)
goto failure;
PORT_ZFree(formatted.data, modulusLen);
*outputLen = modulusLen;
return SECSuccess;
failure:
if (formatted.data != NULL)
PORT_ZFree(formatted.data, modulusLen);
return SECFailure;
}
/* XXX Doesn't set error code */
SECStatus
RSA_DecryptRaw(RSAPrivateKey *key,
unsigned char *output,
unsigned int *outputLen,
unsigned int maxOutputLen,
const unsigned char *input,
unsigned int inputLen)
{
SECStatus rv;
unsigned int modulusLen = rsa_modulusLen(&key->modulus);
if (modulusLen > maxOutputLen)
goto failure;
if (inputLen != modulusLen)
goto failure;
rv = RSA_PrivateKeyOp(key, output, input);
if (rv != SECSuccess)
goto failure;
*outputLen = modulusLen;
return SECSuccess;
failure:
return SECFailure;
}
/*
* Decodes an EME-OAEP encoded block, validating the encoding in constant
* time.
* Described in RFC 3447, section 7.1.2.
* input contains the encoded block, after decryption.
* label is the optional value L that was associated with the message.
* On success, the original message and message length will be stored in
* output and outputLen.
*/
static SECStatus
eme_oaep_decode(unsigned char *output,
unsigned int *outputLen,
unsigned int maxOutputLen,
const unsigned char *input,
unsigned int inputLen,
HASH_HashType hashAlg,
HASH_HashType maskHashAlg,
const unsigned char *label,
unsigned int labelLen)
{
const SECHashObject *hash;
void *hashContext;
SECStatus rv = SECFailure;
unsigned char labelHash[HASH_LENGTH_MAX];
unsigned int i;
unsigned int maskLen;
unsigned int paddingOffset;
unsigned char *mask = NULL;
unsigned char *tmpOutput = NULL;
unsigned char isGood;
unsigned char foundPaddingEnd;
hash = HASH_GetRawHashObject(hashAlg);
/* 1.c */
if (inputLen < (hash->length * 2) + 2) {
PORT_SetError(SEC_ERROR_INPUT_LEN);
return SECFailure;
}
/* Step 3.a - Generate lHash */
hashContext = (*hash->create)();
if (hashContext == NULL) {
PORT_SetError(SEC_ERROR_NO_MEMORY);
return SECFailure;
}
(*hash->begin)(hashContext);
if (labelLen > 0)
(*hash->update)(hashContext, label, labelLen);
(*hash->end)(hashContext, labelHash, &i, sizeof(labelHash));
(*hash->destroy)(hashContext, PR_TRUE);
tmpOutput = (unsigned char *)PORT_Alloc(inputLen);
if (tmpOutput == NULL) {
PORT_SetError(SEC_ERROR_NO_MEMORY);
goto done;
}
maskLen = inputLen - hash->length - 1;
mask = (unsigned char *)PORT_Alloc(maskLen);
if (mask == NULL) {
PORT_SetError(SEC_ERROR_NO_MEMORY);
goto done;
}
PORT_Memcpy(tmpOutput, input, inputLen);
/* 3.c - Generate seedMask */
MGF1(maskHashAlg, mask, hash->length, &tmpOutput[1 + hash->length],
inputLen - hash->length - 1);
/* 3.d - Unmask seed */
for (i = 0; i < hash->length; ++i)
tmpOutput[1 + i] ^= mask[i];
/* 3.e - Generate dbMask */
MGF1(maskHashAlg, mask, maskLen, &tmpOutput[1], hash->length);
/* 3.f - Unmask DB */
for (i = 0; i < maskLen; ++i)
tmpOutput[1 + hash->length + i] ^= mask[i];
/* 3.g - Compare Y, lHash, and PS in constant time
* Warning: This code is timing dependent and must not disclose which of
* these were invalid.
*/
paddingOffset = 0;
isGood = 1;
foundPaddingEnd = 0;
/* Compare Y */
isGood &= constantTimeEQ8(0x00, tmpOutput[0]);
/* Compare lHash and lHash' */
isGood &= constantTimeCompare(&labelHash[0],
&tmpOutput[1 + hash->length],
hash->length);
/* Compare that the padding is zero or more zero octets, followed by a
* 0x01 octet */
for (i = 1 + (hash->length * 2); i < inputLen; ++i) {
unsigned char isZero = constantTimeEQ8(0x00, tmpOutput[i]);
unsigned char isOne = constantTimeEQ8(0x01, tmpOutput[i]);
/* non-constant time equivalent:
* if (tmpOutput[i] == 0x01 && !foundPaddingEnd)
* paddingOffset = i;
*/
paddingOffset = constantTimeCondition(isOne & ~foundPaddingEnd, i,
paddingOffset);
/* non-constant time equivalent:
* if (tmpOutput[i] == 0x01)
* foundPaddingEnd = true;
*
* Note: This may yield false positives, as it will be set whenever
* a 0x01 byte is encountered. If there was bad padding (eg:
* 0x03 0x02 0x01), foundPaddingEnd will still be set to true, and
* paddingOffset will still be set to 2.
*/
foundPaddingEnd = constantTimeCondition(isOne, 1, foundPaddingEnd);
/* non-constant time equivalent:
* if (tmpOutput[i] != 0x00 && tmpOutput[i] != 0x01 &&
* !foundPaddingEnd) {
* isGood = false;
* }
*
* Note: This may yield false positives, as a message (and padding)
* that is entirely zeros will result in isGood still being true. Thus
* it's necessary to check foundPaddingEnd is positive below.
*/
isGood = constantTimeCondition(~foundPaddingEnd & ~isZero, 0, isGood);
}
/* While both isGood and foundPaddingEnd may have false positives, they
* cannot BOTH have false positives. If both are not true, then an invalid
* message was received. Note, this comparison must still be done in constant
* time so as not to leak either condition.
*/
if (!(isGood & foundPaddingEnd)) {
PORT_SetError(SEC_ERROR_BAD_DATA);
goto done;
}
/* End timing dependent code */
++paddingOffset; /* Skip the 0x01 following the end of PS */
*outputLen = inputLen - paddingOffset;
if (*outputLen > maxOutputLen) {
PORT_SetError(SEC_ERROR_OUTPUT_LEN);
goto done;
}
if (*outputLen)
PORT_Memcpy(output, &tmpOutput[paddingOffset], *outputLen);
rv = SECSuccess;
done:
if (mask)
PORT_ZFree(mask, maskLen);
if (tmpOutput)
PORT_ZFree(tmpOutput, inputLen);
return rv;
}
/*
* Generate an EME-OAEP encoded block for encryption
* Described in RFC 3447, section 7.1.1
* We use input instead of M for the message to be encrypted
* label is the optional value L to be associated with the message.
*/
static SECStatus
eme_oaep_encode(unsigned char *em,
unsigned int emLen,
const unsigned char *input,
unsigned int inputLen,
HASH_HashType hashAlg,
HASH_HashType maskHashAlg,
const unsigned char *label,
unsigned int labelLen,
const unsigned char *seed,
unsigned int seedLen)
{
const SECHashObject *hash;
void *hashContext;
SECStatus rv;
unsigned char *mask;
unsigned int reservedLen;
unsigned int dbMaskLen;
unsigned int i;
hash = HASH_GetRawHashObject(hashAlg);
PORT_Assert(seed == NULL || seedLen == hash->length);
/* Step 1.b */
reservedLen = (2 * hash->length) + 2;
if (emLen < reservedLen || inputLen > (emLen - reservedLen)) {
PORT_SetError(SEC_ERROR_INPUT_LEN);
return SECFailure;
}
/*
* From RFC 3447, Section 7.1
* +----------+---------+-------+
* DB = | lHash | PS | M |
* +----------+---------+-------+
* |
* +----------+ V
* | seed |--> MGF ---> xor
* +----------+ |
* | |
* +--+ V |
* |00| xor <----- MGF <-----|
* +--+ | |
* | | |
* V V V
* +--+----------+----------------------------+
* EM = |00|maskedSeed| maskedDB |
* +--+----------+----------------------------+
*
* We use mask to hold the result of the MGF functions, and all other
* values are generated in their final resting place.
*/
*em = 0x00;
/* Step 2.a - Generate lHash */
hashContext = (*hash->create)();
if (hashContext == NULL) {
PORT_SetError(SEC_ERROR_NO_MEMORY);
return SECFailure;
}
(*hash->begin)(hashContext);
if (labelLen > 0)
(*hash->update)(hashContext, label, labelLen);
(*hash->end)(hashContext, &em[1 + hash->length], &i, hash->length);
(*hash->destroy)(hashContext, PR_TRUE);
/* Step 2.b - Generate PS */
if (emLen - reservedLen - inputLen > 0) {
PORT_Memset(em + 1 + (hash->length * 2), 0x00,
emLen - reservedLen - inputLen);
}
/* Step 2.c. - Generate DB
* DB = lHash || PS || 0x01 || M
* Note that PS and lHash have already been placed into em at their
* appropriate offsets. This just copies M into place
*/
em[emLen - inputLen - 1] = 0x01;
if (inputLen)
PORT_Memcpy(em + emLen - inputLen, input, inputLen);
if (seed == NULL) {
/* Step 2.d - Generate seed */
rv = RNG_GenerateGlobalRandomBytes(em + 1, hash->length);
if (rv != SECSuccess) {
return rv;
}
} else {
/* For Known Answer Tests, copy the supplied seed. */
PORT_Memcpy(em + 1, seed, seedLen);
}
/* Step 2.e - Generate dbMask*/
dbMaskLen = emLen - hash->length - 1;
mask = (unsigned char *)PORT_Alloc(dbMaskLen);
if (mask == NULL) {
PORT_SetError(SEC_ERROR_NO_MEMORY);
return SECFailure;
}
MGF1(maskHashAlg, mask, dbMaskLen, em + 1, hash->length);
/* Step 2.f - Compute maskedDB*/
for (i = 0; i < dbMaskLen; ++i)
em[1 + hash->length + i] ^= mask[i];
/* Step 2.g - Generate seedMask */
MGF1(maskHashAlg, mask, hash->length, &em[1 + hash->length], dbMaskLen);
/* Step 2.h - Compute maskedSeed */
for (i = 0; i < hash->length; ++i)
em[1 + i] ^= mask[i];
PORT_ZFree(mask, dbMaskLen);
return SECSuccess;
}
SECStatus
RSA_EncryptOAEP(RSAPublicKey *key,
HASH_HashType hashAlg,
HASH_HashType maskHashAlg,
const unsigned char *label,
unsigned int labelLen,
const unsigned char *seed,
unsigned int seedLen,
unsigned char *output,
unsigned int *outputLen,
unsigned int maxOutputLen,
const unsigned char *input,
unsigned int inputLen)
{
SECStatus rv = SECFailure;
unsigned int modulusLen = rsa_modulusLen(&key->modulus);
unsigned char *oaepEncoded = NULL;
if (maxOutputLen < modulusLen) {
PORT_SetError(SEC_ERROR_OUTPUT_LEN);
return SECFailure;
}
if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
return SECFailure;
}
if ((labelLen == 0 && label != NULL) ||
(labelLen > 0 && label == NULL)) {
PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
return SECFailure;
}
oaepEncoded = (unsigned char *)PORT_Alloc(modulusLen);
if (oaepEncoded == NULL) {
PORT_SetError(SEC_ERROR_NO_MEMORY);
return SECFailure;
}
rv = eme_oaep_encode(oaepEncoded, modulusLen, input, inputLen,
hashAlg, maskHashAlg, label, labelLen, seed, seedLen);
if (rv != SECSuccess)
goto done;
rv = RSA_PublicKeyOp(key, output, oaepEncoded);
if (rv != SECSuccess)
goto done;
*outputLen = modulusLen;
done:
PORT_Free(oaepEncoded);
return rv;
}
SECStatus
RSA_DecryptOAEP(RSAPrivateKey *key,
HASH_HashType hashAlg,
HASH_HashType maskHashAlg,
const unsigned char *label,
unsigned int labelLen,
unsigned char *output,
unsigned int *outputLen,
unsigned int maxOutputLen,
const unsigned char *input,
unsigned int inputLen)
{
SECStatus rv = SECFailure;
unsigned int modulusLen = rsa_modulusLen(&key->modulus);
unsigned char *oaepEncoded = NULL;
if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
return SECFailure;
}
if (inputLen != modulusLen) {
PORT_SetError(SEC_ERROR_INPUT_LEN);
return SECFailure;
}
if ((labelLen == 0 && label != NULL) ||
(labelLen > 0 && label == NULL)) {
PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
return SECFailure;
}
oaepEncoded = (unsigned char *)PORT_Alloc(modulusLen);
if (oaepEncoded == NULL) {
PORT_SetError(SEC_ERROR_NO_MEMORY);
return SECFailure;
}
rv = RSA_PrivateKeyOpDoubleChecked(key, oaepEncoded, input);
if (rv != SECSuccess) {
goto done;
}
rv = eme_oaep_decode(output, outputLen, maxOutputLen, oaepEncoded,
modulusLen, hashAlg, maskHashAlg, label,
labelLen);
done:
if (oaepEncoded)
PORT_ZFree(oaepEncoded, modulusLen);
return rv;
}
/* XXX Doesn't set error code */
SECStatus
RSA_EncryptBlock(RSAPublicKey *key,
unsigned char *output,
unsigned int *outputLen,
unsigned int maxOutputLen,
const unsigned char *input,
unsigned int inputLen)
{
SECStatus rv;
unsigned int modulusLen = rsa_modulusLen(&key->modulus);
SECItem formatted;
SECItem unformatted;
formatted.data = NULL;
if (maxOutputLen < modulusLen)
goto failure;
unformatted.len = inputLen;
unformatted.data = (unsigned char *)input;
formatted.data = NULL;
rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockPublic,
&unformatted);
if (rv != SECSuccess)
goto failure;
rv = RSA_PublicKeyOp(key, output, formatted.data);
if (rv != SECSuccess)
goto failure;
PORT_ZFree(formatted.data, modulusLen);
*outputLen = modulusLen;
return SECSuccess;
failure:
if (formatted.data != NULL)
PORT_ZFree(formatted.data, modulusLen);
return SECFailure;
}
static HMACContext *
rsa_GetHMACContext(const SECHashObject *hash, RSAPrivateKey *key,
const unsigned char *input, unsigned int inputLen)
{
unsigned char keyHash[HASH_LENGTH_MAX];
void *hashContext;
HMACContext *hmac = NULL;
unsigned int privKeyLen = key->privateExponent.len;
unsigned int keyLen;
SECStatus rv;
/* first get the key hash (should store in the key structure) */
PORT_Memset(keyHash, 0, sizeof(keyHash));
hashContext = (*hash->create)();
if (hashContext == NULL) {
return NULL;
}
(*hash->begin)(hashContext);
if (privKeyLen < inputLen) {
int padLen = inputLen - privKeyLen;
while (padLen > sizeof(keyHash)) {
(*hash->update)(hashContext, keyHash, sizeof(keyHash));
padLen -= sizeof(keyHash);
}
(*hash->update)(hashContext, keyHash, padLen);
}
(*hash->update)(hashContext, key->privateExponent.data, privKeyLen);
(*hash->end)(hashContext, keyHash, &keyLen, sizeof(keyHash));
(*hash->destroy)(hashContext, PR_TRUE);
/* now create the hmac key */
hmac = HMAC_Create(hash, keyHash, keyLen, PR_TRUE);
if (hmac == NULL) {
PORT_Memset(keyHash, 0, sizeof(keyHash));
return NULL;
}
HMAC_Begin(hmac);
HMAC_Update(hmac, input, inputLen);
rv = HMAC_Finish(hmac, keyHash, &keyLen, sizeof(keyHash));
if (rv != SECSuccess) {
PORT_Memset(keyHash, 0, sizeof(keyHash));
HMAC_Destroy(hmac, PR_TRUE);
return NULL;
}
/* Finally set the new key into the hash context. We
* reuse the original context allocated above so we don't
* need to allocate and free another one */
rv = HMAC_ReInit(hmac, hash, keyHash, keyLen, PR_TRUE);
PORT_Memset(keyHash, 0, sizeof(keyHash));
if (rv != SECSuccess) {
HMAC_Destroy(hmac, PR_TRUE);
return NULL;
}
return hmac;
}
static SECStatus
rsa_HMACPrf(HMACContext *hmac, const char *label, int labelLen,
int hashLength, unsigned char *output, int length)
{
unsigned char iterator[2] = { 0, 0 };
unsigned char encodedLen[2] = { 0, 0 };
unsigned char hmacLast[HASH_LENGTH_MAX];
unsigned int left = length;
unsigned int hashReturn;
SECStatus rv = SECSuccess;
/* encodedLen is in bits, length is in bytes, thus the shifts
* do an implied multiply by 8 */
encodedLen[0] = (length >> 5) & 0xff;
encodedLen[1] = (length << 3) & 0xff;
while (left > hashLength) {
HMAC_Begin(hmac);
HMAC_Update(hmac, iterator, 2);
HMAC_Update(hmac, (const unsigned char *)label, labelLen);
HMAC_Update(hmac, encodedLen, 2);
rv = HMAC_Finish(hmac, output, &hashReturn, hashLength);
if (rv != SECSuccess) {
return rv;
}
iterator[1]++;
if (iterator[1] == 0)
iterator[0]++;
left -= hashLength;
output += hashLength;
}
if (left) {
HMAC_Begin(hmac);
HMAC_Update(hmac, iterator, 2);
HMAC_Update(hmac, (const unsigned char *)label, labelLen);
HMAC_Update(hmac, encodedLen, 2);
rv = HMAC_Finish(hmac, hmacLast, &hashReturn, sizeof(hmacLast));
if (rv != SECSuccess) {
return rv;
}
PORT_Memcpy(output, hmacLast, left);
PORT_Memset(hmacLast, 0, sizeof(hmacLast));
}
return rv;
}
/* This function takes a 16-bit input number and
* creates the smallest mask which covers
* the whole number. Examples:
* 0x81 -> 0xff
* 0x1af -> 0x1ff
* 0x4d1 -> 0x7ff
*/
static int
makeMask16(int len)
{
// or the high bit in each bit location
len |= (len >> 1);
len |= (len >> 2);
len |= (len >> 4);
len |= (len >> 8);
return len;
}
#define STRING_AND_LENGTH(s) s, sizeof(s) - 1
static int
rsa_GetErrorLength(HMACContext *hmac, int hashLen, int maxLegalLen)
{
unsigned char out[128 * 2];
unsigned char *outp;
int outLength = 0;
int lengthMask;
SECStatus rv;
lengthMask = makeMask16(maxLegalLen);
rv = rsa_HMACPrf(hmac, STRING_AND_LENGTH("length"), hashLen,
out, sizeof(out));
if (rv != SECSuccess) {
return -1;
}
for (outp = out; outp < out + sizeof(out); outp += 2) {
int candidate = outp[0] << 8 | outp[1];
candidate = candidate & lengthMask;
outLength = PORT_CT_SEL(PORT_CT_LT(candidate, maxLegalLen),
candidate, outLength);
}
PORT_Memset(out, 0, sizeof(out));
return outLength;
}
/*
* This function can only fail in environmental cases: Programming errors
* and out of memory situations. It can't fail if the keys are valid and
* the inputs are the proper size. If the actual RSA decryption fails, a
* fake value and a fake length, both of which have already been generated
* based on the key and input, are returned.
* Applications are expected to detect decryption failures based on the fact
* that the decrypted value (usually a key) doesn't validate. The prevents
* Blecheinbaucher style attacks against the key. */
SECStatus
RSA_DecryptBlock(RSAPrivateKey *key,
unsigned char *output,
unsigned int *outputLen,
unsigned int maxOutputLen,
const unsigned char *input,
unsigned int inputLen)
{
SECStatus rv;
PRUint32 fail;
unsigned int modulusLen = rsa_modulusLen(&key->modulus);
unsigned int i;
unsigned char *buffer = NULL;
unsigned char *errorBuffer = NULL;
unsigned char *bp = NULL;
unsigned char *ep = NULL;
unsigned int outLen = modulusLen;
unsigned int maxLegalLen = modulusLen - 10;
unsigned int errorLength;
const SECHashObject *hashObj;
HMACContext *hmac = NULL;
/* failures in the top section indicate failures in the environment
* (memory) or the library. OK to return errors in these cases because
* it doesn't provide any oracle information to attackers. */
if (inputLen != modulusLen || modulusLen < 10) {
PORT_SetError(SEC_ERROR_INVALID_ARGS);
return SECFailure;
}
/* Allocate enough space to decrypt */
buffer = PORT_ZAlloc(modulusLen);
if (!buffer) {
goto loser;
}
errorBuffer = PORT_ZAlloc(modulusLen);
if (!errorBuffer) {
goto loser;
}
hashObj = HASH_GetRawHashObject(HASH_AlgSHA256);
if (hashObj == NULL) {
goto loser;
}
/* calculate the values to return in the error case rather than
* the actual returned values. This data is the same for the
* same input and private key. */
hmac = rsa_GetHMACContext(hashObj, key, input, inputLen);
if (hmac == NULL) {
goto loser;
}
errorLength = rsa_GetErrorLength(hmac, hashObj->length, maxLegalLen);
if (((int)errorLength) < 0) {
goto loser;
}
/* we always have to generate a full moduluslen error string. Otherwise
* we create a timing dependency on errorLength, which could be used to
* determine the difference between errorLength and outputLen and tell
* us that there was a pkcs1 decryption failure */
rv = rsa_HMACPrf(hmac, STRING_AND_LENGTH("message"),
hashObj->length, errorBuffer, modulusLen);
if (rv != SECSuccess) {
goto loser;
}
HMAC_Destroy(hmac, PR_TRUE);
hmac = NULL;
/* From here on out, we will always return success. If there is
* an error, we will return deterministic output based on the key
* and the input data. */
rv = RSA_PrivateKeyOp(key, buffer, input);
fail = PORT_CT_NE(rv, SECSuccess);
fail |= PORT_CT_NE(buffer[0], RSA_BLOCK_FIRST_OCTET) | PORT_CT_NE(buffer[1], RSA_BlockPublic);
/* There have to be at least 8 bytes of padding. */
for (i = 2; i < 10; i++) {
fail |= PORT_CT_EQ(buffer[i], RSA_BLOCK_AFTER_PAD_OCTET);
}
for (i = 10; i < modulusLen; i++) {
unsigned int newLen = modulusLen - i - 1;
PRUint32 condition = PORT_CT_EQ(buffer[i], RSA_BLOCK_AFTER_PAD_OCTET) & PORT_CT_EQ(outLen, modulusLen);
outLen = PORT_CT_SEL(condition, newLen, outLen);
}
// this can only happen if a zero wasn't found above
fail |= PORT_CT_GE(outLen, modulusLen);
outLen = PORT_CT_SEL(fail, errorLength, outLen);
/* index into the correct buffer. Do it before we truncate outLen if the
* application was asking for less data than we can return */
bp = buffer + modulusLen - outLen;
ep = errorBuffer + modulusLen - outLen;
/* at this point, outLen returns no information about decryption failures,
* no need to hide its value. maxOutputLen is how much data the
* application is expecting, which is also not sensitive. */
if (outLen > maxOutputLen) {
outLen = maxOutputLen;
}
/* we can't use PORT_Memcpy because caching could create a time dependency
* on the status of fail. */
for (i = 0; i < outLen; i++) {
output[i] = PORT_CT_SEL(fail, ep[i], bp[i]);
}
*outputLen = outLen;
PORT_Free(buffer);
PORT_Free(errorBuffer);
return SECSuccess;
loser:
if (hmac) {
HMAC_Destroy(hmac, PR_TRUE);
}
PORT_Free(buffer);
PORT_Free(errorBuffer);
return SECFailure;
}
/*
* Encode a RSA-PSS signature.
* Described in RFC 3447, section 9.1.1.
* We use mHash instead of M as input.
* emBits from the RFC is just modBits - 1, see section 8.1.1.
* We only support MGF1 as the MGF.
*/
SECStatus
RSA_EMSAEncodePSS(unsigned char *em,
unsigned int emLen,
unsigned int emBits,
const unsigned char *mHash,
HASH_HashType hashAlg,
HASH_HashType maskHashAlg,
const unsigned char *salt,
unsigned int saltLen)
{
const SECHashObject *hash;
void *hash_context;
unsigned char *dbMask;
unsigned int dbMaskLen;
unsigned int i;
SECStatus rv;
hash = HASH_GetRawHashObject(hashAlg);
dbMaskLen = emLen - hash->length - 1;
/* Step 3 */
if (emLen < hash->length + saltLen + 2) {
PORT_SetError(SEC_ERROR_OUTPUT_LEN);
return SECFailure;
}
/* Step 4 */
if (salt == NULL) {
rv = RNG_GenerateGlobalRandomBytes(&em[dbMaskLen - saltLen], saltLen);
if (rv != SECSuccess) {
return rv;
}
} else {
PORT_Memcpy(&em[dbMaskLen - saltLen], salt, saltLen);
}
/* Step 5 + 6 */
/* Compute H and store it at its final location &em[dbMaskLen]. */
hash_context = (*hash->create)();
if (hash_context == NULL) {
PORT_SetError(SEC_ERROR_NO_MEMORY);
return SECFailure;
}
(*hash->begin)(hash_context);
(*hash->update)(hash_context, eightZeros, 8);
(*hash->update)(hash_context, mHash, hash->length);
(*hash->update)(hash_context, &em[dbMaskLen - saltLen], saltLen);
(*hash->end)(hash_context, &em[dbMaskLen], &i, hash->length);
(*hash->destroy)(hash_context, PR_TRUE);
/* Step 7 + 8 */
PORT_Memset(em, 0, dbMaskLen - saltLen - 1);
em[dbMaskLen - saltLen - 1] = 0x01;
/* Step 9 */
dbMask = (unsigned char *)PORT_Alloc(dbMaskLen);
if (dbMask == NULL) {
PORT_SetError(SEC_ERROR_NO_MEMORY);
return SECFailure;
}
MGF1(maskHashAlg, dbMask, dbMaskLen, &em[dbMaskLen], hash->length);
/* Step 10 */
for (i = 0; i < dbMaskLen; i++)
em[i] ^= dbMask[i];
PORT_Free(dbMask);
/* Step 11 */
em[0] &= 0xff >> (8 * emLen - emBits);
/* Step 12 */
em[emLen - 1] = 0xbc;
return SECSuccess;
}
/*
* Verify a RSA-PSS signature.
* Described in RFC 3447, section 9.1.2.
* We use mHash instead of M as input.
* emBits from the RFC is just modBits - 1, see section 8.1.2.
* We only support MGF1 as the MGF.
*/
static SECStatus
emsa_pss_verify(const unsigned char *mHash,
const unsigned char *em,
unsigned int emLen,
unsigned int emBits,
HASH_HashType hashAlg,
HASH_HashType maskHashAlg,
unsigned int saltLen)
{
const SECHashObject *hash;
void *hash_context;
unsigned char *db;
unsigned char *H_; /* H' from the RFC */
unsigned int i;
unsigned int dbMaskLen;
unsigned int zeroBits;
SECStatus rv;
hash = HASH_GetRawHashObject(hashAlg);
dbMaskLen = emLen - hash->length - 1;
/* Step 3 + 4 */
if ((emLen < (hash->length + saltLen + 2)) ||
(em[emLen - 1] != 0xbc)) {
PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
return SECFailure;
}
/* Step 6 */
zeroBits = 8 * emLen - emBits;
if (em[0] >> (8 - zeroBits)) {
PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
return SECFailure;
}
/* Step 7 */
db = (unsigned char *)PORT_Alloc(dbMaskLen);
if (db == NULL) {
PORT_SetError(SEC_ERROR_NO_MEMORY);
return SECFailure;
}
/* &em[dbMaskLen] points to H, used as mgfSeed */
MGF1(maskHashAlg, db, dbMaskLen, &em[dbMaskLen], hash->length);
/* Step 8 */
for (i = 0; i < dbMaskLen; i++) {
db[i] ^= em[i];
}
/* Step 9 */
db[0] &= 0xff >> zeroBits;
/* Step 10 */
for (i = 0; i < (dbMaskLen - saltLen - 1); i++) {
if (db[i] != 0) {
PORT_Free(db);
PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
return SECFailure;
}
}
if (db[dbMaskLen - saltLen - 1] != 0x01) {
PORT_Free(db);
PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
return SECFailure;
}
/* Step 12 + 13 */
H_ = (unsigned char *)PORT_Alloc(hash->length);
if (H_ == NULL) {
PORT_Free(db);
PORT_SetError(SEC_ERROR_NO_MEMORY);
return SECFailure;
}
hash_context = (*hash->create)();
if (hash_context == NULL) {
PORT_Free(db);
PORT_Free(H_);
PORT_SetError(SEC_ERROR_NO_MEMORY);
return SECFailure;
}
(*hash->begin)(hash_context);
(*hash->update)(hash_context, eightZeros, 8);
(*hash->update)(hash_context, mHash, hash->length);
(*hash->update)(hash_context, &db[dbMaskLen - saltLen], saltLen);
(*hash->end)(hash_context, H_, &i, hash->length);
(*hash->destroy)(hash_context, PR_TRUE);
PORT_Free(db);
/* Step 14 */
if (PORT_Memcmp(H_, &em[dbMaskLen], hash->length) != 0) {
PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
rv = SECFailure;
} else {
rv = SECSuccess;
}
PORT_Free(H_);
return rv;
}
SECStatus
RSA_SignPSS(RSAPrivateKey *key,
HASH_HashType hashAlg,
HASH_HashType maskHashAlg,
const unsigned char *salt,
unsigned int saltLength,
unsigned char *output,
unsigned int *outputLen,
unsigned int maxOutputLen,
const unsigned char *input,
unsigned int inputLen)
{
SECStatus rv = SECSuccess;
unsigned int modulusLen = rsa_modulusLen(&key->modulus);
unsigned int modulusBits = rsa_modulusBits(&key->modulus);
unsigned int emLen = modulusLen;
unsigned char *pssEncoded, *em;
if (maxOutputLen < modulusLen) {
PORT_SetError(SEC_ERROR_OUTPUT_LEN);
return SECFailure;
}
if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
return SECFailure;
}
pssEncoded = em = (unsigned char *)PORT_Alloc(modulusLen);
if (pssEncoded == NULL) {
PORT_SetError(SEC_ERROR_NO_MEMORY);
return SECFailure;
}
/* len(em) == ceil((modulusBits - 1) / 8). */
if (modulusBits % 8 == 1) {
em[0] = 0;
emLen--;
em++;
}
rv = RSA_EMSAEncodePSS(em, emLen, modulusBits - 1, input, hashAlg,
maskHashAlg, salt, saltLength);
if (rv != SECSuccess)
goto done;
// This sets error codes upon failure.
rv = RSA_PrivateKeyOpDoubleChecked(key, output, pssEncoded);
*outputLen = modulusLen;
done:
PORT_Free(pssEncoded);
return rv;
}
SECStatus
RSA_CheckSignPSS(RSAPublicKey *key,
HASH_HashType hashAlg,
HASH_HashType maskHashAlg,
unsigned int saltLength,
const unsigned char *sig,
unsigned int sigLen,
const unsigned char *hash,
unsigned int hashLen)
{
SECStatus rv;
unsigned int modulusLen = rsa_modulusLen(&key->modulus);
unsigned int modulusBits = rsa_modulusBits(&key->modulus);
unsigned int emLen = modulusLen;
unsigned char *buffer, *em;
if (sigLen != modulusLen) {
PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
return SECFailure;
}
if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
return SECFailure;
}
buffer = em = (unsigned char *)PORT_Alloc(modulusLen);
if (!buffer) {
PORT_SetError(SEC_ERROR_NO_MEMORY);
return SECFailure;
}
rv = RSA_PublicKeyOp(key, buffer, sig);
if (rv != SECSuccess) {
PORT_Free(buffer);
PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
return SECFailure;
}
/* len(em) == ceil((modulusBits - 1) / 8). */
if (modulusBits % 8 == 1) {
emLen--;
em++;
}
rv = emsa_pss_verify(hash, em, emLen, modulusBits - 1, hashAlg,
maskHashAlg, saltLength);
PORT_Free(buffer);
return rv;
}
SECStatus
RSA_Sign(RSAPrivateKey *key,
unsigned char *output,
unsigned int *outputLen,
unsigned int maxOutputLen,
const unsigned char *input,
unsigned int inputLen)
{
SECStatus rv = SECFailure;
unsigned int modulusLen = rsa_modulusLen(&key->modulus);
SECItem formatted = { siBuffer, NULL, 0 };
SECItem unformatted = { siBuffer, (unsigned char *)input, inputLen };
if (maxOutputLen < modulusLen) {
PORT_SetError(SEC_ERROR_OUTPUT_LEN);
goto done;
}
rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockPrivate,
&unformatted);
if (rv != SECSuccess) {
PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
goto done;
}
// This sets error codes upon failure.
rv = RSA_PrivateKeyOpDoubleChecked(key, output, formatted.data);
*outputLen = modulusLen;
done:
if (formatted.data != NULL) {
PORT_ZFree(formatted.data, modulusLen);
}
return rv;
}
SECStatus
RSA_CheckSign(RSAPublicKey *key,
const unsigned char *sig,
unsigned int sigLen,
const unsigned char *data,
unsigned int dataLen)
{
SECStatus rv = SECFailure;
unsigned int modulusLen = rsa_modulusLen(&key->modulus);
unsigned int i;
unsigned char *buffer = NULL;
if (sigLen != modulusLen) {
PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
goto done;
}
/*
* 0x00 || BT || Pad || 0x00 || ActualData
*
* The "3" below is the first octet + the second octet + the 0x00
* octet that always comes just before the ActualData.
*/
if (dataLen > modulusLen - (3 + RSA_BLOCK_MIN_PAD_LEN)) {
PORT_SetError(SEC_ERROR_BAD_DATA);
goto done;
}
buffer = (unsigned char *)PORT_Alloc(modulusLen + 1);
if (!buffer) {
PORT_SetError(SEC_ERROR_NO_MEMORY);
goto done;
}
if (RSA_PublicKeyOp(key, buffer, sig) != SECSuccess) {
PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
goto done;
}
/*
* check the padding that was used
*/
if (buffer[0] != RSA_BLOCK_FIRST_OCTET ||
buffer[1] != (unsigned char)RSA_BlockPrivate) {
PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
goto done;
}
for (i = 2; i < modulusLen - dataLen - 1; i++) {
if (buffer[i] != RSA_BLOCK_PRIVATE_PAD_OCTET) {
PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
goto done;
}
}
if (buffer[i] != RSA_BLOCK_AFTER_PAD_OCTET) {
PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
goto done;
}
/*
* make sure we get the same results
*/
if (PORT_Memcmp(buffer + modulusLen - dataLen, data, dataLen) == 0) {
rv = SECSuccess;
}
done:
if (buffer) {
PORT_Free(buffer);
}
return rv;
}
SECStatus
RSA_CheckSignRecover(RSAPublicKey *key,
unsigned char *output,
unsigned int *outputLen,
unsigned int maxOutputLen,
const unsigned char *sig,
unsigned int sigLen)
{
SECStatus rv = SECFailure;
unsigned int modulusLen = rsa_modulusLen(&key->modulus);
unsigned int i;
unsigned char *buffer = NULL;
unsigned int padLen;
if (sigLen != modulusLen) {
PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
goto done;
}
buffer = (unsigned char *)PORT_Alloc(modulusLen + 1);
if (!buffer) {
PORT_SetError(SEC_ERROR_NO_MEMORY);
goto done;
}
if (RSA_PublicKeyOp(key, buffer, sig) != SECSuccess) {
PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
goto done;
}
*outputLen = 0;
/*
* check the padding that was used
*/
if (buffer[0] != RSA_BLOCK_FIRST_OCTET ||
buffer[1] != (unsigned char)RSA_BlockPrivate) {
PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
goto done;
}
for (i = 2; i < modulusLen; i++) {
if (buffer[i] == RSA_BLOCK_AFTER_PAD_OCTET) {
*outputLen = modulusLen - i - 1;
break;
}
if (buffer[i] != RSA_BLOCK_PRIVATE_PAD_OCTET) {
PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
goto done;
}
}
padLen = i - 2;
if (padLen < RSA_BLOCK_MIN_PAD_LEN) {
PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
goto done;
}
if (*outputLen == 0) {
PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
goto done;
}
if (*outputLen > maxOutputLen) {
PORT_SetError(SEC_ERROR_OUTPUT_LEN);
goto done;
}
PORT_Memcpy(output, buffer + modulusLen - *outputLen, *outputLen);
rv = SECSuccess;
done:
if (buffer) {
PORT_Free(buffer);
}
return rv;
}