Check return from OpenSSL allocation functions. Fixes #2721

This commit is contained in:
dmiller 2026-06-18 23:26:06 +00:00
parent 1e1881ee65
commit b3620f74c3
5 changed files with 87 additions and 11 deletions

View file

@ -153,6 +153,8 @@ static char *make_nonce(const struct timeval *tv)
Snprintf(time_buf, sizeof(time_buf), "%lu.%06lu",
(long unsigned) tv->tv_sec, (long unsigned) tv->tv_usec);
md5 = EVP_MD_CTX_new();
if (md5 == NULL)
bye("Failed to allocate MD context.");
EVP_DigestInit_ex(md5, EVP_md5(), NULL);
EVP_DigestUpdate(md5, secret, sizeof(secret));
EVP_DigestUpdate(md5, ":", 1);
@ -181,6 +183,8 @@ static void make_response(char buf[EVP_MAX_MD_SIZE * 2 + 1],
/* Calculate H(A1). */
md5 = EVP_MD_CTX_new();
if (md5 == NULL)
bye("Failed to allocate MD context.");
EVP_DigestInit_ex(md5, md, NULL);
EVP_DigestUpdate(md5, username, strlen(username));
EVP_DigestUpdate(md5, ":", 1);

View file

@ -485,6 +485,7 @@ static int ssl_gen_cert(X509 **cert, EVP_PKEY **key)
const char *commonName = "localhost";
char dNSName[128];
int rc;
unsigned long err = 0;
#if OPENSSL_VERSION_NUMBER < 0x30000000L
int ret = 0;
RSA *rsa = NULL;
@ -492,29 +493,52 @@ static int ssl_gen_cert(X509 **cert, EVP_PKEY **key)
*cert = NULL;
*key = NULL;
ERR_clear_error();
/* Generate a private key. */
*key = EVP_PKEY_new();
if (*key == NULL)
goto err;
do {
rc = -1;
if (rsa != NULL) {
RSA_free(rsa);
rsa = NULL;
}
/* Generate RSA key. */
bne = BN_new();
if (bne == NULL)
break;
ret = BN_set_word(bne, RSA_F4);
if (ret != 1)
goto err;
break;
rsa = RSA_new();
if (rsa == NULL)
break;
ret = RSA_generate_key_ex(rsa, DEFAULT_KEY_BITS, bne, NULL);
if (ret != 1)
goto err;
break;
BN_free(bne);
bne = NULL;
rc = RSA_check_key(rsa);
} while (rc == 0);
if (rc == -1)
bye("Error generating RSA key: %s", ERR_error_string(ERR_get_error(), NULL));
if (bne != NULL) {
BN_free(bne);
bne = NULL;
}
if (rc == -1 || rsa == NULL) {
if (rsa != NULL) {
RSA_free(rsa);
rsa = NULL;
}
goto err;
}
if (EVP_PKEY_assign_RSA(*key, rsa) == 0) {
RSA_free(rsa);
rsa = NULL;
goto err;
}
#else
@ -610,6 +634,9 @@ err:
if (*key != NULL)
EVP_PKEY_free(*key);
while (0 != (err = ERR_get_error()))
loguser("SSL error: %s", ERR_error_string(err, NULL));
return 0;
}

View file

@ -307,10 +307,7 @@ static int gen_cert(X509 **cert, EVP_PKEY **key,
if (*key == NULL)
goto err;
do {
if (bne != NULL) {
BN_free(bne);
bne = NULL;
}
rc = -1;
if (rsa != NULL) {
RSA_free(rsa);
rsa = NULL;
@ -319,17 +316,29 @@ static int gen_cert(X509 **cert, EVP_PKEY **key,
bne = BN_new();
ret = BN_set_word(bne, RSA_F4);
if (ret != 1)
goto err;
break;
rsa = RSA_new();
ret = RSA_generate_key_ex(rsa, KEY_BITS, bne, NULL);
if (ret != 1)
goto err;
break;
BN_free(bne);
bne = NULL;
/* Check RSA key. */
rc = RSA_check_key(rsa);
} while (rc == 0);
if (rc == -1)
if (bne != NULL) {
BN_free(bne);
bne = NULL;
}
if (rc == -1 || rsa == NULL) {
if (rsa != NULL) {
RSA_free(rsa);
rsa = NULL;
}
goto err;
}
if (EVP_PKEY_assign_RSA(*key, rsa) == 0) {
RSA_free(rsa);
rsa = NULL;

View file

@ -128,6 +128,10 @@ int Crypto::aes128_cbc_encrypt(u8 *inbuff, size_t inlen, u8 *dst_buff, u8 *key,
int flen=0, flen2=0;
#if HAVE_OPAQUE_EVP_PKEY
EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new();
if (ctx == NULL) {
nping_print(DBG_4, "Failed to allocate cipher context");
return OP_FAILURE;
}
#else
EVP_CIPHER_CTX stack_ctx;
EVP_CIPHER_CTX *ctx = &stack_ctx;
@ -167,6 +171,10 @@ int Crypto::aes128_cbc_decrypt(u8 *inbuff, size_t inlen, u8 *dst_buff, u8 *key,
int flen1=0, flen2=0;
#if HAVE_OPAQUE_EVP_PKEY
EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new();
if (ctx == NULL) {
nping_print(DBG_4, "Failed to allocate cipher context");
return OP_FAILURE;
}
#else
EVP_CIPHER_CTX stack_ctx;
EVP_CIPHER_CTX *ctx = &stack_ctx;
@ -221,6 +229,10 @@ u8 *Crypto::deriveKey(const u8 *from, size_t fromlen, size_t *final_len){
static u8 next[MAX(SHA256_HASH_LEN, EVP_MAX_MD_SIZE)];
unsigned int lastlen;
EVP_MD_CTX *ctx = EVP_MD_CTX_new();
if (ctx == NULL) {
nping_print(DBG_4, "Failed to allocate MD context");
return NULL;
}
if( EVP_MD_size(EVP_sha256()) != SHA256_HASH_LEN )
nping_fatal(QT_2, "OpenSSL is broken. SHA256 len is %d\n", EVP_MD_size(EVP_sha256()) );

View file

@ -58,6 +58,7 @@ static int l_bignum_bin2bn( lua_State *L ) /** bignum_bin2bn( string s ) */
size_t len;
const unsigned char * s = (unsigned char *) luaL_checklstring( L, 1, &len );
BIGNUM * num = BN_new();
if (!num) return NSE_SSL_LUA_ERR(L);
BN_bin2bn( s, len, num );
return nse_pushbn(L, num, true);
}
@ -66,6 +67,7 @@ static int l_bignum_dec2bn( lua_State *L ) /** bignum_dec2bn( string s ) */
{
const char * s = luaL_checkstring( L, 1 );
BIGNUM * num = BN_new();
if (!num) return NSE_SSL_LUA_ERR(L);
BN_dec2bn( &num, s );
return nse_pushbn(L, num, true);
}
@ -74,6 +76,7 @@ static int l_bignum_hex2bn( lua_State *L ) /** bignum_hex2bn( string s ) */
{
const char * s = luaL_checkstring( L, 1 );
BIGNUM * num = BN_new();
if (!num) return NSE_SSL_LUA_ERR(L);
BN_hex2bn( &num, s );
return nse_pushbn(L, num, true);
}
@ -82,6 +85,7 @@ static int l_bignum_rand( lua_State *L ) /** bignum_rand( number bits ) */
{
size_t bits = luaL_checkinteger( L, 1 );
BIGNUM * num = BN_new();
if (!num) return NSE_SSL_LUA_ERR(L);
BN_rand( num, bits, -1, 0 );
return nse_pushbn(L, num, true);
}
@ -92,7 +96,12 @@ static int l_bignum_mod_exp( lua_State *L ) /** bignum_mod_exp( BIGNUM a, BIGNUM
bignum_data_t * p = (bignum_data_t *) luaL_checkudata(L, 2, "BIGNUM");
bignum_data_t * m = (bignum_data_t *) luaL_checkudata(L, 3, "BIGNUM");
BIGNUM * result = BN_new();
if (!result) return NSE_SSL_LUA_ERR(L);
BN_CTX * ctx = BN_CTX_new();
if (!ctx) {
BN_free(result);
return NSE_SSL_LUA_ERR(L);
}
BN_mod_exp( result, a->bn, p->bn, m->bn, ctx );
BN_CTX_free( ctx );
return nse_pushbn(L, result, true);
@ -103,8 +112,18 @@ static int l_bignum_div( lua_State *L ) /* bignum_div( BIGNUM a, BIGNUM d ) */
bignum_data_t * a = (bignum_data_t *) luaL_checkudata(L, 1, "BIGNUM");
bignum_data_t * d = (bignum_data_t *) luaL_checkudata(L, 2, "BIGNUM");
BIGNUM * dv = BN_new();
if (!dv) return NSE_SSL_LUA_ERR(L);
BIGNUM * rem = BN_new();
if (!rem) {
BN_free(dv);
return NSE_SSL_LUA_ERR(L);
}
BN_CTX * ctx = BN_CTX_new();
if (!ctx) {
BN_free(dv);
BN_free(rem);
return NSE_SSL_LUA_ERR(L);
}
BN_div(dv, rem, a->bn, d->bn, ctx);
BN_CTX_free( ctx );
nse_pushbn(L, dv, true);
@ -117,6 +136,7 @@ static int l_bignum_add( lua_State *L ) /** bignum_add( BIGNUM a, BIGNUM b ) */
bignum_data_t * a = (bignum_data_t *) luaL_checkudata(L, 1, "BIGNUM");
bignum_data_t * b = (bignum_data_t *) luaL_checkudata(L, 2, "BIGNUM");
BIGNUM * result = BN_new();
if (!result) return NSE_SSL_LUA_ERR(L);
BN_add( result, a->bn, b->bn );
return nse_pushbn(L, result, true);
}
@ -163,6 +183,7 @@ static int l_bignum_is_prime( lua_State *L ) /** bignum_is_prime( BIGNUM p ) */
{
bignum_data_t * p = (bignum_data_t *) luaL_checkudata( L, 1, "BIGNUM" );
BN_CTX * ctx = BN_CTX_new();
if (!ctx) return NSE_SSL_LUA_ERR(L);
int is_prime =
#if OPENSSL_VERSION_NUMBER < 0x30000000L
BN_is_prime_ex( p->bn, BN_prime_checks, ctx, NULL );
@ -178,6 +199,7 @@ static int l_bignum_is_safe_prime( lua_State *L ) /** bignum_is_safe_prime( BIGN
{
bignum_data_t * p = (bignum_data_t *) luaL_checkudata( L, 1, "BIGNUM" );
BN_CTX * ctx = BN_CTX_new();
if (!ctx) return NSE_SSL_LUA_ERR(L);
int is_prime =
#if OPENSSL_VERSION_NUMBER < 0x30000000L
BN_is_prime_ex( p->bn, BN_prime_checks, ctx, NULL );
@ -396,6 +418,7 @@ static int l_encrypt(lua_State *L) /** encrypt( string algorithm, string key, st
#if HAVE_OPAQUE_STRUCTS
EVP_CIPHER_CTX *cipher_ctx = EVP_CIPHER_CTX_new();
if (!cipher_ctx) return NSE_SSL_LUA_ERR(L);
#else
EVP_CIPHER_CTX stack_ctx;
EVP_CIPHER_CTX *cipher_ctx = &stack_ctx;
@ -458,6 +481,7 @@ static int l_decrypt(lua_State *L) /** decrypt( string algorithm, string key, st
#if HAVE_OPAQUE_STRUCTS
EVP_CIPHER_CTX *cipher_ctx = EVP_CIPHER_CTX_new();
if (!cipher_ctx) return NSE_SSL_LUA_ERR(L);
#else
EVP_CIPHER_CTX stack_ctx;
EVP_CIPHER_CTX *cipher_ctx = &stack_ctx;