crypto: fix checkPrime crash with large buffers

Fixes: #56512
PR-URL: #56559
Reviewed-By: Colin Ihrig <[email protected]>
Reviewed-By: James M Snell <[email protected]>
Reviewed-By: Joyee Cheung <[email protected]>

Author: santigimeno

src/crypto/crypto_random.cc

@@ -176,6 +176,11 @@ Maybe<void> CheckPrimeTraits::AdditionalConfig(
   ArrayBufferOrViewContents<unsigned char> candidate(args[offset]);
 
   params->candidate = BignumPointer(candidate.data(), candidate.size());
+  if (!params->candidate) {
+    ThrowCryptoError(
+        Environment::GetCurrent(args), ERR_get_error(), "BignumPointer");
+    return Nothing<void>();
+  }
 
   CHECK(args[offset + 1]->IsInt32());  // Checks
   params->checks = args[offset + 1].As<Int32>()->Value();

test/parallel/test-crypto-prime.js

@@ -254,6 +254,19 @@ for (const checks of [-(2 ** 31), -1, 2 ** 31, 2 ** 32 - 1, 2 ** 32, 2 ** 50]) {
   });
 }
 
+{
+  const bytes = Buffer.alloc(67108864);
+  bytes[0] = 0x1;
+  assert.throws(() => checkPrime(bytes, common.mustNotCall()), {
+    code: 'ERR_OSSL_BN_BIGNUM_TOO_LONG',
+    message: /bignum too long/
+  });
+  assert.throws(() => checkPrimeSync(bytes), {
+    code: 'ERR_OSSL_BN_BIGNUM_TOO_LONG',
+    message: /bignum too long/
+  });
+}
+
 assert(!checkPrimeSync(Buffer.from([0x1])));
 assert(checkPrimeSync(Buffer.from([0x2])));
 assert(checkPrimeSync(Buffer.from([0x3])));