fix(pkey) use group bits instead of ECDSA_sig to get parameter size in ECDSA signature (#102)

Author: fffonion

lib/resty/openssl/auxiliary/ecdsa.lua

@@ -7,7 +7,7 @@ local C = ffi.C
 require "resty.openssl.include.ecdsa"
 local bn_lib = require "resty.openssl.bn"
 local format_error = require("resty.openssl.err").format_error
-local floor = math.floor
+local ceil = math.ceil
 
 local _M = {}
 
@@ -26,20 +26,25 @@ SEQUENCE {
   The binary form is typically 64 bytes.
 ]]
 
-local function sig_size(ec_key)
-  local sz = C.ECDSA_size(ec_key)
-  if sz <= 8 then
-    error("failed to get ECDSA signature size", 2)
+local function group_size(ec_key)
+  local group = C.EC_KEY_get0_group(ec_key)
+  if group == nil then
+    assert("failed to get EC group", 2)
   end
-  -- 2 bytes for ASN.1 DER header, 2 bytes for length, 2 bytes for each 2 integers, left 2 integers so /2
-  return (sz - 8) / 2
+
+  local sz = C.EC_GROUP_order_bits(group)
+  if sz <= 0 then
+    assert("failed to get EC group order bits", 2)
+  end
+
+  return ceil(sz / 8)
 end
 
 _M.sig_der2raw = function(der, ec_key)
   if ec_key == nil then
     error("ec_key is required", 2)
   end
-  local psize = sig_size(ec_key)
+  local psize = group_size(ec_key)
 
   local buf = ffi.new("const unsigned char*", der)
   local buf_ptr = ffi.new("const unsigned char*[1]", buf)
@@ -82,11 +87,8 @@ _M.sig_raw2der = function(bin, ec_key)
   if ec_key == nil then
     error("ec_key is required", 2)
   end
-  -- p521 private key x point is 65 bytes and y point is 66 bytes
-  -- 65+66+8 = 139
-  -- division by two results in a decimal and hence messes with
-  -- signature length calculation
-  local psize = floor(sig_size(ec_key))
+
+  local psize = group_size(ec_key)
 
   if #bin ~= psize * 2 then
     return nil, "invalid signature length, expect " .. (psize * 2) .. " but got " .. #bin

lib/resty/openssl/include/ecdsa.lua

@@ -12,5 +12,5 @@ ffi.cdef [[
   int i2d_ECDSA_SIG(const ECDSA_SIG *sig, unsigned char **pp);
   ECDSA_SIG *d2i_ECDSA_SIG(ECDSA_SIG **sig, const unsigned char **pp, long len);
 
-  int ECDSA_size(const EC_KEY *eckey);
+  int EC_GROUP_order_bits(const EC_GROUP *group);
 ]]

t/openssl/pkey.t

@@ -1332,8 +1332,7 @@ nilpkey:sign: ecdsa.sig_raw2der: invalid signature length, expect 64 but got \\d
 --- config
     location =/t {
         content_by_lua_block {
-            if not require("resty.openssl.version").OPENSSL_11_OR_LATER or
-                   require("resty.openssl.version").OPENSSL_3X then
+            if not require("resty.openssl.version").OPENSSL_11_OR_LATER then
                 ngx.say("132\n96\ntrue\ntrue")
                 ngx.exit(0)
             end
@@ -1347,20 +1346,18 @@ nilpkey:sign: ecdsa.sig_raw2der: invalid signature length, expect 64 but got \\d
                 type = "EC",
                 curve = "secp384r1",
             }))
-            local digest_512 = myassert(require("resty.openssl.digest").new("SHA512"))
-            local digest_384 = myassert(require("resty.openssl.digest").new("SHA384"))
+            local digest = myassert(require("resty.openssl.digest").new("SHA256"))
 
-            myassert(digest_512:update("🕶️", "+1s"))
-            myassert(digest_384:update("🕶️", "+1s"))
+            myassert(digest:update("🕶️", "+1s"))
 
-            local s_512 = myassert(p_521:sign(digest_512, nil, nil, opts))
+            local s_512 = myassert(p_521:sign(digest, nil, nil, opts))
             ngx.say(#s_512)
-            local s_384 = myassert(p_384:sign(digest_384, nil, nil, opts))
+            local s_384 = myassert(p_384:sign(digest, nil, nil, opts))
             ngx.say(#s_384)
 
-            local v_512 = myassert(p_521:verify(s_512, digest_512, nil, nil, opts))
+            local v_512 = myassert(p_521:verify(s_512, digest, nil, nil, opts))
             ngx.say(v_512)
-            local v_384 = myassert(p_384:verify(s_384, digest_384, nil, nil, opts))
+            local v_384 = myassert(p_384:verify(s_384, digest, nil, nil, opts))
             ngx.say(v_384)
         }
     }