Fix zero stripping in %g number formatting.

Reported by pwnhacker0x18.

(cherry picked from commit 343ce0e)

LuaJIT uses `lj_strfmt_wfnum()` for number formatting. In the situation
when the precision (`prec`) and amount of digits (`hilen`) for the
decimal representation are the same and `ndhi` == 0, the `ndlo` part
becomes 64 (the size of the `nd` stack buffer), and the overflow occurs.

This patch adds the corresponding mask (0x3f == 63) for the `ndlo`
incrementation result.

Sergey Kaplun:
* added the description and the test for the problem

Part of tarantool/tarantool#9595

Author: Mike Pall

src/lj_strfmt_num.c

@@ -454,7 +454,8 @@ static char *lj_strfmt_wfnum(SBuf *sb, SFormat sf, lua_Number n, char *p)
 	    prec--;
 	    if (!i) {
 	      if (ndlo == ndhi) { prec = 0; break; }
-	      lj_strfmt_wuint9(tail, nd[++ndlo]);
+	      ndlo = (ndlo + 1) & 0x3f;
+	      lj_strfmt_wuint9(tail, nd[ndlo]);
 	      i = 9;
 	    }
 	  }

test/tarantool-tests/lj-1149-g-number-formating-bufov.test.lua

@@ -0,0 +1,20 @@
+local tap = require('tap')
+
+-- Test file to demonstrate stack-buffer-overflow in the
+-- `lj_strfmt_wfnum()` call.
+-- See also: https://github.com/LuaJIT/LuaJIT/issues/1149.
+
+local test = tap.test('lj-1149-g-number-formating-bufov')
+test:plan(1)
+
+-- XXX: The test shows stack-buffer-overflow only under ASAN.
+-- The number value for the test has the same precision
+-- (`prec` = 5) and amount of digits (`hilen` = 5) for the decimal
+-- representation. Hence, with `ndhi` == 0, the `ndlo` part
+-- becomes 64 (the size of the `nd` stack buffer), and the overflow
+-- occurs.
+-- See details in the <src/lj_strfmt_num.c>:`lj_strfmt_wfnum()`.
+test:is(string.format('%7g', 0x1.144399609d407p+401), '5.5733e+120',
+        'correct format %7g result')
+
+test:done(true)