Skip to content

Commit e393448

Browse files
hexoscotthexoscott
authored
Inters speed up 3 (#1699)
* move nodevalue8 to uint64 * node value 12 from big int to uint64 * memdb changes to use uint64 rather than big int for keys
1 parent 589e949 commit e393448

14 files changed

+417
-220
lines changed

‎smt/pkg/blockinfo/keys.go

+14-2
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,20 @@ func KeyTxLogs(txIndex, logIndex *big.Int) (*utils.NodeKey, error) {
4646
return nil, errors.New("nil key")
4747
}
4848

49-
txIndexKey := utils.ScalarToArrayBig(txIndex)
50-
key1 := utils.NodeValue8{txIndexKey[0], txIndexKey[1], txIndexKey[2], txIndexKey[3], txIndexKey[4], txIndexKey[5], big.NewInt(int64(IndexBlockHeaderLogs)), big.NewInt(0)}
49+
txIndexKey, err := utils.ScalarToArrayUint64(txIndex)
50+
if err != nil {
51+
return nil, err
52+
}
53+
key1 := utils.NodeValue8{
54+
txIndexKey[0],
55+
txIndexKey[1],
56+
txIndexKey[2],
57+
txIndexKey[3],
58+
txIndexKey[4],
59+
txIndexKey[5],
60+
IndexBlockHeaderLogs,
61+
0,
62+
}
5163

5264
logIndexArray := utils.ScalarToArrayBig(logIndex)
5365
lia, err := utils.NodeValue8FromBigIntArray(logIndexArray)

‎smt/pkg/db/mdbx.go

+21-22
Original file line numberDiff line numberDiff line change
@@ -162,14 +162,14 @@ func (m *EriDb) CollectAccountValue(key utils.NodeKey, value utils.NodeValue8) {
162162
}
163163

164164
func (m *EriDb) CollectKeySource(key utils.NodeKey, value []byte) {
165-
keyConc := utils.ArrayToScalar(key[:])
166-
m.keySourceCollector.Collect(keyConc.Bytes(), value)
165+
keyBytes := utils.ArrayToBytes(key[:])
166+
m.keySourceCollector.Collect(keyBytes, value)
167167
}
168168

169169
func (m *EriDb) CollectHashKey(key utils.NodeKey, value utils.NodeKey) {
170-
keyConc := utils.ArrayToScalar(key[:])
171-
valConc := utils.ArrayToScalar(value[:])
172-
m.hashKeyCollector.Collect(keyConc.Bytes(), valConc.Bytes())
170+
keyBytes := utils.ArrayToBytes(key[:])
171+
valBytes := utils.ArrayToBytes(value[:])
172+
m.hashKeyCollector.Collect(keyBytes, valBytes)
173173
}
174174

175175
func (m *EriDb) CollectSmt(key utils.NodeKey, value utils.NodeValue12) {
@@ -313,21 +313,21 @@ func (m *EriDb) InsertAccountValue(key utils.NodeKey, value utils.NodeValue8) er
313313
}
314314

315315
func (m *EriDb) InsertKeySource(key utils.NodeKey, value []byte) error {
316-
keyConc := utils.ArrayToScalar(key[:])
316+
keyBytes := utils.ArrayToBytes(key[:])
317317

318-
return m.tx.Put(TableMetadata, keyConc.Bytes(), value)
318+
return m.tx.Put(TableMetadata, keyBytes, value)
319319
}
320320

321321
func (m *EriDb) DeleteKeySource(key utils.NodeKey) error {
322-
keyConc := utils.ArrayToScalar(key[:])
322+
keyBytes := utils.ArrayToBytes(key[:])
323323

324-
return m.tx.Delete(TableMetadata, keyConc.Bytes())
324+
return m.tx.Delete(TableMetadata, keyBytes)
325325
}
326326

327327
func (m *EriRoDb) GetKeySource(key utils.NodeKey) ([]byte, error) {
328-
keyConc := utils.ArrayToScalar(key[:])
328+
keyBytes := utils.ArrayToBytes(key[:])
329329

330-
data, err := m.kvTxRo.GetOne(TableMetadata, keyConc.Bytes())
330+
data, err := m.kvTxRo.GetOne(TableMetadata, keyBytes)
331331
if err != nil {
332332
return nil, err
333333
}
@@ -340,28 +340,27 @@ func (m *EriRoDb) GetKeySource(key utils.NodeKey) ([]byte, error) {
340340
}
341341

342342
func (m *EriDb) InsertHashKey(key utils.NodeKey, value utils.NodeKey) error {
343-
keyConc := utils.ArrayToScalar(key[:])
343+
keyBytes := utils.ArrayToBytes(key[:])
344+
valBytes := utils.ArrayToBytes(value[:])
344345

345-
valConc := utils.ArrayToScalar(value[:])
346-
347-
return m.tx.Put(TableHashKey, keyConc.Bytes(), valConc.Bytes())
346+
return m.tx.Put(TableHashKey, keyBytes, valBytes)
348347
}
349348

350349
func (m *EriDb) DeleteHashKey(key utils.NodeKey) error {
351-
keyConc := utils.ArrayToScalar(key[:])
352-
return m.tx.Delete(TableHashKey, keyConc.Bytes())
350+
keyBytes := utils.ArrayToBytes(key[:])
351+
return m.tx.Delete(TableHashKey, keyBytes)
353352
}
354353

355354
func (m *EriRoDb) GetHashKey(key utils.NodeKey) (utils.NodeKey, error) {
356-
keyConc := utils.ArrayToScalar(key[:])
355+
keyBytes := utils.ArrayToBytes(key[:])
357356

358-
data, err := m.kvTxRo.GetOne(TableHashKey, keyConc.Bytes())
357+
data, err := m.kvTxRo.GetOne(TableHashKey, keyBytes)
359358
if err != nil {
360359
return utils.NodeKey{}, err
361360
}
362361

363362
if data == nil {
364-
return utils.NodeKey{}, fmt.Errorf("hash key %x not found", keyConc.Bytes())
363+
return utils.NodeKey{}, fmt.Errorf("hash key %x not found", keyBytes)
365364
}
366365

367366
nv := big.NewInt(0).SetBytes(data)
@@ -422,7 +421,7 @@ func (m *EriRoDb) GetDb() map[string][]string {
422421

423422
allFirst8PaddedWithZeros := true
424423
for i := 0; i < 8; i++ {
425-
if !strings.HasPrefix(fmt.Sprintf("%016s", val[i].Text(16)), "00000000") {
424+
if !strings.HasPrefix(fmt.Sprintf("%016x", val[i]), "00000000") {
426425
allFirst8PaddedWithZeros = false
427426
break
428427
}
@@ -435,7 +434,7 @@ func (m *EriRoDb) GetDb() map[string][]string {
435434
outputArr := make([]string, truncationLength)
436435
for i := 0; i < truncationLength; i++ {
437436
if i < len(val) {
438-
outputArr[i] = fmt.Sprintf("%016s", val[i].Text(16))
437+
outputArr[i] = fmt.Sprintf("%016x", val[i])
439438
} else {
440439
outputArr[i] = "0000000000000000"
441440
}

‎smt/pkg/db/mdbx_test.go

+3-7
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package db
22

33
import (
44
"context"
5-
"math/big"
65
"testing"
76

87
"github.com/ledgerwatch/erigon-lib/kv/mdbx"
@@ -19,8 +18,7 @@ func TestEriDb(t *testing.T) {
1918

2019
// The key and value we're going to test
2120
key := utils.NodeKey{1, 2, 3, 4}
22-
value := utils.NodeValue12{big.NewInt(1), big.NewInt(2), big.NewInt(3), big.NewInt(4), big.NewInt(5), big.NewInt(6),
23-
big.NewInt(7), big.NewInt(8), big.NewInt(9), big.NewInt(10), big.NewInt(11), big.NewInt(12)}
21+
value := utils.NodeValue12{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}
2422

2523
// Testing Insert method
2624
err = db.Insert(key, value)
@@ -41,8 +39,7 @@ func TestEriDbBatch(t *testing.T) {
4139

4240
// The key and value we're going to test
4341
key := utils.NodeKey{1, 2, 3, 4}
44-
value := utils.NodeValue12{big.NewInt(1), big.NewInt(2), big.NewInt(3), big.NewInt(4), big.NewInt(5), big.NewInt(6),
45-
big.NewInt(7), big.NewInt(8), big.NewInt(9), big.NewInt(10), big.NewInt(11), big.NewInt(12)}
42+
value := utils.NodeValue12{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}
4643

4744
quit := make(chan struct{})
4845

@@ -67,8 +64,7 @@ func TestEriDbBatch(t *testing.T) {
6764

6865
// Inserting a different key-value pair within a batch
6966
altKey := utils.NodeKey{5, 6, 7, 8}
70-
altValue := utils.NodeValue12{big.NewInt(13), big.NewInt(14), big.NewInt(15), big.NewInt(16), big.NewInt(17), big.NewInt(18),
71-
big.NewInt(19), big.NewInt(20), big.NewInt(21), big.NewInt(22), big.NewInt(23), big.NewInt(24)}
67+
altValue := utils.NodeValue12{13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}
7268

7369
err = db.Insert(altKey, altValue)
7470
assert.NoError(t, err)

‎smt/pkg/db/mem-db.go

+28-30
Original file line numberDiff line numberDiff line change
@@ -81,13 +81,15 @@ func (m *MemDb) Get(key utils.NodeKey) (utils.NodeValue12, error) {
8181
m.lock.RLock() // Lock for reading
8282
defer m.lock.RUnlock() // Make sure to unlock when done
8383

84-
keyConc := utils.ArrayToScalar(key[:])
85-
86-
k := utils.ConvertBigIntToHex(keyConc)
84+
k := utils.ConvertArrayToHex(key[:])
8785

8886
values := utils.NodeValue12{}
8987
for i, v := range m.Db[k] {
90-
values[i] = utils.ConvertHexToBigInt(v)
88+
asUint64, err := utils.ConvertHexToUint64(v)
89+
if err != nil {
90+
return utils.NodeValue12{}, err
91+
}
92+
values[i] = asUint64
9193
}
9294

9395
return values, nil
@@ -97,12 +99,11 @@ func (m *MemDb) Insert(key utils.NodeKey, value utils.NodeValue12) error {
9799
m.lock.Lock() // Lock for writing
98100
defer m.lock.Unlock() // Make sure to unlock when done
99101

100-
keyConc := utils.ArrayToScalar(key[:])
101-
k := utils.ConvertBigIntToHex(keyConc)
102+
k := utils.ConvertArrayToHex(key[:])
102103

103104
values := make([]string, 12)
104105
for i, v := range value {
105-
values[i] = utils.ConvertBigIntToHex(v)
106+
values[i] = utils.ConvertUint64ToHex(v)
106107
}
107108

108109
m.Db[k] = values
@@ -121,8 +122,7 @@ func (m *MemDb) DeleteByNodeKey(key utils.NodeKey) error {
121122
m.lock.Lock() // Lock for writing
122123
defer m.lock.Unlock() // Make sure to unlock when done
123124

124-
keyConc := utils.ArrayToScalar(key[:])
125-
k := utils.ConvertBigIntToHex(keyConc)
125+
k := utils.ConvertArrayToHex(key[:])
126126
delete(m.Db, k)
127127
return nil
128128
}
@@ -131,13 +131,15 @@ func (m *MemDb) GetAccountValue(key utils.NodeKey) (utils.NodeValue8, error) {
131131
m.lock.RLock() // Lock for reading
132132
defer m.lock.RUnlock() // Make sure to unlock when done
133133

134-
keyConc := utils.ArrayToScalar(key[:])
135-
136-
k := utils.ConvertBigIntToHex(keyConc)
134+
k := utils.ConvertArrayToHex(key[:])
137135

138136
values := utils.NodeValue8{}
139137
for i, v := range m.DbAccVal[k] {
140-
values[i] = utils.ConvertHexToBigInt(v)
138+
asUint64, err := utils.ConvertHexToUint64(v)
139+
if err != nil {
140+
return utils.NodeValue8{}, err
141+
}
142+
values[i] = asUint64
141143
}
142144

143145
return values, nil
@@ -147,12 +149,11 @@ func (m *MemDb) InsertAccountValue(key utils.NodeKey, value utils.NodeValue8) er
147149
m.lock.Lock() // Lock for writing
148150
defer m.lock.Unlock() // Make sure to unlock when done
149151

150-
keyConc := utils.ArrayToScalar(key[:])
151-
k := utils.ConvertBigIntToHex(keyConc)
152+
k := utils.ConvertArrayToHex(key[:])
152153

153154
values := make([]string, 8)
154155
for i, v := range value {
155-
values[i] = utils.ConvertBigIntToHex(v)
156+
values[i] = utils.ConvertUint64ToHex(v)
156157
}
157158

158159
m.DbAccVal[k] = values
@@ -163,29 +164,29 @@ func (m *MemDb) InsertKeySource(key utils.NodeKey, value []byte) error {
163164
m.lock.Lock() // Lock for writing
164165
defer m.lock.Unlock() // Make sure to unlock when done
165166

166-
keyConc := utils.ArrayToScalar(key[:])
167+
k := utils.ConvertArrayToHex(key[:])
167168

168-
m.DbKeySource[keyConc.String()] = value
169+
m.DbKeySource[k] = value
169170
return nil
170171
}
171172

172173
func (m *MemDb) DeleteKeySource(key utils.NodeKey) error {
173174
m.lock.Lock() // Lock for writing
174175
defer m.lock.Unlock() // Make sure to unlock when done
175176

176-
keyConc := utils.ArrayToScalar(key[:])
177+
k := utils.ConvertArrayToHex(key[:])
177178

178-
delete(m.DbKeySource, keyConc.String())
179+
delete(m.DbKeySource, k)
179180
return nil
180181
}
181182

182183
func (m *MemDb) GetKeySource(key utils.NodeKey) ([]byte, error) {
183184
m.lock.RLock() // Lock for reading
184185
defer m.lock.RUnlock() // Make sure to unlock when done
185186

186-
keyConc := utils.ArrayToScalar(key[:])
187+
k := utils.ConvertArrayToHex(key[:])
187188

188-
s, ok := m.DbKeySource[keyConc.String()]
189+
s, ok := m.DbKeySource[k]
189190

190191
if !ok {
191192
return nil, ErrNotFound
@@ -198,21 +199,19 @@ func (m *MemDb) InsertHashKey(key utils.NodeKey, value utils.NodeKey) error {
198199
m.lock.Lock() // Lock for writing
199200
defer m.lock.Unlock() // Make sure to unlock when done
200201

201-
keyConc := utils.ArrayToScalar(key[:])
202-
k := utils.ConvertBigIntToHex(keyConc)
202+
k := utils.ConvertArrayToHex(key[:])
203203

204-
valConc := utils.ArrayToScalar(value[:])
204+
valBytes := utils.ArrayToBytes(value[:])
205205

206-
m.DbHashKey[k] = valConc.Bytes()
206+
m.DbHashKey[k] = valBytes
207207
return nil
208208
}
209209

210210
func (m *MemDb) DeleteHashKey(key utils.NodeKey) error {
211211
m.lock.Lock() // Lock for writing
212212
defer m.lock.Unlock() // Make sure to unlock when done
213213

214-
keyConc := utils.ArrayToScalar(key[:])
215-
k := utils.ConvertBigIntToHex(keyConc)
214+
k := utils.ConvertArrayToHex(key[:])
216215

217216
delete(m.DbHashKey, k)
218217
return nil
@@ -222,8 +221,7 @@ func (m *MemDb) GetHashKey(key utils.NodeKey) (utils.NodeKey, error) {
222221
m.lock.RLock() // Lock for reading
223222
defer m.lock.RUnlock() // Make sure to unlock when done
224223

225-
keyConc := utils.ArrayToScalar(key[:])
226-
k := utils.ConvertBigIntToHex(keyConc)
224+
k := utils.ConvertArrayToHex(key[:])
227225

228226
s, ok := m.DbHashKey[k]
229227

‎smt/pkg/db/mem-db_test.go

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package db
22

33
import (
4-
"math/big"
54
"testing"
65

76
"github.com/stretchr/testify/assert"
@@ -14,8 +13,7 @@ func TestMemDb(t *testing.T) {
1413

1514
// The key and value we're going to test
1615
key := utils.NodeKey{1, 2, 3, 4}
17-
value := utils.NodeValue12{big.NewInt(1), big.NewInt(2), big.NewInt(3), big.NewInt(4), big.NewInt(5), big.NewInt(6),
18-
big.NewInt(7), big.NewInt(8), big.NewInt(9), big.NewInt(10), big.NewInt(11), big.NewInt(12)}
16+
value := utils.NodeValue12{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}
1917

2018
// Testing Insert method
2119
err := db.Insert(key, value)

0 commit comments

Comments
 (0)