tbcd: move block header cache from items to size (#360)

* back this up

* Backup cache purge

* Joshua comments

* Purge cache in batches instead of taking the mutex all the time

* One more Joshua comment

* nominally working of block cache minus eviction

* Remove unused Purge

* Add LRU mechanics

* Add tests

* Fix insert of element

* Test misses and purges as well

* Limit cache to 1gb by default and rig up prometheus

* Derp disable promPollVerbose

* Fix stats races (thanks joshua) and add stats to header cache as well

* Test header cache as well and make checks more readable

* added extra basic test for headercache

* fix naming, form joshua

* Fixup of New in antonio code

---------

Co-authored-by: AL-CT <[email protected]>

Author: marcopeereboom

cmd/tbcd/tbcd.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2024 Hemi Labs, Inc.
+// Copyright (c) 2024-2025 Hemi Labs, Inc.
 // Use of this source code is governed by the MIT License,
 // which can be found in the LICENSE file.
 
@@ -25,7 +25,8 @@ const (
 	defaultLogLevel = daemonName + "=INFO;tbc=INFO;level=INFO"
 	defaultNetwork  = "testnet3" // XXX make this mainnet
 	defaultHome     = "~/." + daemonName
-	bhsDefault      = int(1e6) // enough for mainnet
+	bDefaultSize    = "1gb"   // ~640 blocks on mainnet
+	bhsDefaultSize  = "128mb" // enough for mainnet
 )
 
 var (
@@ -46,16 +47,16 @@ var (
 			Help:         "enable auto utxo and tx indexes",
 			Print:        config.PrintAll,
 		},
-		"TBC_BLOCK_CACHE": config.Config{
-			Value:        &cfg.BlockCache,
-			DefaultValue: 250,
-			Help:         "number of cached blocks",
+		"TBC_BLOCK_CACHE_SIZE": config.Config{
+			Value:        &cfg.BlockCacheSize,
+			DefaultValue: bDefaultSize,
+			Help:         "size of block cache",
 			Print:        config.PrintAll,
 		},
-		"TBC_BLOCKHEADER_CACHE": config.Config{
-			Value:        &cfg.BlockheaderCache,
-			DefaultValue: bhsDefault,
-			Help:         "number of cached blockheaders",
+		"TBC_BLOCKHEADER_CACHE_SIZE": config.Config{
+			Value:        &cfg.BlockheaderCacheSize,
+			DefaultValue: bhsDefaultSize,
+			Help:         "size of blockheader cache",
 			Print:        config.PrintAll,
 		},
 		"TBC_BLOCK_SANITY": config.Config{

database/tbcd/database.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2024 Hemi Labs, Inc.
+// Copyright (c) 2024-2025 Hemi Labs, Inc.
 // Use of this source code is governed by the MIT License,
 // which can be found in the LICENSE file.
 
@@ -91,6 +91,7 @@ type Database interface {
 	BlockHeaderBest(ctx context.Context) (*BlockHeader, error) // return canonical
 	BlockHeaderByHash(ctx context.Context, hash *chainhash.Hash) (*BlockHeader, error)
 	BlockHeaderGenesisInsert(ctx context.Context, wbh *wire.BlockHeader, height uint64, diff *big.Int) error
+	BlockHeaderCacheStats() CacheStats
 
 	// Block headers
 	BlockHeadersByHeight(ctx context.Context, height uint64) ([]BlockHeader, error)
@@ -103,6 +104,7 @@ type Database interface {
 	BlockInsert(ctx context.Context, b *btcutil.Block) (int64, error)
 	// BlocksInsert(ctx context.Context, bs []*btcutil.Block) (int64, error)
 	BlockByHash(ctx context.Context, hash *chainhash.Hash) (*btcutil.Block, error)
+	BlockCacheStats() CacheStats
 
 	// Transactions
 	BlockUtxoUpdate(ctx context.Context, direction int, utxos map[Outpoint]CacheOutput) error
@@ -413,3 +415,12 @@ func TxIdBlockHashFromTxKey(txKey TxKey) (*chainhash.Hash, *chainhash.Hash, erro
 	}
 	return txId, blockHash, nil
 }
+
+// Cache
+type CacheStats struct {
+	Hits   int
+	Misses int
+	Purges int
+	Size   int
+	Items  int
+}

database/tbcd/level/blockcache.go

@@ -0,0 +1,117 @@
+// Copyright (c) 2025 Hemi Labs, Inc.
+// Use of this source code is governed by the MIT License,
+// which can be found in the LICENSE file.
+
+package level
+
+import (
+	"container/list"
+	"fmt"
+	"sync"
+
+	"github.com/btcsuite/btcd/btcutil"
+	"github.com/btcsuite/btcd/chaincfg/chainhash"
+
+	"github.com/hemilabs/heminetwork/database/tbcd"
+)
+
+var blockSize = 1677721 // ~1.6MB rough size of a mainnet block as of Jan 2025
+
+type blockElement struct {
+	element *list.Element
+	block   []byte
+}
+
+type lowIQLRU struct {
+	mtx sync.Mutex
+
+	size int // this is the approximate max size
+
+	m         map[chainhash.Hash]blockElement
+	totalSize int
+
+	// lru list, when used move to back of the list
+	l *list.List
+
+	// stats
+	c tbcd.CacheStats
+}
+
+func (l *lowIQLRU) Put(v *btcutil.Block) {
+	l.mtx.Lock()
+	defer l.mtx.Unlock()
+
+	hash := v.Hash()
+	if _, ok := l.m[*hash]; ok {
+		return
+	}
+
+	block, err := v.Bytes()
+	if err != nil {
+		// data corruption, panic
+		panic(err)
+	}
+
+	// evict first element in list
+	if l.totalSize+len(block) > l.size {
+		// LET THEM EAT PANIC
+		re := l.l.Front()
+		rha := l.l.Remove(re)
+		rh := rha.(*chainhash.Hash)
+		l.totalSize -= len(l.m[*rh].block)
+		delete(l.m, *rh)
+		l.c.Purges++
+	}
+
+	// block lookup and lru append
+	l.m[*hash] = blockElement{element: l.l.PushBack(hash), block: block}
+	l.totalSize += len(block)
+
+	l.c.Size = l.totalSize
+}
+
+func (l *lowIQLRU) Get(k *chainhash.Hash) (*btcutil.Block, bool) {
+	l.mtx.Lock()
+	defer l.mtx.Unlock()
+
+	be, ok := l.m[*k]
+	if !ok {
+		l.c.Misses++
+		return nil, false
+	}
+	b, err := btcutil.NewBlockFromBytes(be.block)
+	if err != nil {
+		// panic for diagnostics at this time
+		panic(err)
+	}
+
+	// update access
+	l.l.MoveToBack(be.element)
+
+	l.c.Hits++
+
+	return b, true
+}
+
+func (l *lowIQLRU) Stats() tbcd.CacheStats {
+	l.mtx.Lock()
+	defer l.mtx.Unlock()
+	l.c.Items = len(l.m)
+	return l.c
+}
+
+func lowIQLRUSizeNew(size int) (*lowIQLRU, error) {
+	if size <= 0 {
+		return nil, fmt.Errorf("invalid size: %v", size)
+	}
+	// approximate number of blocks
+	count := size / blockSize
+	if count <= 0 {
+		return nil, fmt.Errorf("invalid count: %v", count)
+	}
+	return &lowIQLRU{
+		size: size,
+		m:    make(map[chainhash.Hash]blockElement, count),
+		l:    list.New(),
+	}, nil
+}