Skip to content

core/rawdb: integrate eradb backend for RPC #31604

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 27 commits into
base: master
Choose a base branch
from
Open

core/rawdb: integrate eradb backend for RPC #31604

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
e264697
core/rawdb: scaffold era database
s1na Apr 2, 2025
923979a
use glob instead of scanning
s1na Apr 3, 2025
24b53d8
Merge branch 'master' into eradb
s1na Apr 8, 2025
f9414ab
import errors
s1na Apr 8, 2025
d9bbfff
use cache, raw getters
s1na Apr 9, 2025
94d0129
era raw getters
s1na Apr 9, 2025
0b9fa5c
freezer eradb instance
s1na Apr 9, 2025
9ab2492
glob for network
s1na Apr 10, 2025
d22e7fe
instantiate eradb outside of freezer
s1na Apr 10, 2025
537542a
rm era object accessors
s1na Apr 10, 2025
bce14d9
clean up
s1na Apr 10, 2025
3251f31
const for open file limit
s1na Apr 10, 2025
ff91834
fix raw body
s1na Apr 10, 2025
6886656
fix test
s1na Apr 10, 2025
42f8b42
fix lint
s1na Apr 11, 2025
5e01475
add flag for eradb
s1na Apr 14, 2025
d1ca170
close evicted era files
s1na Apr 15, 2025
b293a3b
rename resolveEra
s1na Apr 16, 2025
63fe201
fix onEvicted
s1na Apr 16, 2025
03182b5
close OnReplaced
s1na Apr 16, 2025
77ed745
add comment
s1na Apr 16, 2025
2f6fc26
resolve merge conflict
s1na Apr 17, 2025
1e84060
fix lint
s1na Apr 17, 2025
4a4eef2
core/rawdb: move erdb integration into chain freezer
rjl493456442 Apr 18, 2025
3bac92e
core/rawdb, node: fix era path resolution
rjl493456442 Apr 18, 2025
589fc65
core/rawdb/eradb: slim down the testdata
rjl493456442 Apr 18, 2025
e6d5806
eth/ethconfig: fix config gen
rjl493456442 Apr 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/geth/chaincmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ func initGenesis(ctx *cli.Context) error {
overrides.OverrideVerkle = &v
}

chaindb, err := stack.OpenDatabaseWithFreezer("chaindata", 0, 0, ctx.String(utils.AncientFlag.Name), "", false)
chaindb, err := stack.OpenDatabaseWithFreezer("chaindata", 0, 0, ctx.String(utils.AncientFlag.Name), "", false, "")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should probably make this a config struct

if err != nil {
utils.Fatalf("Failed to open database: %v", err)
}
Expand Down
11 changes: 10 additions & 1 deletion cmd/utils/flags.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,11 @@ var (
Usage: "Root directory for ancient data (default = inside chaindata)",
Category: flags.EthCategory,
}
EraFlag = &flags.DirectoryFlag{
Name: "datadir.era",
Usage: "Root directory for era1 history (default = inside ancient/chain)",
Category: flags.EthCategory,
}
MinFreeDiskSpaceFlag = &flags.DirectoryFlag{
Name: "datadir.minfreedisk",
Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)",
Expand Down Expand Up @@ -977,6 +982,7 @@ var (
DatabaseFlags = []cli.Flag{
DataDirFlag,
AncientFlag,
EraFlag,
RemoteDBFlag,
DBEngineFlag,
StateSchemeFlag,
Expand Down Expand Up @@ -1630,6 +1636,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
if ctx.IsSet(AncientFlag.Name) {
cfg.DatabaseFreezer = ctx.String(AncientFlag.Name)
}
if ctx.IsSet(EraFlag.Name) {
cfg.DatabaseEra = ctx.String(EraFlag.Name)
}

if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
Expand Down Expand Up @@ -2100,7 +2109,7 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.
}
chainDb = remotedb.New(client)
default:
chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "eth/db/chaindata/", readonly)
chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "eth/db/chaindata/", readonly, ctx.String(EraFlag.Name))
}
if err != nil {
Fatalf("Could not open database: %v", err)
Expand Down
2 changes: 1 addition & 1 deletion cmd/utils/history_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ func TestHistoryImportAndExport(t *testing.T) {
}

// Now import Era.
db2, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
db2, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
if err != nil {
panic(err)
}
Expand Down
25 changes: 22 additions & 3 deletions common/lru/basiclru.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,11 @@ package lru
// This type is not safe for concurrent use.
// The zero value is not valid, instances must be created using NewCache.
type BasicLRU[K comparable, V any] struct {
list *list[K]
items map[K]cacheItem[K, V]
cap int
list *list[K]
items map[K]cacheItem[K, V]
cap int
onEvicted func(key K, value V)
onReplaced func(key K, value V)
}

type cacheItem[K any, V any] struct {
Expand All @@ -50,6 +52,9 @@ func (c *BasicLRU[K, V]) Add(key K, value V) (evicted bool) {
item, ok := c.items[key]
if ok {
// Already exists in cache.
if c.onReplaced != nil {
c.onReplaced(key, item.value)
}
item.value = value
c.items[key] = item
c.list.moveToFront(item.elem)
Expand All @@ -59,6 +64,10 @@ func (c *BasicLRU[K, V]) Add(key K, value V) (evicted bool) {
var elem *listElem[K]
if c.Len() >= c.cap {
elem = c.list.removeLast()
if c.onEvicted != nil {
v := c.items[elem.v]
c.onEvicted(elem.v, v.value)
}
delete(c.items, elem.v)
evicted = true
} else {
Expand Down Expand Up @@ -142,6 +151,16 @@ func (c *BasicLRU[K, V]) RemoveOldest() (key K, value V, ok bool) {
return key, item.value, true
}

// OnEvicted sets a callback function to be called when an item is evicted from the cache.
func (c *BasicLRU[K, V]) OnEvicted(fn func(k K, v V)) {
c.onEvicted = fn
}

// OnReplaced sets a callback function to be called when an item is replaced in the cache.
func (c *BasicLRU[K, V]) OnReplaced(fn func(k K, v V)) {
c.onReplaced = fn
}

// Keys returns all keys in the cache.
func (c *BasicLRU[K, V]) Keys() []K {
keys := make([]K, 0, len(c.items))
Expand Down
16 changes: 16 additions & 0 deletions common/lru/lru.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,3 +93,19 @@ func (c *Cache[K, V]) Keys() []K {

return c.cache.Keys()
}

// OnEvicted sets a callback function to be called when an item is evicted from the cache.
func (c *Cache[K, V]) OnEvicted(fn func(key K, value V)) {
c.mu.Lock()
defer c.mu.Unlock()

c.cache.OnEvicted(fn)
}

// OnReplaced sets a callback function to be called when an item is replaced in the cache.
func (c *Cache[K, V]) OnReplaced(fn func(key K, value V)) {
c.mu.Lock()
defer c.mu.Unlock()

c.cache.OnReplaced(fn)
}
8 changes: 4 additions & 4 deletions core/blockchain_repair_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1769,7 +1769,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false, "")
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
Expand Down Expand Up @@ -1854,7 +1854,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
if err != nil {
t.Fatalf("Failed to reopen persistent key-value database: %v", err)
}
db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false, "")
if err != nil {
t.Fatalf("Failed to reopen persistent freezer database: %v", err)
}
Expand Down Expand Up @@ -1919,7 +1919,7 @@ func testIssue23496(t *testing.T, scheme string) {
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false, "")
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
Expand Down Expand Up @@ -1977,7 +1977,7 @@ func testIssue23496(t *testing.T, scheme string) {
if err != nil {
t.Fatalf("Failed to reopen persistent key-value database: %v", err)
}
db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false, "")
if err != nil {
t.Fatalf("Failed to reopen persistent freezer database: %v", err)
}
Expand Down
2 changes: 1 addition & 1 deletion core/blockchain_sethead_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1973,7 +1973,7 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false, "")
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
Expand Down
4 changes: 2 additions & 2 deletions core/blockchain_snapshot_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false, "")
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
Expand Down Expand Up @@ -261,7 +261,7 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) {
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
newdb, err := rawdb.NewDatabaseWithFreezer(pdb, snaptest.ancient, "", false)
newdb, err := rawdb.NewDatabaseWithFreezer(pdb, snaptest.ancient, "", false, "")
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
Expand Down
22 changes: 11 additions & 11 deletions core/blockchain_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -738,7 +738,7 @@ func testFastVsFullChains(t *testing.T, scheme string) {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
// Freezer style fast import the chain.
ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
if err != nil {
t.Fatalf("failed to create temp freezer db: %v", err)
}
Expand Down Expand Up @@ -824,7 +824,7 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) {

// makeDb creates a db instance for testing.
makeDb := func() ethdb.Database {
db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
if err != nil {
t.Fatalf("failed to create temp freezer db: %v", err)
}
Expand Down Expand Up @@ -1623,7 +1623,7 @@ func testLargeReorgTrieGC(t *testing.T, scheme string) {
competitor, _ := GenerateChain(genesis.Config, shared[len(shared)-1], engine, genDb, 2*state.TriesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) })

// Import the shared chain and the original canonical one
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
defer db.Close()

chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil)
Expand Down Expand Up @@ -1689,7 +1689,7 @@ func testBlockchainRecovery(t *testing.T, scheme string) {
_, blocks, receipts := GenerateChainWithGenesis(gspec, ethash.NewFaker(), int(height), nil)

// Import the chain as a ancient-first node and ensure all pointers are updated
ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false)
ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, "")
if err != nil {
t.Fatalf("failed to create temp freezer db: %v", err)
}
Expand Down Expand Up @@ -1747,7 +1747,7 @@ func testLowDiffLongChain(t *testing.T, scheme string) {
})

// Import the canonical chain
diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
defer diskdb.Close()

chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil)
Expand Down Expand Up @@ -1959,7 +1959,7 @@ func testInsertKnownChainData(t *testing.T, typ string, scheme string) {
b.OffsetTime(-9) // A higher difficulty
})
// Import the shared chain and the original canonical one
chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
if err != nil {
t.Fatalf("failed to create temp freezer db: %v", err)
}
Expand Down Expand Up @@ -2122,7 +2122,7 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i
}
})
// Import the shared chain and the original canonical one
chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
if err != nil {
t.Fatalf("failed to create temp freezer db: %v", err)
}
Expand Down Expand Up @@ -2496,7 +2496,7 @@ func testSideImportPrunedBlocks(t *testing.T, scheme string) {
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false, "")
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
Expand Down Expand Up @@ -3403,7 +3403,7 @@ func testSetCanonical(t *testing.T, scheme string) {
}
gen.AddTx(tx)
})
diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
defer diskdb.Close()

chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil)
Expand Down Expand Up @@ -4199,7 +4199,7 @@ func testChainReorgSnapSync(t *testing.T, ancientLimit uint64) {
gen.SetCoinbase(common.Address{0: byte(0xb), 19: byte(i)})
})

db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
defer db.Close()

chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil)
Expand Down Expand Up @@ -4315,7 +4315,7 @@ func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64,
config := DefaultCacheConfigWithScheme(rawdb.PathScheme)
config.ChainHistoryMode = history.KeepPostMerge

db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
defer db.Close()
chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), genesis, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil)
defer chain.Stop()
Expand Down
8 changes: 4 additions & 4 deletions core/rawdb/accessors_chain_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -412,7 +412,7 @@ func checkReceiptsRLP(have, want types.Receipts) error {
func TestAncientStorage(t *testing.T) {
// Freezer style fast import the chain.
frdir := t.TempDir()
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false)
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, "")
if err != nil {
t.Fatalf("failed to create database with ancient backend")
}
Expand Down Expand Up @@ -465,7 +465,7 @@ func TestAncientStorage(t *testing.T) {
}

func TestWriteAncientHeaderChain(t *testing.T) {
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), t.TempDir(), "", false)
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), t.TempDir(), "", false, "")
if err != nil {
t.Fatalf("failed to create database with ancient backend")
}
Expand Down Expand Up @@ -582,7 +582,7 @@ func TestHashesInRange(t *testing.T) {
func BenchmarkWriteAncientBlocks(b *testing.B) {
// Open freezer database.
frdir := b.TempDir()
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false)
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, "")
if err != nil {
b.Fatalf("failed to create database with ancient backend")
}
Expand Down Expand Up @@ -886,7 +886,7 @@ func TestHeadersRLPStorage(t *testing.T) {
// Have N headers in the freezer
frdir := t.TempDir()

db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false)
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, "")
if err != nil {
t.Fatalf("failed to create database with ancient backend")
}
Expand Down
Loading