Skip to content

cmd, core, trie: verkle-capable geth init #28270

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Nov 14, 2023
Merged

cmd, core, trie: verkle-capable geth init #28270

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
3249aa6
cmd, core, trie: verkle-capable `geth init`
gballet Oct 6, 2023
9a95be4
review feedback + root node resolution in OpenTrie
gballet Oct 9, 2023
30daf0a
more review feedback
gballet Oct 9, 2023
6bd9981
add a trie parameter to OpenStorageTrie
gballet Oct 9, 2023
a02c50c
review feedback + add test
gballet Oct 19, 2023
40ebb2b
adding some (failing) tests
gballet Oct 19, 2023
412075b
core, trie: fix failed test
rjl493456442 Oct 20, 2023
d326ba1
core, light, trie: polish code (#303)
rjl493456442 Oct 31, 2023
8754ebc
update go-verkle
gballet Oct 31, 2023
1379079
trie/utils: address comment from martin
rjl493456442 Nov 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cmd/geth/chaincmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ func initGenesis(ctx *cli.Context) error {
}
defer chaindb.Close()

triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false)
triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false, genesis.IsVerkle())
defer triedb.Close()

_, hash, err := core.SetupGenesisBlockWithOverride(chaindb, triedb, genesis, &overrides)
Expand Down Expand Up @@ -485,7 +485,7 @@ func dump(ctx *cli.Context) error {
if err != nil {
return err
}
triedb := utils.MakeTrieDatabase(ctx, db, true, true) // always enable preimage lookup
triedb := utils.MakeTrieDatabase(ctx, db, true, true, false) // always enable preimage lookup
defer triedb.Close()

state, err := state.New(root, state.NewDatabaseWithNodeDB(db, triedb), nil)
Expand Down
2 changes: 1 addition & 1 deletion cmd/geth/dbcmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,7 @@ func dbDumpTrie(ctx *cli.Context) error {
db := utils.MakeChainDatabase(ctx, stack, true)
defer db.Close()

triedb := utils.MakeTrieDatabase(ctx, db, false, true)
triedb := utils.MakeTrieDatabase(ctx, db, false, true, false)
defer triedb.Close()

var (
Expand Down
8 changes: 4 additions & 4 deletions cmd/geth/snapshot.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ func verifyState(ctx *cli.Context) error {
log.Error("Failed to load head block")
return errors.New("no head block")
}
triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true)
triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true, false)
defer triedb.Close()

snapConfig := snapshot.Config{
Expand Down Expand Up @@ -260,7 +260,7 @@ func traverseState(ctx *cli.Context) error {
chaindb := utils.MakeChainDatabase(ctx, stack, true)
defer chaindb.Close()

triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true)
triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true, false)
defer triedb.Close()

headBlock := rawdb.ReadHeadBlock(chaindb)
Expand Down Expand Up @@ -369,7 +369,7 @@ func traverseRawState(ctx *cli.Context) error {
chaindb := utils.MakeChainDatabase(ctx, stack, true)
defer chaindb.Close()

triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true)
triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true, false)
defer triedb.Close()

headBlock := rawdb.ReadHeadBlock(chaindb)
Expand Down Expand Up @@ -533,7 +533,7 @@ func dumpState(ctx *cli.Context) error {
if err != nil {
return err
}
triedb := utils.MakeTrieDatabase(ctx, db, false, true)
triedb := utils.MakeTrieDatabase(ctx, db, false, true, false)
defer triedb.Close()

snapConfig := snapshot.Config{
Expand Down
6 changes: 3 additions & 3 deletions cmd/geth/verkle.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ func checkChildren(root verkle.VerkleNode, resolver verkle.NodeResolverFn) error
return fmt.Errorf("could not find child %x in db: %w", childC, err)
}
// depth is set to 0, the tree isn't rebuilt so it's not a problem
childN, err := verkle.ParseNode(childS, 0, childC[:])
childN, err := verkle.ParseNode(childS, 0)
if err != nil {
return fmt.Errorf("decode error child %x in db: %w", child.Commitment().Bytes(), err)
}
Expand Down Expand Up @@ -145,7 +145,7 @@ func verifyVerkle(ctx *cli.Context) error {
if err != nil {
return err
}
root, err := verkle.ParseNode(serializedRoot, 0, rootC[:])
root, err := verkle.ParseNode(serializedRoot, 0)
if err != nil {
return err
}
Expand Down Expand Up @@ -195,7 +195,7 @@ func expandVerkle(ctx *cli.Context) error {
if err != nil {
return err
}
root, err := verkle.ParseNode(serializedRoot, 0, rootC[:])
root, err := verkle.ParseNode(serializedRoot, 0)
if err != nil {
return err
}
Expand Down
3 changes: 2 additions & 1 deletion cmd/utils/flags.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2213,9 +2213,10 @@ func MakeConsolePreloads(ctx *cli.Context) []string {
}

// MakeTrieDatabase constructs a trie database based on the configured scheme.
func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, readOnly bool) *trie.Database {
func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, readOnly bool, isVerkle bool) *trie.Database {
config := &trie.Config{
Preimages: preimage,
IsVerkle: isVerkle,
}
scheme, err := rawdb.ParseStateScheme(ctx.String(StateSchemeFlag.Name), disk)
if err != nil {
Expand Down
23 changes: 20 additions & 3 deletions core/genesis.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import (
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
)

//go:generate go run github.com/fjl/gencodec -type Genesis -field-override genesisSpecMarshaling -out gen_genesis.go
Expand Down Expand Up @@ -121,10 +122,20 @@ func (ga *GenesisAlloc) UnmarshalJSON(data []byte) error {
}

// hash computes the state root according to the genesis specification.
func (ga *GenesisAlloc) hash() (common.Hash, error) {
func (ga *GenesisAlloc) hash(isVerkle bool) (common.Hash, error) {
// If a genesis-time verkle trie is requested, create a trie config
// with the verkle trie enabled so that the tree can be initialized
// as such.
var config *trie.Config
if isVerkle {
config = &trie.Config{
PathDB: pathdb.Defaults,
IsVerkle: true,
}
}
// Create an ephemeral in-memory database for computing hash,
// all the derived states will be discarded to not pollute disk.
db := state.NewDatabase(rawdb.NewMemoryDatabase())
db := state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), config)
statedb, err := state.New(types.EmptyRootHash, db, nil)
if err != nil {
return common.Hash{}, err
Expand Down Expand Up @@ -410,9 +421,15 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig {
}
}

// IsVerkle indicates whether the state is already stored in a verkle
// tree at genesis time.
func (g *Genesis) IsVerkle() bool {
return g.Config.IsVerkle(new(big.Int).SetUint64(g.Number), g.Timestamp)
}

// ToBlock returns the genesis block according to genesis specification.
func (g *Genesis) ToBlock() *types.Block {
root, err := g.Alloc.hash()
root, err := g.Alloc.hash(g.IsVerkle())
if err != nil {
panic(err)
}
Expand Down
66 changes: 65 additions & 1 deletion core/genesis_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package core

import (
"bytes"
"encoding/json"
"math/big"
"reflect"
Expand Down Expand Up @@ -231,7 +232,7 @@ func TestReadWriteGenesisAlloc(t *testing.T) {
{1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}},
{2}: {Balance: big.NewInt(2), Storage: map[common.Hash]common.Hash{{2}: {2}}},
}
hash, _ = alloc.hash()
hash, _ = alloc.hash(false)
)
blob, _ := json.Marshal(alloc)
rawdb.WriteGenesisStateSpec(db, hash, blob)
Expand Down Expand Up @@ -261,3 +262,66 @@ func newDbConfig(scheme string) *trie.Config {
}
return &trie.Config{PathDB: pathdb.Defaults}
}

func TestVerkleGenesisCommit(t *testing.T) {
var verkleTime uint64 = 0
verkleConfig := &params.ChainConfig{
ChainID: big.NewInt(1),
HomesteadBlock: big.NewInt(0),
DAOForkBlock: nil,
DAOForkSupport: false,
EIP150Block: big.NewInt(0),
EIP155Block: big.NewInt(0),
EIP158Block: big.NewInt(0),
ByzantiumBlock: big.NewInt(0),
ConstantinopleBlock: big.NewInt(0),
PetersburgBlock: big.NewInt(0),
IstanbulBlock: big.NewInt(0),
MuirGlacierBlock: big.NewInt(0),
BerlinBlock: big.NewInt(0),
LondonBlock: big.NewInt(0),
ArrowGlacierBlock: big.NewInt(0),
GrayGlacierBlock: big.NewInt(0),
MergeNetsplitBlock: nil,
ShanghaiTime: &verkleTime,
CancunTime: &verkleTime,
PragueTime: &verkleTime,
VerkleTime: &verkleTime,
TerminalTotalDifficulty: big.NewInt(0),
TerminalTotalDifficultyPassed: true,
Ethash: nil,
Clique: nil,
}

genesis := &Genesis{
BaseFee: big.NewInt(params.InitialBaseFee),
Config: verkleConfig,
Timestamp: verkleTime,
Difficulty: big.NewInt(0),
Alloc: GenesisAlloc{
{1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}},
},
}

expected := common.Hex2Bytes("14398d42be3394ff8d50681816a4b7bf8d8283306f577faba2d5bc57498de23b")
got := genesis.ToBlock().Root().Bytes()
if !bytes.Equal(got, expected) {
t.Fatalf("invalid genesis state root, expected %x, got %x", expected, got)
}

db := rawdb.NewMemoryDatabase()
triedb := trie.NewDatabase(db, &trie.Config{IsVerkle: true, PathDB: pathdb.Defaults})
block := genesis.MustCommit(db, triedb)
if !bytes.Equal(block.Root().Bytes(), expected) {
t.Fatalf("invalid genesis state root, expected %x, got %x", expected, got)
}

// Test that the trie is verkle
if !triedb.IsVerkle() {
t.Fatalf("expected trie to be verkle")
}

if !rawdb.ExistsAccountTrieNode(db, nil) {
t.Fatal("could not find node")
}
}
51 changes: 34 additions & 17 deletions core/state/database.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"errors"
"fmt"

"github.com/crate-crypto/go-ipa/banderwagon"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/lru"
"github.com/ethereum/go-ethereum/core/rawdb"
Expand All @@ -28,6 +29,7 @@ import (
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/trienode"
"github.com/ethereum/go-ethereum/trie/utils"
)

const (
Expand All @@ -36,6 +38,12 @@ const (

// Cache size granted for caching clean code.
codeCacheSize = 64 * 1024 * 1024

// commitmentSize is the size of commitment stored in cache.
commitmentSize = banderwagon.UncompressedSize

// Cache item granted for caching commitment results.
commitmentCacheItems = 64 * 1024 * 1024 / (commitmentSize + common.AddressLength)
)

// Database wraps access to tries and contract code.
Expand All @@ -44,7 +52,7 @@ type Database interface {
OpenTrie(root common.Hash) (Trie, error)

// OpenStorageTrie opens the storage trie of an account.
OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash) (Trie, error)
OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, trie Trie) (Trie, error)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rjl493456442 I added this trie parameter, which is a reference to the account trie. Since there is only one verkle tree, this is the trick that I use to make sure that storage writes also go to the main tree.


// CopyTrie returns an independent copy of the given trie.
CopyTrie(Trie) Trie
Expand All @@ -70,11 +78,6 @@ type Trie interface {
// TODO(fjl): remove this when StateTrie is removed
GetKey([]byte) []byte

// GetStorage returns the value for key stored in the trie. The value bytes
// must not be modified by the caller. If a node was not found in the database,
// a trie.MissingNodeError is returned.
GetStorage(addr common.Address, key []byte) ([]byte, error)

// GetAccount abstracts an account read from the trie. It retrieves the
// account blob from the trie with provided account address and decodes it
// with associated decoding algorithm. If the specified account is not in
Expand All @@ -83,27 +86,32 @@ type Trie interface {
// be returned.
GetAccount(address common.Address) (*types.StateAccount, error)

// UpdateStorage associates key with value in the trie. If value has length zero,
// any existing value is deleted from the trie. The value bytes must not be modified
// by the caller while they are stored in the trie. If a node was not found in the
// database, a trie.MissingNodeError is returned.
UpdateStorage(addr common.Address, key, value []byte) error
// GetStorage returns the value for key stored in the trie. The value bytes
// must not be modified by the caller. If a node was not found in the database,
// a trie.MissingNodeError is returned.
GetStorage(addr common.Address, key []byte) ([]byte, error)

// UpdateAccount abstracts an account write to the trie. It encodes the
// provided account object with associated algorithm and then updates it
// in the trie with provided address.
UpdateAccount(address common.Address, account *types.StateAccount) error

// UpdateContractCode abstracts code write to the trie. It is expected
// to be moved to the stateWriter interface when the latter is ready.
UpdateContractCode(address common.Address, codeHash common.Hash, code []byte) error
// UpdateStorage associates key with value in the trie. If value has length zero,
// any existing value is deleted from the trie. The value bytes must not be modified
// by the caller while they are stored in the trie. If a node was not found in the
// database, a trie.MissingNodeError is returned.
UpdateStorage(addr common.Address, key, value []byte) error

// DeleteAccount abstracts an account deletion from the trie.
DeleteAccount(address common.Address) error

// DeleteStorage removes any existing value for key from the trie. If a node
// was not found in the database, a trie.MissingNodeError is returned.
DeleteStorage(addr common.Address, key []byte) error

// DeleteAccount abstracts an account deletion from the trie.
DeleteAccount(address common.Address) error
// UpdateContractCode abstracts code write to the trie. It is expected
// to be moved to the stateWriter interface when the latter is ready.
UpdateContractCode(address common.Address, codeHash common.Hash, code []byte) error

// Hash returns the root hash of the trie. It does not write to the database and
// can be used even if the trie doesn't have one.
Expand Down Expand Up @@ -170,6 +178,9 @@ type cachingDB struct {

// OpenTrie opens the main account trie at a specific root hash.
func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
if db.triedb.IsVerkle() {
return trie.NewVerkleTrie(root, db.triedb, utils.NewPointCache(commitmentCacheItems))
}
tr, err := trie.NewStateTrie(trie.StateTrieID(root), db.triedb)
if err != nil {
return nil, err
Expand All @@ -178,7 +189,13 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
}

// OpenStorageTrie opens the storage trie of an account.
func (db *cachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash) (Trie, error) {
func (db *cachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, self Trie) (Trie, error) {
// In the verkle case, there is only one tree. But the two-tree structure
// is hardcoded in the codebase. So we need to return the same trie in this
// case.
if db.triedb.IsVerkle() {
return self, nil
}
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

see comment above: returning the unique verkle tree in that case.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What if the passed trie is nil?

tr, err := trie.NewStateTrie(trie.StorageTrieID(stateRoot, crypto.Keccak256Hash(address.Bytes()), root), db.triedb)
if err != nil {
return nil, err
Expand Down
2 changes: 1 addition & 1 deletion core/state/iterator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ func (it *nodeIterator) step() error {
address := common.BytesToAddress(preimage)

// Traverse the storage slots belong to the account
dataTrie, err := it.state.db.OpenStorageTrie(it.state.originalRoot, address, account.Root)
dataTrie, err := it.state.db.OpenStorageTrie(it.state.originalRoot, address, account.Root, it.state.trie)
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion core/state/state_object.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ func (s *stateObject) getTrie() (Trie, error) {
s.trie = s.db.prefetcher.trie(s.addrHash, s.data.Root)
}
if s.trie == nil {
tr, err := s.db.db.OpenStorageTrie(s.db.originalRoot, s.address, s.data.Root)
tr, err := s.db.db.OpenStorageTrie(s.db.originalRoot, s.address, s.data.Root, s.db.trie)
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion core/state/statedb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -998,7 +998,7 @@ func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (boo
// employed when the associated state snapshot is not available. It iterates the
// storage slots along with all internal trie nodes via trie directly.
func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) {
tr, err := s.db.OpenStorageTrie(s.originalRoot, addr, root)
tr, err := s.db.OpenStorageTrie(s.originalRoot, addr, root, s.trie)
if err != nil {
return false, 0, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err)
}
Expand Down
4 changes: 3 additions & 1 deletion core/state/trie_prefetcher.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,9 @@ func (sf *subfetcher) loop() {
}
sf.trie = trie
} else {
trie, err := sf.db.OpenStorageTrie(sf.state, sf.addr, sf.root)
// The trie argument can be nil as verkle doesn't support prefetching
// yet. TODO FIX IT(rjl493456442), otherwise code will panic here.
trie, err := sf.db.OpenStorageTrie(sf.state, sf.addr, sf.root, nil)
if err != nil {
log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err)
return
Expand Down
Loading