Merge pull request #2508 from CortexFoundation/dev

reduce allocations in derivesha

Author: ucwong

core/types/block.go

@@ -237,7 +237,10 @@ type storageblock struct {
 // block.
 //
 // relevant portions of the header.
-func NewBlock(header *Header, body *Body, receipts []*Receipt, hasher Hasher) *Block {
+//
+// The receipt's bloom must already calculated for the block's bloom to be
+// correctly calculated.
+func NewBlock(header *Header, body *Body, receipts []*Receipt, hasher ListHasher) *Block {
 	if body == nil {
 		body = &Body{}
 	}

core/types/derive_sha.go

@@ -27,7 +27,7 @@ import (
 	"github.com/CortexFoundation/CortexTheseus/rlp"
 )
 
-// hasherPool holds LegacyKeccak256 hashers for rlpHash.
+// hasherPool holds LegacyKeccak256 buffer for rlpHash.
 var hasherPool = sync.Pool{
 	New: func() interface{} { return crypto.NewKeccakState() },
 }
@@ -75,10 +75,66 @@ func prefixedRlpHash(prefix byte, x interface{}) (h common.Hash) {
 	return h
 }
 
-// TrieHasher is the tool used to calculate the hash of derivable list.
-// This is internal, do not use.
-type TrieHasher interface {
+// ListHasher defines the interface for computing the hash of a derivable list.
+type ListHasher interface {
+	// Reset clears the internal state of the hasher, preparing it for reuse.
 	Reset()
-	Update([]byte, []byte) error
+
+	// Update inserts the given key-value pair into the hasher.
+	// The implementation must copy the provided slices, allowing the caller
+	// to safely modify them after the call returns.
+	Update(key []byte, value []byte) error
+
+	// Hash computes and returns the final hash of all inserted key-value pairs.
 	Hash() common.Hash
 }
+
+// DerivableList is the input to DeriveSha.
+// It is implemented by the 'Transactions' and 'Receipts' types.
+// This is internal, do not use these methods.
+type DerivableList interface {
+	Len() int
+	EncodeIndex(int, *bytes.Buffer)
+}
+
+// encodeForDerive encodes the element in the list at the position i into the buffer.
+func encodeForDerive(list DerivableList, i int, buf *bytes.Buffer) []byte {
+	buf.Reset()
+	list.EncodeIndex(i, buf)
+	return buf.Bytes()
+}
+
+// DeriveSha creates the tree hashes of transactions, receipts, and withdrawals in a block header.
+func DeriveSha(list DerivableList, hasher ListHasher) common.Hash {
+	hasher.Reset()
+
+	// Allocate a buffer for value encoding. As the hasher is claimed that all
+	// supplied key value pairs will be copied by hasher and safe to reuse the
+	// encoding buffer.
+	valueBuf := encodeBufferPool.Get().(*bytes.Buffer)
+	defer encodeBufferPool.Put(valueBuf)
+
+	// StackTrie requires values to be inserted in increasing hash order, which is not the
+	// order that `list` provides hashes in. This insertion sequence ensures that the
+	// order is correct.
+	//
+	// The error returned by hasher is omitted because hasher will produce an incorrect
+	// hash in case any error occurs.
+	var indexBuf []byte
+	for i := 1; i < list.Len() && i <= 0x7f; i++ {
+		indexBuf = rlp.AppendUint64(indexBuf[:0], uint64(i))
+		value := encodeForDerive(list, i, valueBuf)
+		hasher.Update(indexBuf, value)
+	}
+	if list.Len() > 0 {
+		indexBuf = rlp.AppendUint64(indexBuf[:0], 0)
+		value := encodeForDerive(list, 0, valueBuf)
+		hasher.Update(indexBuf, value)
+	}
+	for i := 0x80; i < list.Len(); i++ {
+		indexBuf = rlp.AppendUint64(indexBuf[:0], uint64(i))
+		value := encodeForDerive(list, i, valueBuf)
+		hasher.Update(indexBuf, value)
+	}
+	return hasher.Hash()
+}

core/types/hashing.go

@@ -293,6 +293,13 @@ func (r Receipts) GetRlp(i int) []byte {
 	return bytes
 }
 
+// EncodeIndex encodes the i'th receipt to w.
+func (rs Receipts) EncodeIndex(i int, w *bytes.Buffer) {
+	r := rs[i]
+	data := &receiptRLP{r.statusEncoding(), r.CumulativeGasUsed, r.Bloom, r.Logs}
+	rlp.Encode(w, data)
+}
+
 // DeriveFields fills the receipts with their computed fields based on consensus
 // data and contextual infos like containing block and transactions.
 func (r Receipts) DeriveFields(config *params.ChainConfig, hash common.Hash, number uint64, time uint64, txs []*Transaction) error {

core/types/receipt.go

@@ -17,6 +17,7 @@
 package types
 
 import (
+	"bytes"
 	"container/heap"
 	"errors"
 	"fmt"
@@ -292,6 +293,14 @@ func (s Transactions) GetRlp(i int) []byte {
 	return enc
 }
 
+// EncodeIndex encodes the i'th transaction to w. Note that this does not check for errors
+// because we assume that *Transaction will only ever contain valid txs that were either
+// constructed by decoding or via public API in this package.
+func (s Transactions) EncodeIndex(i int, w *bytes.Buffer) {
+	tx := s[i]
+	rlp.Encode(w, &tx.data)
+}
+
 // TxDifference returns a new set of transactions that are present in a but not in b.
 func TxDifference(a, b Transactions) Transactions {
 	keep := make(Transactions, 0, len(a))

core/types/transaction.go

@@ -32,8 +32,8 @@ func newBytesPool(sliceCap, nitems int) *bytesPool {
 	}
 }
 
-// Get returns a slice. Safe for concurrent use.
-func (bp *bytesPool) Get() []byte {
+// get returns a slice. Safe for concurrent use.
+func (bp *bytesPool) get() []byte {
 	select {
 	case b := <-bp.c:
 		return b
@@ -42,18 +42,18 @@ func (bp *bytesPool) Get() []byte {
 	}
 }
 
-// GetWithSize returns a slice with specified byte slice size.
-func (bp *bytesPool) GetWithSize(s int) []byte {
-	b := bp.Get()
+// getWithSize returns a slice with specified byte slice size.
+func (bp *bytesPool) getWithSize(s int) []byte {
+	b := bp.get()
 	if cap(b) < s {
 		return make([]byte, s)
 	}
 	return b[:s]
 }
 
-// Put returns a slice to the pool. Safe for concurrent use. This method
+// put returns a slice to the pool. Safe for concurrent use. This method
 // will ignore slices that are too small or too large (>3x the cap)
-func (bp *bytesPool) Put(b []byte) {
+func (bp *bytesPool) put(b []byte) {
 	if c := cap(b); c < bp.w || c > 3*bp.w {
 		return
 	}
@@ -62,3 +62,40 @@ func (bp *bytesPool) Put(b []byte) {
 	default:
 	}
 }
+
+// unsafeBytesPool is a pool for byte slices. It is not safe for concurrent use.
+type unsafeBytesPool struct {
+	items [][]byte
+	w     int
+}
+
+// newUnsafeBytesPool creates a new unsafeBytesPool. The sliceCap sets the
+// capacity of newly allocated slices, and the nitems determines how many
+// items the pool will hold, at maximum.
+func newUnsafeBytesPool(sliceCap, nitems int) *unsafeBytesPool {
+	return &unsafeBytesPool{
+		items: make([][]byte, 0, nitems),
+		w:     sliceCap,
+	}
+}
+
+// Get returns a slice with pre-allocated space.
+func (bp *unsafeBytesPool) get() []byte {
+	if len(bp.items) > 0 {
+		last := bp.items[len(bp.items)-1]
+		bp.items = bp.items[:len(bp.items)-1]
+		return last
+	}
+	return make([]byte, 0, bp.w)
+}
+
+// put returns a slice to the pool. This method will ignore slices that are
+// too small or too large (>3x the cap)
+func (bp *unsafeBytesPool) put(b []byte) {
+	if c := cap(b); c < bp.w || c > 3*bp.w {
+		return
+	}
+	if len(bp.items) < cap(bp.items) {
+		bp.items = append(bp.items, b)
+	}
+}

trie/bytepool.go

@@ -0,0 +1,56 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package trie
+
+import (
+	"bytes"
+
+	"github.com/CortexFoundation/CortexTheseus/common"
+)
+
+// ListHasher is a wrapper of the Merkle-Patricia-Trie, which implements
+// types.ListHasher. Compared to a Trie instance, the Update method of this
+// type always deep-copies its input slices.
+//
+// This implementation is very inefficient in terms of memory allocation,
+// compared with StackTrie. It exists only for correctness comparison purposes.
+type ListHasher struct {
+	tr *Trie
+}
+
+// NewListHasher initializes the list hasher.
+func NewListHasher() *ListHasher {
+	return &ListHasher{
+		tr: NewEmpty(nil),
+	}
+}
+
+// Reset clears the internal state prepares the ListHasher for reuse.
+func (h *ListHasher) Reset() {
+	h.tr.reset()
+}
+
+// Update inserts a key-value pair into the trie.
+func (h *ListHasher) Update(key []byte, value []byte) error {
+	key, value = bytes.Clone(key), bytes.Clone(value)
+	return h.tr.Update(key, value)
+}
+
+// Hash computes the root hash of all inserted key-value pairs.
+func (h *ListHasher) Hash() common.Hash {
+	return h.tr.Hash()
+}