Faster pool block iteration backwards via PoolBlock iteration cache, added benchmark to SideChain

p2pool/sidechain/fake_server.go

@@ -6,12 +6,13 @@ import (
 	"git.gammaspectra.live/P2Pool/p2pool-observer/monero/client"
 	p2pooltypes "git.gammaspectra.live/P2Pool/p2pool-observer/p2pool/types"
 	"git.gammaspectra.live/P2Pool/p2pool-observer/types"
-	"sync/atomic"
+	"sync"
 )
 
 type FakeServer struct {
-	consensus  *Consensus
-	lastHeader atomic.Pointer[mainblock.Header]
+	consensus   *Consensus
+	headersLock sync.Mutex
+	headers     map[uint64]*mainblock.Header
 }
 
 func (s *FakeServer) Context() context.Context {
@@ -50,7 +51,9 @@ func (s *FakeServer) GetChainMainByHash(hash types.Hash) *ChainMain {
 	return nil
 }
 func (s *FakeServer) GetMinimalBlockHeaderByHeight(height uint64) *mainblock.Header {
-	if h := s.lastHeader.Load(); h != nil && h.Height == height {
+	s.headersLock.Lock()
+	defer s.headersLock.Unlock()
+	if h, ok := s.headers[height]; ok {
 		return h
 	}
 	if h, err := s.ClientRPC().GetBlockHeaderByHeight(height, context.Background()); err != nil {
@@ -67,7 +70,7 @@ func (s *FakeServer) GetMinimalBlockHeaderByHeight(height uint64) *mainblock.Hea
 			Difficulty:   types.DifficultyFrom64(h.BlockHeader.Difficulty),
 			Id:           types.MustHashFromString(h.BlockHeader.Hash),
 		}
-		s.lastHeader.Store(header)
+		s.headers[height] = header
 		return header
 	}
 }
@@ -99,5 +102,6 @@ func (s *FakeServer) ClearCachedBlocks() {
 func GetFakeTestServer(consensus *Consensus) *FakeServer {
 	return &FakeServer{
 		consensus: consensus,
+		headers:   make(map[uint64]*mainblock.Header),
 	}
 }

p2pool/sidechain/poolblock.go

@@ -72,6 +72,13 @@ func (s UniquePoolBlockSlice) GetHeight(height uint64) (result UniquePoolBlockSl
 	return result
 }
 
+// IterationCache Used for fast scan backwards, and of uncles
+// Only maybe available in verified blocks
+type IterationCache struct {
+	Parent *PoolBlock
+	Uncles []*PoolBlock
+}
+
 type PoolBlock struct {
 	Main mainblock.Block `json:"main"`
 
@@ -88,6 +95,33 @@ type PoolBlock struct {
 
 	LocalTimestamp     uint64       `json:"-"`
 	CachedShareVersion ShareVersion `json:"share_version"`
+
+	iterationCache *IterationCache
+}
+
+func (b *PoolBlock) iteratorGetParent(getByTemplateId GetByTemplateIdFunc) *PoolBlock {
+	if b.iterationCache == nil {
+		return getByTemplateId(b.Side.Parent)
+	}
+	return b.iterationCache.Parent
+}
+
+func (b *PoolBlock) iteratorUncles(getByTemplateId GetByTemplateIdFunc, uncleFunc func(uncle *PoolBlock)) error {
+	if b.iterationCache == nil {
+		for _, uncleId := range b.Side.Uncles {
+			uncle := getByTemplateId(uncleId)
+			if uncle == nil {
+				return fmt.Errorf("could not find uncle %s", uncleId)
+			}
+			uncleFunc(uncle)
+		}
+	} else {
+		for _, uncle := range b.iterationCache.Uncles {
+			uncleFunc(uncle)
+		}
+	}
+
+	return nil
 }
 
 // NewShareFromExportedBytes

p2pool/sidechain/sidechain.go

@@ -96,8 +96,8 @@ func NewSideChain(server P2PoolInterface) *SideChain {
 		blocksByTemplateId:                swiss.NewMap[types.Hash, *PoolBlock](uint32(server.Consensus().ChainWindowSize*2 + 300)),
 		blocksByHeight:                    swiss.NewMap[uint64, []*PoolBlock](uint32(server.Consensus().ChainWindowSize*2 + 300)),
 		preAllocatedShares:                PreAllocateShares(server.Consensus().ChainWindowSize * 2),
-		preAllocatedDifficultyData:        make([]DifficultyData, server.Consensus().ChainWindowSize*2),
-		preAllocatedDifficultyDifferences: make([]uint32, server.Consensus().ChainWindowSize*2),
+		preAllocatedDifficultyData:        make([]DifficultyData, 0, server.Consensus().ChainWindowSize*2),
+		preAllocatedDifficultyDifferences: make([]uint32, 0, server.Consensus().ChainWindowSize*2),
 		preAllocatedSharesPool:            NewPreAllocatedSharesPool(server.Consensus().ChainWindowSize * 2),
 		preAllocatedBuffer:                make([]byte, 0, PoolBlockMaxTemplateSize),
 		preAllocatedMinedBlocks:           make([]types.Hash, 0, 6*UncleBlockDepth*2+1),
@@ -488,14 +488,27 @@ func (c *SideChain) verifyLoop(blockToVerify *PoolBlock) (err error) {
 
 			// This block is now verified
 
+			// Fill cache here
+			block.iterationCache = &IterationCache{
+				Parent: c.getParent(block),
+				Uncles: nil,
+			}
+
+			if len(block.Side.Uncles) > 0 {
+				block.iterationCache.Uncles = make([]*PoolBlock, 0, len(block.Side.Uncles))
+				for _, uncleId := range block.Side.Uncles {
+					block.iterationCache.Uncles = append(block.iterationCache.Uncles, c.getPoolBlockByTemplateId(uncleId))
+				}
+			}
+
+			c.fillPoolBlockTransactionParentIndices(block)
+
 			if isLongerChain, _ := c.isLongerChain(highestBlock, block); isLongerChain {
 				highestBlock = block
 			} else if highestBlock != nil && highestBlock.Side.Height > block.Side.Height {
 				log.Printf("[SideChain] block at height = %d, id = %s, is not a longer chain than height = %d, id = %s", block.Side.Height, block.SideTemplateId(c.Consensus()), highestBlock.Side.Height, highestBlock.SideTemplateId(c.Consensus()))
 			}
 
-			c.fillPoolBlockTransactionParentIndices(block)
-
 			if block.WantBroadcast.Load() && !block.Broadcasted.Swap(true) {
 				if block.Depth.Load() < UncleBlockDepth {
 					c.server.Broadcast(block)
@@ -879,13 +892,17 @@ func (c *SideChain) pruneOldBlocks() {
 		for i := len(v) - 1; i >= 0; i-- {
 			block := v[i]
 			if block.Depth.Load() >= pruneDistance || (curTime >= (block.LocalTimestamp + pruneDelay)) {
-				if c.blocksByTemplateId.Has(block.SideTemplateId(c.Consensus())) {
-					c.blocksByTemplateId.Delete(block.SideTemplateId(c.Consensus()))
+				templateId := block.SideTemplateId(c.Consensus())
+				if c.blocksByTemplateId.Has(templateId) {
+					c.blocksByTemplateId.Delete(templateId)
 					numBlocksPruned++
 				} else {
 					log.Printf("[SideChain] blocksByHeight and blocksByTemplateId are inconsistent at height = %d, id = %s", height, block.SideTemplateId(c.Consensus()))
 				}
 				v = slices.Delete(v, i, i+1)
+
+				// Empty cache here
+				block.iterationCache = nil
 			}
 		}
 

p2pool/sidechain/sidechain_test.go

@@ -2,6 +2,7 @@ package sidechain
 
 import (
 	"git.gammaspectra.live/P2Pool/p2pool-observer/monero/client"
+	"git.gammaspectra.live/P2Pool/p2pool-observer/types"
 	"io"
 	"log"
 	"os"
@@ -131,3 +132,119 @@ func TestSideChainMiniPreFork(t *testing.T) {
 
 	testSideChain(s, t, f, 2424349, 2696040, block2420028, block2420027)
 }
+
+func benchmarkResetState(tip, parent *PoolBlock, templateId types.Hash, fullId FullId, difficulty types.Difficulty, s *SideChain) {
+	//Remove states in maps
+	s.blocksByHeight.Delete(tip.Side.Height)
+	s.blocksByTemplateId.Delete(templateId)
+	s.seenBlocks.Delete(fullId)
+
+	// Update tip and depths
+	tip.Depth.Store(0)
+	parent.Depth.Store(0)
+	s.chainTip.Store(parent)
+	s.syncTip.Store(parent)
+	s.currentDifficulty.Store(&difficulty)
+	s.updateDepths(parent)
+
+	// Update verification state
+	tip.Verified.Store(false)
+	tip.Invalid.Store(false)
+	tip.iterationCache = nil
+}
+
+func benchSideChain(b *testing.B, s *SideChain, tipHash types.Hash) {
+	b.StopTimer()
+
+	tip := s.GetChainTip()
+
+	for tip.SideTemplateId(s.Consensus()) != tipHash {
+		s.blocksByHeight.Delete(tip.Side.Height)
+		s.blocksByTemplateId.Delete(tip.SideTemplateId(s.Consensus()))
+		s.seenBlocks.Delete(tip.FullId())
+
+		tip = s.GetParent(tip)
+		if tip == nil {
+			b.Error("nil tip")
+			return
+		}
+	}
+	templateId := tip.SideTemplateId(s.Consensus())
+	fullId := tip.FullId()
+
+	parent := s.GetParent(tip)
+
+	difficulty, _, _ := s.GetDifficulty(parent)
+
+	benchmarkResetState(tip, parent, templateId, fullId, difficulty, s)
+
+	var err error
+
+	b.StartTimer()
+	for i := 0; i < b.N; i++ {
+		benchmarkResetState(tip, parent, templateId, fullId, difficulty, s)
+		_, err, _ = s.AddPoolBlockExternal(tip)
+		if err != nil {
+			b.Error(err)
+			return
+		}
+	}
+}
+
+var benchLoadedSideChain *SideChain
+
+func TestMain(m *testing.M) {
+
+	var isBenchmark bool
+	for _, arg := range os.Args {
+		if arg == "-test.bench" {
+			isBenchmark = true
+		}
+	}
+
+	if isBenchmark {
+		benchLoadedSideChain = NewSideChain(GetFakeTestServer(ConsensusDefault))
+
+		f, err := os.Open("testdata/sidechain_dump.dat")
+		if err != nil {
+			panic(err)
+		}
+		defer f.Close()
+
+		testSideChain(benchLoadedSideChain, nil, f, 4957203, 2870010)
+
+		tip := benchLoadedSideChain.GetChainTip()
+
+		// Pre-calculate PoW
+		for i := 0; i < 5; i++ {
+			tip.PowHashWithError(benchLoadedSideChain.Consensus().GetHasher(), benchLoadedSideChain.getSeedByHeightFunc())
+			tip = benchLoadedSideChain.GetParent(tip)
+		}
+	}
+
+	os.Exit(m.Run())
+}
+
+func BenchmarkSideChainDefault_AddPoolBlockExternal(b *testing.B) {
+	b.ReportAllocs()
+	benchSideChain(b, benchLoadedSideChain, types.MustHashFromString("61ecfc1c7738eacd8b815d2e28f124b31962996ae3af4121621b5c5501f19c5d"))
+}
+
+func BenchmarkSideChainDefault_GetDifficulty(b *testing.B) {
+	b.ReportAllocs()
+	tip := benchLoadedSideChain.GetChainTip()
+
+	b.ResetTimer()
+	var verifyError, invalidError error
+	for i := 0; i < b.N; i++ {
+		_, verifyError, invalidError = benchLoadedSideChain.getDifficulty(tip)
+		if verifyError != nil {
+			b.Error(verifyError)
+			return
+		}
+		if invalidError != nil {
+			b.Error(invalidError)
+			return
+		}
+	}
+}

p2pool/sidechain/utils.go

@@ -4,7 +4,6 @@ import (
 	"encoding/binary"
 	"fmt"
 	"git.gammaspectra.live/P2Pool/p2pool-observer/monero"
-	"git.gammaspectra.live/P2Pool/p2pool-observer/monero/address"
 	"git.gammaspectra.live/P2Pool/p2pool-observer/monero/block"
 	"git.gammaspectra.live/P2Pool/p2pool-observer/monero/crypto"
 	"git.gammaspectra.live/P2Pool/p2pool-observer/monero/randomx"
@@ -210,33 +209,31 @@ func BlocksInPPLNSWindow(tip *PoolBlock, consensus *Consensus, difficultyByHeigh
 	for {
 		curWeight := cur.Side.Difficulty
 
-		for _, uncleId := range cur.Side.Uncles {
-			if uncle := getByTemplateId(uncleId); uncle == nil {
-				//cannot find uncles
-				return 0, fmt.Errorf("could not find uncle %s", uncleId.String())
-			} else {
-				// Skip uncles which are already out of PPLNS window
-				if (tip.Side.Height - uncle.Side.Height) >= consensus.ChainWindowSize {
-					continue
-				}
-
-				// Take some % of uncle's weight into this share
-				unclePenalty := uncle.Side.Difficulty.Mul64(consensus.UnclePenalty).Div64(100)
-				uncleWeight := uncle.Side.Difficulty.Sub(unclePenalty)
-				newPplnsWeight := pplnsWeight.Add(uncleWeight)
-
-				// Skip uncles that push PPLNS weight above the limit
-				if newPplnsWeight.Cmp(maxPplnsWeight) > 0 {
-					continue
-				}
-				curWeight = curWeight.Add(unclePenalty)
-
-				if addWeightFunc != nil {
-					addWeightFunc(uncle, uncleWeight)
-				}
-
-				pplnsWeight = newPplnsWeight
+		if err := cur.iteratorUncles(getByTemplateId, func(uncle *PoolBlock) {
+			// Skip uncles which are already out of PPLNS window
+			if (tip.Side.Height - uncle.Side.Height) >= consensus.ChainWindowSize {
+				return
 			}
+
+			// Take some % of uncle's weight into this share
+			unclePenalty := uncle.Side.Difficulty.Mul64(consensus.UnclePenalty).Div64(100)
+			uncleWeight := uncle.Side.Difficulty.Sub(unclePenalty)
+			newPplnsWeight := pplnsWeight.Add(uncleWeight)
+
+			// Skip uncles that push PPLNS weight above the limit
+			if newPplnsWeight.Cmp(maxPplnsWeight) > 0 {
+				return
+			}
+			curWeight = curWeight.Add(unclePenalty)
+
+			if addWeightFunc != nil {
+				addWeightFunc(uncle, uncleWeight)
+			}
+
+			pplnsWeight = newPplnsWeight
+
+		}); err != nil {
+			return 0, err
 		}
 
 		// Always add non-uncle shares even if PPLNS weight goes above the limit
@@ -265,7 +262,7 @@ func BlocksInPPLNSWindow(tip *PoolBlock, consensus *Consensus, difficultyByHeigh
 		}
 
 		parentId := cur.Side.Parent
-		cur = getByTemplateId(parentId)
+		cur = cur.iteratorGetParent(getByTemplateId)
 
 		if cur == nil {
 			return 0, fmt.Errorf("could not find parent %s", parentId.String())
@@ -278,21 +275,17 @@ func GetSharesOrdered(tip *PoolBlock, consensus *Consensus, difficultyByHeight b
 	index := 0
 	l := len(preAllocatedShares)
 
-	insertShare := func(weight types.Difficulty, a address.PackedAddress) {
+	if bottomHeight, err := BlocksInPPLNSWindow(tip, consensus, difficultyByHeight, getByTemplateId, func(b *PoolBlock, weight types.Difficulty) {
 		if index < l {
-			preAllocatedShares[index].Address = a
+			preAllocatedShares[index].Address = b.GetAddress()
 			preAllocatedShares[index].Weight = weight
 		} else {
 			preAllocatedShares = append(preAllocatedShares, &Share{
-				Address: a,
+				Address: b.GetAddress(),
 				Weight:  weight,
 			})
 		}
 		index++
-	}
-
-	if bottomHeight, err := BlocksInPPLNSWindow(tip, consensus, difficultyByHeight, getByTemplateId, func(b *PoolBlock, weight types.Difficulty) {
-		insertShare(weight, b.GetAddress())
 	}); err != nil {
 		return nil, 0
 	} else {
@@ -344,43 +337,35 @@ func ShuffleSequence(shareVersion ShareVersion, privateKeySeed types.Hash, items
 	}
 }
 
+// GetDifficulty Gets the difficulty at tip
+// preAllocatedDifficultyData should contain enough capacity to fit all entries to iterate through.
+// preAllocatedTimestampDifferences should contain enough capacity to fit all differences.
 func GetDifficulty(tip *PoolBlock, consensus *Consensus, getByTemplateId GetByTemplateIdFunc, preAllocatedDifficultyData []DifficultyData, preAllocatedTimestampDifferences []uint32) (difficulty types.Difficulty, verifyError, invalidError error) {
-	index := 0
-	l := len(preAllocatedDifficultyData)
 
-	insertDifficultyData := func(cumDiff types.Difficulty, timestamp uint64) {
-		if index < l {
-			preAllocatedDifficultyData[index].CumulativeDifficulty = cumDiff
-			preAllocatedDifficultyData[index].Timestamp = timestamp
-		} else {
-			preAllocatedDifficultyData = append(preAllocatedDifficultyData, DifficultyData{
-				CumulativeDifficulty: cumDiff,
-				Timestamp:            timestamp,
-			})
-		}
-		index++
-	}
+	difficultyData := preAllocatedDifficultyData[:0]
 
 	cur := tip
 	var blockDepth uint64
 	var oldestTimestamp uint64 = math.MaxUint64
+
 	for {
-		oldestTimestamp = min(oldestTimestamp, cur.Main.Timestamp)
-		insertDifficultyData(cur.Side.CumulativeDifficulty, cur.Main.Timestamp)
+		difficultyData = append(difficultyData, DifficultyData{
+			CumulativeDifficulty: cur.Side.CumulativeDifficulty,
+			Timestamp:            cur.Main.Timestamp,
+		})
 
-		for _, uncleId := range cur.Side.Uncles {
-			if uncle := getByTemplateId(uncleId); uncle == nil {
-				//cannot find uncles
-				return types.ZeroDifficulty, fmt.Errorf("could not find uncle %s", uncleId), nil
-			} else {
-				// Skip uncles which are already out of PPLNS window
-				if (tip.Side.Height - uncle.Side.Height) >= consensus.ChainWindowSize {
-					continue
-				}
-
-				oldestTimestamp = min(oldestTimestamp, uncle.Main.Timestamp)
-				insertDifficultyData(uncle.Side.CumulativeDifficulty, uncle.Main.Timestamp)
+		if err := cur.iteratorUncles(getByTemplateId, func(uncle *PoolBlock) {
+			// Skip uncles which are already out of PPLNS window
+			if (tip.Side.Height - uncle.Side.Height) >= consensus.ChainWindowSize {
+				return
 			}
+
+			difficultyData = append(difficultyData, DifficultyData{
+				CumulativeDifficulty: uncle.Side.CumulativeDifficulty,
+				Timestamp:            uncle.Main.Timestamp,
+			})
+		}); err != nil {
+			return types.ZeroDifficulty, err, nil
 		}
 
 		blockDepth++
@@ -395,24 +380,25 @@ func GetDifficulty(tip *PoolBlock, consensus *Consensus, getByTemplateId GetByTe
 		}
 
 		parentId := cur.Side.Parent
-		cur = getByTemplateId(parentId)
+		cur = cur.iteratorGetParent(getByTemplateId)
 
 		if cur == nil {
-			return types.ZeroDifficulty, fmt.Errorf("could not find parent %s", parentId), nil
+			return types.ZeroDifficulty, fmt.Errorf("could not find parent %s", parentId.String()), nil
 		}
 	}
 
-	difficultyData := preAllocatedDifficultyData[:index]
-
-	if len(difficultyData) > len(preAllocatedTimestampDifferences) {
-		preAllocatedTimestampDifferences = make([]uint32, len(difficultyData))
+	for i := range difficultyData {
+		if difficultyData[i].Timestamp < oldestTimestamp {
+			oldestTimestamp = difficultyData[i].Timestamp
+		}
 	}
 
+	tmpTimestamps := preAllocatedTimestampDifferences[:0]
+
 	// Discard 10% oldest and 10% newest (by timestamp) blocks
 	for i := range difficultyData {
-		preAllocatedTimestampDifferences[i] = uint32(difficultyData[i].Timestamp - oldestTimestamp)
+		tmpTimestamps = append(tmpTimestamps, uint32(difficultyData[i].Timestamp-oldestTimestamp))
 	}
-	tmpTimestamps := preAllocatedTimestampDifferences[:len(difficultyData)]
 
 	cutSize := (len(difficultyData) + 9) / 10
 	index1 := cutSize - 1
@@ -440,13 +426,12 @@ func GetDifficulty(tip *PoolBlock, consensus *Consensus, getByTemplateId GetByTe
 	var diff2 types.Difficulty
 
 	for i := range difficultyData {
-		d := &difficultyData[i]
-		if timestamp1 <= d.Timestamp && d.Timestamp <= timestamp2 {
-			if d.CumulativeDifficulty.Cmp(diff1) < 0 {
-				diff1 = d.CumulativeDifficulty
+		if timestamp1 <= difficultyData[i].Timestamp && difficultyData[i].Timestamp <= timestamp2 {
+			if difficultyData[i].CumulativeDifficulty.Cmp(diff1) < 0 {
+				diff1 = difficultyData[i].CumulativeDifficulty
 			}
-			if diff2.Cmp(d.CumulativeDifficulty) < 0 {
-				diff2 = d.CumulativeDifficulty
+			if diff2.Cmp(difficultyData[i].CumulativeDifficulty) < 0 {
+				diff2 = difficultyData[i].CumulativeDifficulty
 			}
 		}
 	}