Remove go.mod from p2pool/cache

This commit is contained in:
DataHoarder 2024-04-03 19:07:19 +02:00
parent 71c27726be
commit bd934bb5a3
Signed by: DataHoarder
SSH key fingerprint: SHA256:OLTRf6Fl87G52SiR7sWLGNzlJt4WOX+tfI2yxo0z7xk
3 changed files with 0 additions and 488 deletions

View file

@ -1,408 +0,0 @@
package archive
import (
"bytes"
"encoding/binary"
"fmt"
"git.gammaspectra.live/P2Pool/p2pool-observer/monero/block"
"git.gammaspectra.live/P2Pool/p2pool-observer/p2pool/sidechain"
"git.gammaspectra.live/P2Pool/p2pool-observer/types"
"git.gammaspectra.live/P2Pool/p2pool-observer/utils"
bolt "go.etcd.io/bbolt"
"math"
"slices"
"time"
"unsafe"
)
// EpochSize Maximum amount of blocks without a full one
const EpochSize = 32
type Cache struct {
db *bolt.DB
consensus *sidechain.Consensus
difficultyByHeight block.GetDifficultyByHeightFunc
preAllocatedSharesPool *sidechain.PreAllocatedSharesPool
derivationCache sidechain.DerivationCacheInterface
}
var blocksByMainId = []byte("blocksByMainId")
var refByTemplateId = []byte("refByTemplateId")
var refBySideHeight = []byte("refBySideHeight")
var refByMainHeight = []byte("refByMainHeight")
func NewCache(path string, consensus *sidechain.Consensus, difficultyByHeight block.GetDifficultyByHeightFunc) (*Cache, error) {
if db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: time.Second * 5}); err != nil {
return nil, err
} else {
if err = db.Update(func(tx *bolt.Tx) error {
if _, err := tx.CreateBucketIfNotExists(blocksByMainId); err != nil {
return err
} else if _, err := tx.CreateBucketIfNotExists(refByTemplateId); err != nil {
return err
} else if _, err := tx.CreateBucketIfNotExists(refBySideHeight); err != nil {
return err
} else if _, err := tx.CreateBucketIfNotExists(refByMainHeight); err != nil {
return err
}
return nil
}); err != nil {
return nil, err
}
return &Cache{
db: db,
consensus: consensus,
difficultyByHeight: difficultyByHeight,
preAllocatedSharesPool: sidechain.NewPreAllocatedSharesPool(consensus.ChainWindowSize * 2),
derivationCache: sidechain.NewDerivationLRUCache(),
}, nil
}
}
type multiRecord []types.Hash
func multiRecordFromBytes(b []byte) multiRecord {
if len(b) == 0 || (len(b)%types.HashSize) != 0 {
return nil
}
return slices.Clone(unsafe.Slice((*types.Hash)(unsafe.Pointer(&b[0])), len(b)/types.HashSize))
}
func (r multiRecord) Contains(hash types.Hash) bool {
return slices.Contains(r, hash)
}
func (r multiRecord) Bytes() []byte {
if len(r) == 0 {
return nil
}
return slices.Clone(unsafe.Slice((*byte)(unsafe.Pointer(&r[0])), len(r)*types.HashSize))
}
func (c *Cache) Store(block *sidechain.PoolBlock) {
sideId := block.SideTemplateId(c.consensus)
if bytes.Compare(sideId[:], block.CoinbaseExtra(sidechain.SideTemplateId)) != 0 {
//wrong calculated template id
utils.Panicf("wrong template id, expected %s, got %s", types.HashFromBytes(block.CoinbaseExtra(sidechain.SideTemplateId)), sideId)
return
}
mainId := block.MainId()
var sideHeight, mainHeight [8]byte
binary.BigEndian.PutUint64(sideHeight[:], block.Side.Height)
binary.BigEndian.PutUint64(mainHeight[:], block.Main.Coinbase.GenHeight)
if c.ExistsByMainId(mainId) {
return
}
var storePruned, storeCompact bool
fullBlockTemplateHeight := block.Side.Height - (block.Side.Height % EpochSize)
// store full blocks on epoch
if block.Side.Height != fullBlockTemplateHeight {
if len(block.Main.Transactions) == len(block.Main.TransactionParentIndices) && c.loadByTemplateId(block.Side.Parent) != nil {
storeCompact = true
}
if block.Depth.Load() == math.MaxUint64 {
//fallback
if c.existsBySideChainHeightRange(block.Side.Height-c.consensus.ChainWindowSize-1, block.Side.Height-1) {
storePruned = true
}
//fallback for parent-less blocks
if len(c.LoadByTemplateId(block.Side.Parent)) == 0 {
storePruned, storeCompact = false, false
}
} else if block.Depth.Load() < c.consensus.ChainWindowSize {
storePruned = true
}
}
if blob, err := block.AppendBinaryFlags(make([]byte, 0, block.BufferLength()), storePruned, storeCompact); err == nil {
utils.Logf("Archive Cache", "Store block id = %s, template id = %s, height = %d, sidechain height = %d, depth = %d, pruned = %t, compact = %t, blob size = %d bytes", mainId.String(), sideId.String(), block.Main.Coinbase.GenHeight, block.Side.Height, block.Depth.Load(), storePruned, storeCompact, len(blob))
if err = c.db.Update(func(tx *bolt.Tx) error {
b1 := tx.Bucket(blocksByMainId)
var flags uint64
if storePruned {
flags |= 0b1
}
if storeCompact {
flags |= 0b10
}
buf := make([]byte, 0, len(blob)+8)
buf = binary.LittleEndian.AppendUint64(buf, flags)
buf = append(buf, blob...)
if err = b1.Put(mainId[:], buf); err != nil {
return err
}
b2 := tx.Bucket(refByTemplateId)
if records := multiRecordFromBytes(b2.Get(sideId[:])); !records.Contains(mainId) {
records = append(records, mainId)
if err = b2.Put(sideId[:], records.Bytes()); err != nil {
return err
}
}
b3 := tx.Bucket(refBySideHeight)
if records := multiRecordFromBytes(b3.Get(sideHeight[:])); !records.Contains(mainId) {
records = append(records, mainId)
if err = b3.Put(sideHeight[:], records.Bytes()); err != nil {
return err
}
}
b4 := tx.Bucket(refByMainHeight)
if records := multiRecordFromBytes(b4.Get(mainHeight[:])); !records.Contains(mainId) {
records = append(records, mainId)
if err = b4.Put(mainHeight[:], records.Bytes()); err != nil {
return err
}
}
return nil
}); err != nil {
utils.Logf("Archive Cache", "bolt error: %s", err)
}
}
}
func (c *Cache) RemoveByMainId(id types.Hash) {
//TODO
}
func (c *Cache) RemoveByTemplateId(id types.Hash) {
//TODO
}
func (c *Cache) ExistsByMainId(id types.Hash) (result bool) {
_ = c.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(blocksByMainId)
if b.Get(id[:]) != nil {
result = true
}
return nil
})
return result
}
func (c *Cache) loadByMainId(tx *bolt.Tx, id types.Hash) []byte {
b := tx.Bucket(blocksByMainId)
return b.Get(id[:])
}
func (c *Cache) ProcessBlock(b *sidechain.PoolBlock) error {
getTemplateById := func(h types.Hash) *sidechain.PoolBlock {
if bs := c.LoadByTemplateId(h); len(bs) > 0 {
return bs[0]
}
return nil
}
preAllocatedShares := c.preAllocatedSharesPool.Get()
defer c.preAllocatedSharesPool.Put(preAllocatedShares)
_, err := b.PreProcessBlock(c.consensus, c.derivationCache, preAllocatedShares, c.difficultyByHeight, getTemplateById)
return err
}
func (c *Cache) decodeBlock(blob []byte) *sidechain.PoolBlock {
if blob == nil {
return nil
}
flags := binary.LittleEndian.Uint64(blob)
b := &sidechain.PoolBlock{}
reader := bytes.NewReader(blob[8:])
if (flags & 0b10) > 0 {
if err := b.FromCompactReader(c.consensus, c.derivationCache, reader); err != nil {
utils.Logf("Archive Cache", "error decoding block: %s", err)
return nil
}
} else {
if err := b.FromReader(c.consensus, c.derivationCache, reader); err != nil {
utils.Logf("Archive Cache", "error decoding block: %s", err)
return nil
}
}
return b
}
func (c *Cache) LoadByMainId(id types.Hash) *sidechain.PoolBlock {
var blob []byte
_ = c.db.View(func(tx *bolt.Tx) error {
blob = c.loadByMainId(tx, id)
return nil
})
return c.decodeBlock(blob)
}
func (c *Cache) loadByTemplateId(id types.Hash) (r multiRecord) {
_ = c.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(refByTemplateId)
r = multiRecordFromBytes(b.Get(id[:]))
return nil
})
return r
}
func (c *Cache) LoadByTemplateId(id types.Hash) (result sidechain.UniquePoolBlockSlice) {
blocks := make([][]byte, 0, 1)
if err := c.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(refByTemplateId)
r := multiRecordFromBytes(b.Get(id[:]))
for _, h := range r {
if e := c.loadByMainId(tx, h); e != nil {
blocks = append(blocks, e)
} else {
return fmt.Errorf("could not find block %s", h.String())
}
}
return nil
}); err != nil {
utils.Logf("Archive Cache", "error fetching blocks with template id %s, %s", id.String(), err)
return nil
}
for _, buf := range blocks {
if b := c.decodeBlock(buf); b != nil {
result = append(result, b)
}
}
return result
}
func (c *Cache) ScanHeights(startHeight, endHeight uint64) chan sidechain.UniquePoolBlockSlice {
result := make(chan sidechain.UniquePoolBlockSlice)
go func() {
defer close(result)
_ = c.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(refBySideHeight)
var startHeightBytes, endHeightBytes [8]byte
cursor := b.Cursor()
binary.BigEndian.PutUint64(startHeightBytes[:], startHeight)
binary.BigEndian.PutUint64(endHeightBytes[:], endHeight)
k, v := cursor.Seek(startHeightBytes[:])
for {
if k == nil {
return nil
}
r := multiRecordFromBytes(v)
blocks := make(sidechain.UniquePoolBlockSlice, 0, len(r))
for _, h := range r {
if e := c.loadByMainId(tx, h); e != nil {
if bl := c.decodeBlock(e); bl != nil {
blocks = append(blocks, bl)
} else {
return fmt.Errorf("could not decode block %s", h.String())
}
} else {
return fmt.Errorf("could not find block %s", h.String())
}
}
result <- blocks
if bytes.Compare(k, endHeightBytes[:]) >= 0 {
break
}
k, v = cursor.Next()
}
return nil
})
}()
return result
}
func (c *Cache) existsBySideChainHeightRange(startHeight, endHeight uint64) (result bool) {
_ = c.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(refBySideHeight)
var startHeightBytes, endHeightBytes [8]byte
cursor := b.Cursor()
binary.BigEndian.PutUint64(startHeightBytes[:], startHeight)
binary.BigEndian.PutUint64(endHeightBytes[:], endHeight)
expectedHeight := startHeight
k, v := cursor.Seek(startHeightBytes[:])
for {
if k == nil {
return nil
}
h := binary.BigEndian.Uint64(k)
//utils.Logf("height check for %d -> %d: %d, expected %d, len %d", startHeight, endHeight, h, expectedHeight, len(v))
if v == nil || h != expectedHeight {
return nil
}
if bytes.Compare(k, endHeightBytes[:]) > 0 {
return nil
} else if bytes.Compare(k, endHeightBytes[:]) == 0 {
break
}
expectedHeight++
k, v = cursor.Next()
}
result = true
return nil
})
return result
}
func (c *Cache) LoadBySideChainHeight(height uint64) (result sidechain.UniquePoolBlockSlice) {
var sideHeight [8]byte
binary.BigEndian.PutUint64(sideHeight[:], height)
blocks := make([][]byte, 0, 1)
if err := c.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(refBySideHeight)
r := multiRecordFromBytes(b.Get(sideHeight[:]))
for _, h := range r {
if e := c.loadByMainId(tx, h); e != nil {
blocks = append(blocks, e)
} else {
return fmt.Errorf("could not find block %s", h.String())
}
}
return nil
}); err != nil {
utils.Logf("Archive Cache", "error fetching blocks with sidechain height %d, %s", height, err)
return nil
}
for _, buf := range blocks {
if b := c.decodeBlock(buf); b != nil {
result = append(result, b)
}
}
return result
}
func (c *Cache) LoadByMainChainHeight(height uint64) (result sidechain.UniquePoolBlockSlice) {
var mainHeight [8]byte
binary.BigEndian.PutUint64(mainHeight[:], height)
blocks := make([][]byte, 0, 1)
if err := c.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(refByMainHeight)
r := multiRecordFromBytes(b.Get(mainHeight[:]))
for _, h := range r {
if e := c.loadByMainId(tx, h); e != nil {
blocks = append(blocks, e)
} else {
return fmt.Errorf("could not find block %s", h.String())
}
}
return nil
}); err != nil {
utils.Logf("Archive Cache", "error fetching blocks with sidechain height %d, %s", height, err)
return nil
}
for _, buf := range blocks {
if b := c.decodeBlock(buf); b != nil {
result = append(result, b)
}
}
return result
}
func (c *Cache) Close() {
_ = c.db.Close()
}

32
p2pool/cache/go.mod vendored
View file

@ -1,32 +0,0 @@
module git.gammaspectra.live/P2Pool/p2pool-observer/p2pool/cache
go 1.22
replace git.gammaspectra.live/P2Pool/p2pool-observer v0.0.0 => ../../
require (
git.gammaspectra.live/P2Pool/p2pool-observer v0.0.0
go.etcd.io/bbolt v1.3.7
)
require (
git.gammaspectra.live/P2Pool/edwards25519 v0.0.0-20230701100949-027561bd2a33 // indirect
git.gammaspectra.live/P2Pool/go-monero v0.0.0-20230410011208-910450c4a523 // indirect
git.gammaspectra.live/P2Pool/go-randomx v0.0.0-20221027085532-f46adfce03a7 // indirect
git.gammaspectra.live/P2Pool/moneroutil v0.0.0-20230722215223-18ecc51ae61e // indirect
git.gammaspectra.live/P2Pool/randomx-go-bindings v0.0.0-20230514082649-9c5f18cd5a71 // indirect
git.gammaspectra.live/P2Pool/sha3 v0.0.0-20230604092430-04fe7dc6439a // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/dolthub/maphash v0.1.0 // indirect
github.com/dolthub/swiss v0.1.0 // indirect
github.com/floatdrop/lru v1.3.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/holiman/uint256 v1.2.3 // indirect
github.com/jxskiss/base62 v1.1.0 // indirect
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
golang.org/x/crypto v0.11.0 // indirect
golang.org/x/sys v0.10.0 // indirect
lukechampine.com/uint128 v1.3.0 // indirect
)
replace github.com/goccy/go-json => github.com/WeebDataHoarder/go-json v0.0.0-20230730135821-d8f6463bb887

48
p2pool/cache/go.sum vendored
View file

@ -1,48 +0,0 @@
git.gammaspectra.live/P2Pool/edwards25519 v0.0.0-20230701100949-027561bd2a33 h1:BPV7iIiv8T+X7gg9/JfNmEBoH4HXOkw8CR7FN6bBwB8=
git.gammaspectra.live/P2Pool/edwards25519 v0.0.0-20230701100949-027561bd2a33/go.mod h1:336HUKX25mQ1qUtzkwV9Wrqi153tTgUOKcIhpYuF2ts=
git.gammaspectra.live/P2Pool/go-monero v0.0.0-20230410011208-910450c4a523 h1:oIJzm7kQyASS0xlJ79VSWRvvfXp2Qt7M05+E20o9gwE=
git.gammaspectra.live/P2Pool/go-monero v0.0.0-20230410011208-910450c4a523/go.mod h1:TAOAAV972JNDkCzyV5SkbYkKCRvcfhvvFa8LHH4Dg6g=
git.gammaspectra.live/P2Pool/go-randomx v0.0.0-20221027085532-f46adfce03a7 h1:bzHDuu1IgETKqPBOlIdCE2LaZIJ+ZpROSprNn+fnzd8=
git.gammaspectra.live/P2Pool/go-randomx v0.0.0-20221027085532-f46adfce03a7/go.mod h1:3kT0v4AMwT/OdorfH2gRWPwoOrUX/LV03HEeBsaXG1c=
git.gammaspectra.live/P2Pool/moneroutil v0.0.0-20230722215223-18ecc51ae61e h1:ropqS9niQR/ZKCUrlmWe+uDH0fLIyAnCIjkEjyTDgA8=
git.gammaspectra.live/P2Pool/moneroutil v0.0.0-20230722215223-18ecc51ae61e/go.mod h1:Wn5QI7XIMHMpEu10pPspW9h3eGmXQPJwh/4/+Gi3G1U=
git.gammaspectra.live/P2Pool/randomx-go-bindings v0.0.0-20230514082649-9c5f18cd5a71 h1:MgeHHcF+GnCJBWMSzq8XAbc8p/UhNwFruEKCPPJ74YQ=
git.gammaspectra.live/P2Pool/randomx-go-bindings v0.0.0-20230514082649-9c5f18cd5a71/go.mod h1:KQaYHIxGXNHNMQELC7xGLu8xouwvP/dN7iGk681BXmk=
git.gammaspectra.live/P2Pool/sha3 v0.0.0-20230604092430-04fe7dc6439a h1:c24MHv/z+aBYpYNsQHcJqmFuaYInGVixJZgDCXA/4bs=
git.gammaspectra.live/P2Pool/sha3 v0.0.0-20230604092430-04fe7dc6439a/go.mod h1:6wZ0+whl+HZdcRve4R6Rq6jV1fmL1xCYO8Wty6lR008=
github.com/WeebDataHoarder/go-json v0.0.0-20230730135821-d8f6463bb887 h1:P01nqSM+0b6zlPasOFYsqnQSP2dTRVZkanAnY9q/Bcc=
github.com/WeebDataHoarder/go-json v0.0.0-20230730135821-d8f6463bb887/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dolthub/maphash v0.1.0 h1:bsQ7JsF4FkkWyrP3oCnFJgrCUAFbFf3kOl4L/QxPDyQ=
github.com/dolthub/maphash v0.1.0/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4=
github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc=
github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ=
github.com/floatdrop/lru v1.3.0 h1:83abtaKjXcWrPmtzTAk2Ggq8DUKqI29YzrTrB8+vu0c=
github.com/floatdrop/lru v1.3.0/go.mod h1:83zlXKA06Bm32JImNINCiTr0ldadvdAjUe5jSwIaw0s=
github.com/holiman/uint256 v1.2.3 h1:K8UWO1HUJpRMXBxbmaY1Y8IAMZC/RsKB+ArEnnK4l5o=
github.com/holiman/uint256 v1.2.3/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw=
github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw=
github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8=
github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/thepudds/swisstable v0.0.0-20221011152303-9c77dc657777 h1:5u+6YWU2faS+Sr/x8j9yalMpSDUkatNOZWXV3wMUCGQ=
github.com/thepudds/swisstable v0.0.0-20221011152303-9c77dc657777/go.mod h1:4af3KxEsswy6aTzsTcwa8QZUSh4V+80oHdp1QX9uJHA=
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo=
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs=
go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo=
lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=