Added cmd/apitocache tool

2023-11-04 22:33:59 +01:00 · 2023-11-04 22:33:59 +01:00 · 30623dbd55
parent 3c4e0016b8
commit 30623dbd55
6 changed files with 317 additions and 5 deletions
--- a/cmd/apitocache/api.go
+++ b/cmd/apitocache/api.go
@ -0,0 +1,129 @@
+package main
+
+import (
+	"git.gammaspectra.live/P2Pool/p2pool-observer/utils"
+	"io"
+	"net/http"
+	"net/url"
+)
+
+var apiUrl string
+
+func getTypeFromAPI[T any](method string) *T {
+	uri, _ := url.Parse(apiUrl + method)
+	if response, err := http.DefaultClient.Do(&http.Request{
+		Method: "GET",
+		URL:    uri,
+	}); err != nil {
+		return nil
+	} else {
+		defer response.Body.Close()
+		defer io.ReadAll(response.Body)
+		if response.StatusCode == http.StatusOK {
+			var result T
+			decoder := utils.NewJSONDecoder(response.Body)
+			if decoder.Decode(&result) != nil {
+				return nil
+			} else {
+				return &result
+			}
+		} else {
+			return nil
+		}
+	}
+}
+
+func getSliceFromAPI[T any](method string) []T {
+
+	uri, _ := url.Parse(apiUrl + method)
+	if response, err := http.DefaultClient.Do(&http.Request{
+		Method: "GET",
+		URL:    uri,
+	}); err != nil {
+		return nil
+	} else {
+		defer response.Body.Close()
+		defer io.ReadAll(response.Body)
+		if response.StatusCode == http.StatusOK {
+			var result []T
+			decoder := utils.NewJSONDecoder(response.Body)
+			if decoder.Decode(&result) != nil {
+				return nil
+			} else {
+				return result
+			}
+		} else {
+			return nil
+		}
+	}
+}
+
+func getStreamFromAPI[T any](method string) <-chan T {
+	result := make(chan T, 1)
+
+	go func() {
+		defer close(result)
+		uri, _ := url.Parse(apiUrl + method)
+		if response, err := http.DefaultClient.Do(&http.Request{
+			Method: "GET",
+			URL:    uri,
+		}); err != nil {
+			return
+		} else {
+			defer response.Body.Close()
+			defer io.ReadAll(response.Body)
+
+			if response.StatusCode == http.StatusOK {
+
+				var err error
+
+				// Read opening
+				var b [1]byte
+				for {
+					if _, err = response.Body.Read(b[:]); err != nil {
+						return
+					}
+					if b[0] == '[' {
+						break
+					} else if b[0] != ' ' && b[0] != 0xa {
+						return
+					}
+				}
+
+				decoder := utils.NewJSONDecoder(response.Body)
+				for decoder.More() {
+					var item T
+					if err := decoder.Decode(&item); err != nil {
+						return
+					} else {
+						result <- item
+					}
+				}
+			}
+		}
+	}()
+
+	return result
+}
+
+func getFromAPIRaw(method string) []byte {
+	uri, _ := url.Parse(apiUrl + method)
+	if response, err := http.DefaultClient.Do(&http.Request{
+		Method: "GET",
+		URL:    uri,
+	}); err != nil {
+		return nil
+	} else {
+		defer response.Body.Close()
+		defer io.ReadAll(response.Body)
+		if response.StatusCode == http.StatusOK {
+			if data, err := io.ReadAll(response.Body); err != nil {
+				return nil
+			} else {
+				return data
+			}
+		} else {
+			return nil
+		}
+	}
+}
--- a/cmd/apitocache/apitocache.go
+++ b/cmd/apitocache/apitocache.go
@ -0,0 +1,93 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	cmdutils "git.gammaspectra.live/P2Pool/p2pool-observer/cmd/utils"
+	"git.gammaspectra.live/P2Pool/p2pool-observer/p2pool/cache/legacy"
+	"git.gammaspectra.live/P2Pool/p2pool-observer/p2pool/sidechain"
+	"git.gammaspectra.live/P2Pool/p2pool-observer/types"
+	"git.gammaspectra.live/P2Pool/p2pool-observer/utils"
+	"log"
+	"slices"
+	"strconv"
+)
+
+func main() {
+	selectedApiUrl := flag.String("api", "", "Input API url, for example, https://p2pool.observer/api/")
+	outputFile := flag.String("output", "p2pool.cache", "Output p2pool.cache path")
+	fromBlock := flag.String("from", "tip", "Block to start from. Can be an ID or a height")
+
+	flag.Parse()
+
+	apiUrl = *selectedApiUrl
+
+	poolInfo := getTypeFromAPI[cmdutils.PoolInfoResult]("pool_info")
+	if poolInfo == nil {
+		panic("could not fetch consensus")
+	}
+
+	consensusData, _ := utils.MarshalJSON(poolInfo.SideChain.Consensus)
+	consensus, err := sidechain.NewConsensusFromJSON(consensusData)
+	if err != nil {
+		log.Panic(err)
+	}
+
+	log.Printf("Consensus id = %s", consensus.Id)
+
+	cache, err := legacy.NewCache(consensus, *outputFile)
+	if err != nil {
+		log.Panic(err)
+	}
+	defer cache.Close()
+
+	var toFetchUrls []string
+
+	if *fromBlock == "tip" {
+		toFetchUrls = append(toFetchUrls, fmt.Sprintf("redirect/tip/raw"))
+	} else if n, err := strconv.ParseUint(*fromBlock, 10, 0); err == nil {
+		toFetchUrls = append(toFetchUrls, fmt.Sprintf("block_by_height/%d/raw", n))
+	} else {
+		toFetchUrls = append(toFetchUrls, fmt.Sprintf("block_by_id/%s/raw", *fromBlock))
+	}
+
+	var fetches int
+
+	addBlockId := func(h types.Hash) {
+		k := fmt.Sprintf("block_by_id/%s/raw", h)
+		if slices.Contains(toFetchUrls, k) {
+			return
+		}
+		toFetchUrls = append(toFetchUrls, k)
+	}
+
+	for len(toFetchUrls) > 0 {
+		nextUrl := toFetchUrls[0]
+		fmt.Printf("[%d] fetching %s\n", fetches, nextUrl)
+		toFetchUrls = slices.Delete(toFetchUrls, 0, 1)
+
+		rawBlock := getFromAPIRaw(nextUrl)
+		b := &sidechain.PoolBlock{}
+		err := b.UnmarshalBinary(consensus, &sidechain.NilDerivationCache{}, rawBlock)
+		if err != nil {
+			panic(fmt.Errorf("could not fetch block from url %s: %w", nextUrl, err))
+		}
+
+		cache.Store(b)
+
+		for _, u := range b.Side.Uncles {
+			addBlockId(u)
+		}
+		addBlockId(b.Side.Parent)
+
+		fetches++
+
+		if fetches >= legacy.NumBlocks {
+			print("reached max limit of block cache, exiting\n")
+			break
+		}
+
+	}
+
+	cache.Flush()
+}
--- a/cmd/apitocache/go.mod
+++ b/cmd/apitocache/go.mod
@ -0,0 +1,41 @@
+module git.gammaspectra.live/P2Pool/p2pool-observer/cmd/apitocache
+
+go 1.21
+
+replace git.gammaspectra.live/P2Pool/p2pool-observer v0.0.0 => ../../
+
+replace git.gammaspectra.live/P2Pool/p2pool-observer/p2pool/cache v0.0.0 => ../../p2pool/cache
+
+replace git.gammaspectra.live/P2Pool/p2pool-observer/cmd/index v0.0.0 => ../index
+
+replace git.gammaspectra.live/P2Pool/p2pool-observer/cmd/utils v0.0.0 => ../utils
+
+require (
+	git.gammaspectra.live/P2Pool/p2pool-observer v0.0.0
+	git.gammaspectra.live/P2Pool/p2pool-observer/cmd/utils v0.0.0
+	git.gammaspectra.live/P2Pool/p2pool-observer/p2pool/cache v0.0.0
+)
+
+require (
+	git.gammaspectra.live/P2Pool/edwards25519 v0.0.0-20230701100949-027561bd2a33 // indirect
+	git.gammaspectra.live/P2Pool/go-monero v0.0.0-20230410011208-910450c4a523 // indirect
+	git.gammaspectra.live/P2Pool/go-randomx v0.0.0-20221027085532-f46adfce03a7 // indirect
+	git.gammaspectra.live/P2Pool/moneroutil v0.0.0-20230722215223-18ecc51ae61e // indirect
+	git.gammaspectra.live/P2Pool/p2pool-observer/cmd/index v0.0.0 // indirect
+	git.gammaspectra.live/P2Pool/randomx-go-bindings v0.0.0-20230514082649-9c5f18cd5a71 // indirect
+	git.gammaspectra.live/P2Pool/sha3 v0.0.0-20230604092430-04fe7dc6439a // indirect
+	github.com/bahlo/generic-list-go v0.2.0 // indirect
+	github.com/dolthub/maphash v0.1.0 // indirect
+	github.com/dolthub/swiss v0.1.0 // indirect
+	github.com/floatdrop/lru v1.3.0 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/holiman/uint256 v1.2.3 // indirect
+	github.com/jxskiss/base62 v1.1.0 // indirect
+	github.com/lib/pq v1.10.9 // indirect
+	github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
+	golang.org/x/crypto v0.11.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
+	lukechampine.com/uint128 v1.3.0 // indirect
+)
+
+replace github.com/goccy/go-json => github.com/WeebDataHoarder/go-json v0.0.0-20230730135821-d8f6463bb887
--- a/cmd/apitocache/go.sum
+++ b/cmd/apitocache/go.sum
@ -0,0 +1,48 @@
+git.gammaspectra.live/P2Pool/edwards25519 v0.0.0-20230701100949-027561bd2a33 h1:BPV7iIiv8T+X7gg9/JfNmEBoH4HXOkw8CR7FN6bBwB8=
+git.gammaspectra.live/P2Pool/edwards25519 v0.0.0-20230701100949-027561bd2a33/go.mod h1:336HUKX25mQ1qUtzkwV9Wrqi153tTgUOKcIhpYuF2ts=
+git.gammaspectra.live/P2Pool/go-monero v0.0.0-20230410011208-910450c4a523 h1:oIJzm7kQyASS0xlJ79VSWRvvfXp2Qt7M05+E20o9gwE=
+git.gammaspectra.live/P2Pool/go-monero v0.0.0-20230410011208-910450c4a523/go.mod h1:TAOAAV972JNDkCzyV5SkbYkKCRvcfhvvFa8LHH4Dg6g=
+git.gammaspectra.live/P2Pool/go-randomx v0.0.0-20221027085532-f46adfce03a7 h1:bzHDuu1IgETKqPBOlIdCE2LaZIJ+ZpROSprNn+fnzd8=
+git.gammaspectra.live/P2Pool/go-randomx v0.0.0-20221027085532-f46adfce03a7/go.mod h1:3kT0v4AMwT/OdorfH2gRWPwoOrUX/LV03HEeBsaXG1c=
+git.gammaspectra.live/P2Pool/moneroutil v0.0.0-20230722215223-18ecc51ae61e h1:ropqS9niQR/ZKCUrlmWe+uDH0fLIyAnCIjkEjyTDgA8=
+git.gammaspectra.live/P2Pool/moneroutil v0.0.0-20230722215223-18ecc51ae61e/go.mod h1:Wn5QI7XIMHMpEu10pPspW9h3eGmXQPJwh/4/+Gi3G1U=
+git.gammaspectra.live/P2Pool/randomx-go-bindings v0.0.0-20230514082649-9c5f18cd5a71 h1:MgeHHcF+GnCJBWMSzq8XAbc8p/UhNwFruEKCPPJ74YQ=
+git.gammaspectra.live/P2Pool/randomx-go-bindings v0.0.0-20230514082649-9c5f18cd5a71/go.mod h1:KQaYHIxGXNHNMQELC7xGLu8xouwvP/dN7iGk681BXmk=
+git.gammaspectra.live/P2Pool/sha3 v0.0.0-20230604092430-04fe7dc6439a h1:c24MHv/z+aBYpYNsQHcJqmFuaYInGVixJZgDCXA/4bs=
+git.gammaspectra.live/P2Pool/sha3 v0.0.0-20230604092430-04fe7dc6439a/go.mod h1:6wZ0+whl+HZdcRve4R6Rq6jV1fmL1xCYO8Wty6lR008=
+github.com/WeebDataHoarder/go-json v0.0.0-20230730135821-d8f6463bb887 h1:P01nqSM+0b6zlPasOFYsqnQSP2dTRVZkanAnY9q/Bcc=
+github.com/WeebDataHoarder/go-json v0.0.0-20230730135821-d8f6463bb887/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
+github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dolthub/maphash v0.1.0 h1:bsQ7JsF4FkkWyrP3oCnFJgrCUAFbFf3kOl4L/QxPDyQ=
+github.com/dolthub/maphash v0.1.0/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4=
+github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc=
+github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ=
+github.com/floatdrop/lru v1.3.0 h1:83abtaKjXcWrPmtzTAk2Ggq8DUKqI29YzrTrB8+vu0c=
+github.com/floatdrop/lru v1.3.0/go.mod h1:83zlXKA06Bm32JImNINCiTr0ldadvdAjUe5jSwIaw0s=
+github.com/holiman/uint256 v1.2.3 h1:K8UWO1HUJpRMXBxbmaY1Y8IAMZC/RsKB+ArEnnK4l5o=
+github.com/holiman/uint256 v1.2.3/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw=
+github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw=
+github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8=
+github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/thepudds/swisstable v0.0.0-20221011152303-9c77dc657777 h1:5u+6YWU2faS+Sr/x8j9yalMpSDUkatNOZWXV3wMUCGQ=
+github.com/thepudds/swisstable v0.0.0-20221011152303-9c77dc657777/go.mod h1:4af3KxEsswy6aTzsTcwa8QZUSh4V+80oHdp1QX9uJHA=
+github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo=
+github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs=
+golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
+golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
+golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo=
+lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
--- a/go.work
+++ b/go.work
@ -3,6 +3,7 @@ go 1.21
 use (
 	.
 	cmd/api
+	cmd/apitocache
 	cmd/archivetoarchive
 	cmd/archivetoindex
 	cmd/cachetoarchive
--- a/p2pool/cache/legacy/legacy.go
+++ b/p2pool/cache/legacy/legacy.go
@ -13,8 +13,8 @@ import (
 )

 const blockSize = 96 * 1024
-const numBlocks = 4608
-const cacheSize = blockSize * numBlocks
+const NumBlocks = 4608
+const cacheSize = blockSize * NumBlocks

 type Cache struct {
 	f                 *os.File
@ -57,7 +57,7 @@ func (c *Cache) Store(block *sidechain.PoolBlock) {
 			//block too big
 			return
 		}
-		storeIndex := (c.storeIndex.Add(1) % numBlocks) * blockSize
+		storeIndex := (c.storeIndex.Add(1) % NumBlocks) * blockSize
 		_, _ = c.f.WriteAt(binary.LittleEndian.AppendUint32(nil, uint32(len(blob))), int64(storeIndex))
 		_, _ = c.f.WriteAt(blob, int64(storeIndex)+4)
 	}
@ -73,8 +73,8 @@ func (c *Cache) LoadAll(l cache.Loadee) {
 		buf := make([]byte, 0, blockSize)

 		var blocksLoaded int
-		for i := 0; i < numBlocks; i++ {
-			storeIndex := (c.storeIndex.Add(1) % numBlocks) * blockSize
+		for i := 0; i < NumBlocks; i++ {
+			storeIndex := (c.storeIndex.Add(1) % NumBlocks) * blockSize

 			if _, err := c.f.ReadAt(blobLen[:], int64(storeIndex)); err != nil {
 				return