Initial commit, WiP, needs docs

LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2023 WeebDataHoarder
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

go.mod

@@ -0,0 +1,3 @@
+module monero-blocks
+
+go 1.19

main.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"encoding/csv"
+	"flag"
+	"log"
+	"monero-blocks/pool"
+	c3pool_org "monero-blocks/pool/c3pool.org"
+	monero_hashvault_pro "monero-blocks/pool/monero.hashvault.pro"
+	moneroocean_stream "monero-blocks/pool/moneroocean.stream"
+	xmr_2miners_com "monero-blocks/pool/xmr.2miners.com"
+	xmr_nanopool_org "monero-blocks/pool/xmr.nanopool.org"
+	xmrpool_eu "monero-blocks/pool/xmrpool.eu"
+	"os"
+	"strconv"
+	"sync"
+)
+
+func main() {
+	scanDownToHeight := flag.Uint64("height", 2688888, "Height at which scans will stop from the tip. Defaults to v15 upgrade.")
+
+	flag.Parse()
+
+	pools := []pool.Pool{
+		monero_hashvault_pro.New(),
+		xmr_nanopool_org.New(),
+		xmr_2miners_com.New(),
+		xmrpool_eu.New(),
+		c3pool_org.New(),
+		moneroocean_stream.New(),
+	}
+
+	var wg sync.WaitGroup
+
+	allBlocks := make([][]pool.Block, len(pools))
+
+	lowerHeight := *scanDownToHeight
+
+	for i, p := range pools {
+		wg.Add(1)
+		go func(pIndex int, p pool.Pool) {
+			defer wg.Done()
+			var token pool.Token
+			var tempBlocks []pool.Block
+			var lastBlock uint64
+			for {
+				tempBlocks, token = p.GetBlocks(token)
+				for _, b := range tempBlocks {
+					if b.Height < lowerHeight {
+						log.Printf("[%s] Finished: reached height\n", p.Name())
+						return
+					}
+					allBlocks[pIndex] = append(allBlocks[pIndex], b)
+					lastBlock = b.Height
+				}
+				log.Printf("[%s] at %d/%d\n", p.Name(), lastBlock, lowerHeight)
+				if token == nil {
+					log.Printf("[%s] Finished: no more blocks\n", p.Name())
+					return
+				}
+			}
+		}(i, p)
+	}
+
+	wg.Wait()
+
+	f, err := os.Create("blocks.csv")
+	if err != nil {
+		log.Panic(err)
+	}
+	defer f.Close()
+	csvFile := csv.NewWriter(f)
+	defer csvFile.Flush()
+
+	csvFile.Write([]string{"Height", "Id", "Timestamp", "Reward", "Pool"})
+	for {
+		smallIndex := -1
+		smallValue := uint64(0)
+		for i, s := range allBlocks {
+			if len(s) > 0 {
+				if s[0].Height >= smallValue {
+					smallValue = s[0].Height
+					smallIndex = i
+				}
+			}
+		}
+		if smallIndex == -1 {
+			break
+		}
+		csvFile.Write([]string{strconv.FormatUint(allBlocks[smallIndex][0].Height, 10), allBlocks[smallIndex][0].Id.String(), strconv.FormatUint(allBlocks[smallIndex][0].Height, 10), strconv.FormatUint(allBlocks[smallIndex][0].Height, 10), pools[smallIndex].Name()})
+		allBlocks[smallIndex] = allBlocks[smallIndex][1:]
+	}
+
+}

pool/block.go

@@ -0,0 +1,8 @@
+package pool
+
+type Block struct {
+	Id        Hash
+	Height    uint64
+	Reward    uint64
+	Timestamp uint64
+}

pool/c3pool.org/pool.go

@@ -0,0 +1,99 @@
+package c3pool_org
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"monero-blocks/pool"
+	"net/http"
+	"time"
+)
+
+type Pool struct {
+	throttler <-chan time.Time
+}
+
+type pagingToken struct {
+	page   uint64
+	id     pool.Hash
+	height uint64
+}
+
+type blockJson struct {
+	Ts     uint64    `json:"ts"`
+	Hash   pool.Hash `json:"hash"`
+	Height uint64    `json:"height"`
+	Valid  bool      `json:"valid"`
+	Value  uint64    `json:"value"`
+}
+
+func New() *Pool {
+	return &Pool{
+		throttler: time.Tick(time.Second * 5), //One request every five seconds
+	}
+}
+
+func (p *Pool) Name() string {
+	return "c3pool.org"
+}
+
+func (p *Pool) GetBlocks(token pool.Token) ([]pool.Block, pool.Token) {
+
+	var t *pagingToken
+	var ok bool
+
+	var page uint64
+
+	if t, ok = token.(*pagingToken); token != nil && ok {
+		page = t.page
+	} else {
+		t = &pagingToken{}
+	}
+
+	<-p.throttler
+	response, err := http.DefaultClient.Get(fmt.Sprintf("https://api.c3pool.org/pool/blocks?page=%d&limit=500", page))
+	if err != nil {
+		return nil, nil
+	}
+	defer response.Body.Close()
+
+	blockData := make([]blockJson, 0, 500)
+
+	if data, err := io.ReadAll(response.Body); err != nil {
+		return nil, nil
+	} else {
+		if err = json.Unmarshal(data, &blockData); err != nil {
+			return nil, nil
+		}
+	}
+
+	var blocks []pool.Block
+
+	start := t.id == pool.ZeroHash
+	for _, b := range blockData {
+		if b.Height < t.height {
+			start = true
+		}
+		if start && b.Valid {
+			blocks = append(blocks, pool.Block{
+				Id:        b.Hash,
+				Height:    b.Height,
+				Reward:    b.Value,
+				Timestamp: b.Ts,
+			})
+		}
+		if b.Hash == t.id {
+			start = true
+		}
+	}
+
+	if len(blocks) == 0 {
+		return nil, nil
+	}
+
+	return blocks, &pagingToken{
+		id:     blocks[len(blocks)-1].Id,
+		page:   page + 1,
+		height: blocks[len(blocks)-1].Height,
+	}
+}

pool/hash.go

@@ -0,0 +1,65 @@
+package pool
+
+import (
+	"bytes"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+)
+
+const HashSize = 32
+
+type Hash [HashSize]byte
+
+var ZeroHash Hash
+
+func (h Hash) MarshalJSON() ([]byte, error) {
+	return json.Marshal(h.String())
+}
+
+func HashFromString(s string) (Hash, error) {
+	var h Hash
+	if buf, err := hex.DecodeString(s); err != nil {
+		return h, err
+	} else {
+		if len(buf) != HashSize {
+			return h, errors.New("wrong hash size")
+		}
+		copy(h[:], buf)
+		return h, nil
+	}
+}
+
+func HashFromBytes(buf []byte) (h Hash) {
+	if len(buf) != HashSize {
+		return
+	}
+	copy(h[:], buf)
+	return
+}
+
+func (h Hash) Equals(o Hash) bool {
+	return bytes.Compare(h[:], o[:]) == 0
+}
+
+func (h Hash) String() string {
+	return hex.EncodeToString(h[:])
+}
+
+func (h *Hash) UnmarshalJSON(b []byte) error {
+	var s string
+	if err := json.Unmarshal(b, &s); err != nil {
+		return err
+	}
+
+	if buf, err := hex.DecodeString(s); err != nil {
+		return err
+	} else {
+		if len(buf) != HashSize {
+			return errors.New("wrong hash size")
+		}
+
+		copy(h[:], buf)
+		return nil
+	}
+}

pool/monero.hashvault.pro/pool.go

@@ -0,0 +1,98 @@
+package monero_hashvault_pro
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"monero-blocks/pool"
+	"net/http"
+	"time"
+)
+
+type Pool struct {
+	throttler <-chan time.Time
+}
+
+type pagingToken struct {
+	page   uint64
+	id     pool.Hash
+	height uint64
+}
+
+type blockJson struct {
+	Ts     uint64    `json:"ts"`
+	Hash   pool.Hash `json:"hash"`
+	Height uint64    `json:"height"`
+	Valid  bool      `json:"valid"`
+	Value  uint64    `json:"value"`
+}
+
+func New() *Pool {
+	return &Pool{
+		throttler: time.Tick(time.Second * 5), //One request every five seconds
+	}
+}
+
+func (p *Pool) Name() string {
+	return "monero.hashvault.pro"
+}
+
+func (p *Pool) GetBlocks(token pool.Token) ([]pool.Block, pool.Token) {
+	var t *pagingToken
+	var ok bool
+
+	var page uint64
+
+	if t, ok = token.(*pagingToken); token != nil && ok {
+		page = t.page
+	} else {
+		t = &pagingToken{}
+	}
+
+	<-p.throttler
+	response, err := http.DefaultClient.Get(fmt.Sprintf("https://api.hashvault.pro/v3/monero/pool/blocks?limit=500&page=%d", page))
+	if err != nil {
+		return nil, nil
+	}
+	defer response.Body.Close()
+
+	blockData := make([]blockJson, 0, 500)
+
+	if data, err := io.ReadAll(response.Body); err != nil {
+		return nil, nil
+	} else {
+		if err = json.Unmarshal(data, &blockData); err != nil {
+			return nil, nil
+		}
+	}
+
+	var blocks []pool.Block
+
+	start := t.id == pool.ZeroHash
+	for _, b := range blockData {
+		if b.Height < t.height {
+			start = true
+		}
+		if start && b.Valid {
+			blocks = append(blocks, pool.Block{
+				Id:        b.Hash,
+				Height:    b.Height,
+				Reward:    b.Value,
+				Timestamp: b.Ts,
+			})
+		}
+		if b.Hash == t.id {
+			start = true
+		}
+	}
+
+	if len(blocks) == 0 {
+		return nil, nil
+	}
+
+	return blocks, &pagingToken{
+		id:     blocks[len(blocks)-1].Id,
+		page:   page + 1,
+		height: blocks[len(blocks)-1].Height,
+	}
+}

pool/moneroocean.stream/pool.go

@@ -0,0 +1,99 @@
+package moneroocean_stream
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"monero-blocks/pool"
+	"net/http"
+	"time"
+)
+
+type Pool struct {
+	throttler <-chan time.Time
+}
+
+type pagingToken struct {
+	page   uint64
+	id     pool.Hash
+	height uint64
+}
+
+type blockJson struct {
+	Ts     uint64    `json:"ts"`
+	Hash   pool.Hash `json:"hash"`
+	Height uint64    `json:"height"`
+	Valid  bool      `json:"valid"`
+	Value  uint64    `json:"value"`
+}
+
+func New() *Pool {
+	return &Pool{
+		throttler: time.Tick(time.Second * 5), //One request every five seconds
+	}
+}
+
+func (p *Pool) Name() string {
+	return "moneroocean.stream"
+}
+
+func (p *Pool) GetBlocks(token pool.Token) ([]pool.Block, pool.Token) {
+
+	var t *pagingToken
+	var ok bool
+
+	var page uint64
+
+	if t, ok = token.(*pagingToken); token != nil && ok {
+		page = t.page
+	} else {
+		t = &pagingToken{}
+	}
+
+	<-p.throttler
+	response, err := http.DefaultClient.Get(fmt.Sprintf("https://api.moneroocean.stream/pool/blocks?page=%d&limit=500", page))
+	if err != nil {
+		return nil, nil
+	}
+	defer response.Body.Close()
+
+	blockData := make([]blockJson, 0, 500)
+
+	if data, err := io.ReadAll(response.Body); err != nil {
+		return nil, nil
+	} else {
+		if err = json.Unmarshal(data, &blockData); err != nil {
+			return nil, nil
+		}
+	}
+
+	var blocks []pool.Block
+
+	start := t.id == pool.ZeroHash
+	for _, b := range blockData {
+		if b.Height < t.height {
+			start = true
+		}
+		if start && b.Valid {
+			blocks = append(blocks, pool.Block{
+				Id:        b.Hash,
+				Height:    b.Height,
+				Reward:    b.Value,
+				Timestamp: b.Ts,
+			})
+		}
+		if b.Hash == t.id {
+			start = true
+		}
+	}
+
+	if len(blocks) == 0 {
+		return nil, nil
+	}
+
+	return blocks, &pagingToken{
+		id:     blocks[len(blocks)-1].Id,
+		page:   page + 1,
+		height: blocks[len(blocks)-1].Height,
+	}
+}

pool/pool.go

@@ -0,0 +1,9 @@
+package pool
+
+type Pool interface {
+	Name() string
+	GetBlocks(token Token) ([]Block, Token)
+}
+
+// Token Used to pass paging information between calls
+type Token any

pool/xmr.2miners.com/pool.go

@@ -0,0 +1,72 @@
+package xmr_2miners_com
+
+import (
+	"encoding/json"
+	"io"
+	"monero-blocks/pool"
+	"net/http"
+	"time"
+)
+
+type Pool struct {
+	throttler <-chan time.Time
+}
+
+type blocksJson struct {
+	Matured []blockJson `json:"matured"`
+}
+
+type blockJson struct {
+	Ts     uint64    `json:"timestamp"`
+	Hash   pool.Hash `json:"hash"`
+	Height uint64    `json:"height"`
+	Value  uint64    `json:"reward"`
+}
+
+func New() *Pool {
+	return &Pool{
+		throttler: time.Tick(time.Second * 5), //One request every five seconds
+	}
+}
+
+func (p *Pool) Name() string {
+	return "xmr.2miners.com"
+}
+
+// GetBlocks does not support paging
+func (p *Pool) GetBlocks(token pool.Token) ([]pool.Block, pool.Token) {
+
+	<-p.throttler
+	response, err := http.DefaultClient.Get("https://xmr.2miners.com/api/blocks")
+	if err != nil {
+		return nil, nil
+	}
+	defer response.Body.Close()
+
+	var blockData blocksJson
+
+	if data, err := io.ReadAll(response.Body); err != nil {
+		return nil, nil
+	} else {
+		if err = json.Unmarshal(data, &blockData); err != nil {
+			return nil, nil
+		}
+	}
+
+	var blocks []pool.Block
+
+	for _, b := range blockData.Matured {
+		blocks = append(blocks, pool.Block{
+			Id:        b.Hash,
+			Height:    b.Height,
+			Reward:    b.Value,
+			Timestamp: b.Ts * 1000,
+		})
+	}
+
+	if len(blocks) == 0 {
+		return nil, nil
+	}
+
+	return blocks, nil
+}

pool/xmr.nanopool.org/pool.go

@@ -0,0 +1,102 @@
+package xmr_nanopool_org
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"monero-blocks/pool"
+	"net/http"
+	"time"
+)
+
+type Pool struct {
+	throttler <-chan time.Time
+}
+
+type pagingToken struct {
+	page   uint64
+	id     pool.Hash
+	height uint64
+}
+
+type blocksJson struct {
+	Data []blockJson `json:"data"`
+}
+
+type blockJson struct {
+	Ts     uint64    `json:"date"`
+	Hash   pool.Hash `json:"hash"`
+	Height uint64    `json:"block_number"`
+	Status int       `json:"status"`
+	Value  float64   `json:"value"`
+}
+
+func New() *Pool {
+	return &Pool{
+		throttler: time.Tick(time.Second * 5), //One request every five seconds
+	}
+}
+
+func (p *Pool) Name() string {
+	return "xmr.nanopool.org"
+}
+
+func (p *Pool) GetBlocks(token pool.Token) ([]pool.Block, pool.Token) {
+	var t *pagingToken
+	var ok bool
+
+	var page uint64
+
+	if t, ok = token.(*pagingToken); token != nil && ok {
+		page = t.page
+	} else {
+		t = &pagingToken{}
+	}
+
+	<-p.throttler
+	response, err := http.DefaultClient.Get(fmt.Sprintf("https://xmr.nanopool.org/api/v1/pool/blocks/%d/%d", page*500, 500))
+	if err != nil {
+		return nil, nil
+	}
+	defer response.Body.Close()
+
+	var blockData blocksJson
+
+	if data, err := io.ReadAll(response.Body); err != nil {
+		return nil, nil
+	} else {
+		if err = json.Unmarshal(data, &blockData); err != nil {
+			return nil, nil
+		}
+	}
+
+	var blocks []pool.Block
+
+	start := t.id == pool.ZeroHash
+	for _, b := range blockData.Data {
+		if b.Height < t.height {
+			start = true
+		}
+		if start && b.Status == 1 {
+			blocks = append(blocks, pool.Block{
+				Id:        b.Hash,
+				Height:    b.Height,
+				Reward:    uint64(b.Value * 10000000000),
+				Timestamp: b.Ts * 1000,
+			})
+		}
+		if b.Hash == t.id {
+			start = true
+		}
+	}
+
+	if len(blocks) == 0 {
+		return nil, nil
+	}
+
+	return blocks, &pagingToken{
+		id:     blocks[len(blocks)-1].Id,
+		page:   page + 1,
+		height: blocks[len(blocks)-1].Height,
+	}
+}

pool/xmrpool.eu/pool.go

@@ -0,0 +1,117 @@
+package xmrpool_eu
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"math"
+	"monero-blocks/pool"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type Pool struct {
+	throttler <-chan time.Time
+}
+
+type pagingToken struct {
+	height uint64
+	id     pool.Hash
+}
+
+type blockJson struct {
+	Ts     uint64    `json:"ts"`
+	Hash   pool.Hash `json:"hash"`
+	Height uint64    `json:"height"`
+	Valid  bool      `json:"valid"`
+	Value  uint64    `json:"value"`
+}
+
+func New() *Pool {
+	return &Pool{
+		throttler: time.Tick(time.Second * 5), //One request every five seconds
+	}
+}
+
+func (p *Pool) Name() string {
+	return "xmrpool.eu"
+}
+
+func (p *Pool) GetBlocks(token pool.Token) ([]pool.Block, pool.Token) {
+
+	var t *pagingToken
+	var ok bool
+
+	var height uint64 = math.MaxInt32
+	if t, ok = token.(*pagingToken); token != nil && ok {
+		height = t.height
+	} else {
+		t = &pagingToken{}
+	}
+
+	<-p.throttler
+	response, err := http.DefaultClient.Get(fmt.Sprintf("https://web.xmrpool.eu:8119/get_blocks?height=%d", height))
+	if err != nil {
+		return nil, nil
+	}
+	defer response.Body.Close()
+
+	var blockData []string
+
+	if data, err := io.ReadAll(response.Body); err != nil {
+		return nil, nil
+	} else {
+		if err = json.Unmarshal(data, &blockData); err != nil || (len(blockData)%2 != 0) {
+			return nil, nil
+		}
+	}
+
+	var blocks []pool.Block
+
+	for i := 0; i < len(blockData); i += 2 {
+		pieces := strings.Split(blockData[i], ":")
+		if len(pieces) < 4 {
+			return nil, nil
+		}
+		if len(pieces) < 6 {
+			break
+		}
+		hash, _ := pool.HashFromString(pieces[0])
+		ts, _ := strconv.ParseUint(pieces[1], 10, 0)
+		blockHeight, _ := strconv.ParseUint(blockData[i+1], 10, 0)
+		reward, _ := strconv.ParseUint(pieces[5], 10, 0)
+		blocks = append(blocks, pool.Block{
+			Id:        hash,
+			Height:    blockHeight,
+			Reward:    reward,
+			Timestamp: ts * 1000,
+		})
+	}
+
+	start := t.id == pool.ZeroHash
+	for i, b := range blocks {
+		if b.Height < t.height {
+			start = true
+		}
+		if start {
+			return blocks[i:], &pagingToken{
+				id:     blocks[len(blocks)-1].Id,
+				height: blocks[len(blocks)-1].Height,
+			}
+		}
+		if b.Id == t.id {
+			start = true
+		}
+	}
+
+	if len(blocks) == 0 {
+		return nil, nil
+	}
+
+	return nil, &pagingToken{
+		id:     blocks[len(blocks)-1].Id,
+		height: blocks[len(blocks)-1].Height,
+	}
+}