commit 3aae8d71ad592b986b6ad6aa2076ef0887b138d5 Author: WeebDataHoarder <57538841+WeebDataHoarder@users.noreply.github.com> Date: Mon Jan 2 19:11:48 2023 +0100 Initial commit, WiP, needs docs diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f6e7199 --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2023 WeebDataHoarder + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..20ee1ad --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module monero-blocks + +go 1.19 diff --git a/main.go b/main.go new file mode 100644 index 0000000..a1c626b --- /dev/null +++ b/main.go @@ -0,0 +1,94 @@ +package main + +import ( + "encoding/csv" + "flag" + "log" + "monero-blocks/pool" + c3pool_org "monero-blocks/pool/c3pool.org" + monero_hashvault_pro "monero-blocks/pool/monero.hashvault.pro" + moneroocean_stream "monero-blocks/pool/moneroocean.stream" + xmr_2miners_com "monero-blocks/pool/xmr.2miners.com" + xmr_nanopool_org "monero-blocks/pool/xmr.nanopool.org" + xmrpool_eu "monero-blocks/pool/xmrpool.eu" + "os" + "strconv" + "sync" +) + +func main() { + scanDownToHeight := flag.Uint64("height", 2688888, "Height at which scans will stop from the tip. Defaults to v15 upgrade.") + + flag.Parse() + + pools := []pool.Pool{ + monero_hashvault_pro.New(), + xmr_nanopool_org.New(), + xmr_2miners_com.New(), + xmrpool_eu.New(), + c3pool_org.New(), + moneroocean_stream.New(), + } + + var wg sync.WaitGroup + + allBlocks := make([][]pool.Block, len(pools)) + + lowerHeight := *scanDownToHeight + + for i, p := range pools { + wg.Add(1) + go func(pIndex int, p pool.Pool) { + defer wg.Done() + var token pool.Token + var tempBlocks []pool.Block + var lastBlock uint64 + for { + tempBlocks, token = p.GetBlocks(token) + for _, b := range tempBlocks { + if b.Height < lowerHeight { + log.Printf("[%s] Finished: reached height\n", p.Name()) + return + } + allBlocks[pIndex] = append(allBlocks[pIndex], b) + lastBlock = b.Height + } + log.Printf("[%s] at %d/%d\n", p.Name(), lastBlock, lowerHeight) + if token == nil { + log.Printf("[%s] Finished: no more blocks\n", p.Name()) + return + } + } + }(i, p) + } + + wg.Wait() + + f, err := os.Create("blocks.csv") + if err != nil { + log.Panic(err) + } + defer f.Close() + csvFile := csv.NewWriter(f) + defer csvFile.Flush() + + csvFile.Write([]string{"Height", "Id", "Timestamp", "Reward", "Pool"}) + for { + smallIndex := -1 + smallValue := uint64(0) + for i, s := range allBlocks { + if len(s) > 0 { + if s[0].Height >= smallValue { + smallValue = s[0].Height + smallIndex = i + } + } + } + if smallIndex == -1 { + break + } + csvFile.Write([]string{strconv.FormatUint(allBlocks[smallIndex][0].Height, 10), allBlocks[smallIndex][0].Id.String(), strconv.FormatUint(allBlocks[smallIndex][0].Height, 10), strconv.FormatUint(allBlocks[smallIndex][0].Height, 10), pools[smallIndex].Name()}) + allBlocks[smallIndex] = allBlocks[smallIndex][1:] + } + +} diff --git a/pool/block.go b/pool/block.go new file mode 100644 index 0000000..8dcb99e --- /dev/null +++ b/pool/block.go @@ -0,0 +1,8 @@ +package pool + +type Block struct { + Id Hash + Height uint64 + Reward uint64 + Timestamp uint64 +} diff --git a/pool/c3pool.org/pool.go b/pool/c3pool.org/pool.go new file mode 100644 index 0000000..2eb58a6 --- /dev/null +++ b/pool/c3pool.org/pool.go @@ -0,0 +1,99 @@ +package c3pool_org + +import ( + "encoding/json" + "fmt" + "io" + "monero-blocks/pool" + "net/http" + "time" +) + +type Pool struct { + throttler <-chan time.Time +} + +type pagingToken struct { + page uint64 + id pool.Hash + height uint64 +} + +type blockJson struct { + Ts uint64 `json:"ts"` + Hash pool.Hash `json:"hash"` + Height uint64 `json:"height"` + Valid bool `json:"valid"` + Value uint64 `json:"value"` +} + +func New() *Pool { + return &Pool{ + throttler: time.Tick(time.Second * 5), //One request every five seconds + } +} + +func (p *Pool) Name() string { + return "c3pool.org" +} + +func (p *Pool) GetBlocks(token pool.Token) ([]pool.Block, pool.Token) { + + var t *pagingToken + var ok bool + + var page uint64 + + if t, ok = token.(*pagingToken); token != nil && ok { + page = t.page + } else { + t = &pagingToken{} + } + + <-p.throttler + response, err := http.DefaultClient.Get(fmt.Sprintf("https://api.c3pool.org/pool/blocks?page=%d&limit=500", page)) + if err != nil { + return nil, nil + } + defer response.Body.Close() + + blockData := make([]blockJson, 0, 500) + + if data, err := io.ReadAll(response.Body); err != nil { + return nil, nil + } else { + if err = json.Unmarshal(data, &blockData); err != nil { + return nil, nil + } + } + + var blocks []pool.Block + + start := t.id == pool.ZeroHash + for _, b := range blockData { + if b.Height < t.height { + start = true + } + if start && b.Valid { + blocks = append(blocks, pool.Block{ + Id: b.Hash, + Height: b.Height, + Reward: b.Value, + Timestamp: b.Ts, + }) + } + if b.Hash == t.id { + start = true + } + } + + if len(blocks) == 0 { + return nil, nil + } + + return blocks, &pagingToken{ + id: blocks[len(blocks)-1].Id, + page: page + 1, + height: blocks[len(blocks)-1].Height, + } +} diff --git a/pool/hash.go b/pool/hash.go new file mode 100644 index 0000000..6933754 --- /dev/null +++ b/pool/hash.go @@ -0,0 +1,65 @@ +package pool + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "errors" +) + +const HashSize = 32 + +type Hash [HashSize]byte + +var ZeroHash Hash + +func (h Hash) MarshalJSON() ([]byte, error) { + return json.Marshal(h.String()) +} + +func HashFromString(s string) (Hash, error) { + var h Hash + if buf, err := hex.DecodeString(s); err != nil { + return h, err + } else { + if len(buf) != HashSize { + return h, errors.New("wrong hash size") + } + copy(h[:], buf) + return h, nil + } +} + +func HashFromBytes(buf []byte) (h Hash) { + if len(buf) != HashSize { + return + } + copy(h[:], buf) + return +} + +func (h Hash) Equals(o Hash) bool { + return bytes.Compare(h[:], o[:]) == 0 +} + +func (h Hash) String() string { + return hex.EncodeToString(h[:]) +} + +func (h *Hash) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + + if buf, err := hex.DecodeString(s); err != nil { + return err + } else { + if len(buf) != HashSize { + return errors.New("wrong hash size") + } + + copy(h[:], buf) + return nil + } +} diff --git a/pool/monero.hashvault.pro/pool.go b/pool/monero.hashvault.pro/pool.go new file mode 100644 index 0000000..d824cc9 --- /dev/null +++ b/pool/monero.hashvault.pro/pool.go @@ -0,0 +1,98 @@ +package monero_hashvault_pro + +import ( + "encoding/json" + "fmt" + "io" + "monero-blocks/pool" + "net/http" + "time" +) + +type Pool struct { + throttler <-chan time.Time +} + +type pagingToken struct { + page uint64 + id pool.Hash + height uint64 +} + +type blockJson struct { + Ts uint64 `json:"ts"` + Hash pool.Hash `json:"hash"` + Height uint64 `json:"height"` + Valid bool `json:"valid"` + Value uint64 `json:"value"` +} + +func New() *Pool { + return &Pool{ + throttler: time.Tick(time.Second * 5), //One request every five seconds + } +} + +func (p *Pool) Name() string { + return "monero.hashvault.pro" +} + +func (p *Pool) GetBlocks(token pool.Token) ([]pool.Block, pool.Token) { + var t *pagingToken + var ok bool + + var page uint64 + + if t, ok = token.(*pagingToken); token != nil && ok { + page = t.page + } else { + t = &pagingToken{} + } + + <-p.throttler + response, err := http.DefaultClient.Get(fmt.Sprintf("https://api.hashvault.pro/v3/monero/pool/blocks?limit=500&page=%d", page)) + if err != nil { + return nil, nil + } + defer response.Body.Close() + + blockData := make([]blockJson, 0, 500) + + if data, err := io.ReadAll(response.Body); err != nil { + return nil, nil + } else { + if err = json.Unmarshal(data, &blockData); err != nil { + return nil, nil + } + } + + var blocks []pool.Block + + start := t.id == pool.ZeroHash + for _, b := range blockData { + if b.Height < t.height { + start = true + } + if start && b.Valid { + blocks = append(blocks, pool.Block{ + Id: b.Hash, + Height: b.Height, + Reward: b.Value, + Timestamp: b.Ts, + }) + } + if b.Hash == t.id { + start = true + } + } + + if len(blocks) == 0 { + return nil, nil + } + + return blocks, &pagingToken{ + id: blocks[len(blocks)-1].Id, + page: page + 1, + height: blocks[len(blocks)-1].Height, + } +} diff --git a/pool/moneroocean.stream/pool.go b/pool/moneroocean.stream/pool.go new file mode 100644 index 0000000..cd1b71b --- /dev/null +++ b/pool/moneroocean.stream/pool.go @@ -0,0 +1,99 @@ +package moneroocean_stream + +import ( + "encoding/json" + "fmt" + "io" + "monero-blocks/pool" + "net/http" + "time" +) + +type Pool struct { + throttler <-chan time.Time +} + +type pagingToken struct { + page uint64 + id pool.Hash + height uint64 +} + +type blockJson struct { + Ts uint64 `json:"ts"` + Hash pool.Hash `json:"hash"` + Height uint64 `json:"height"` + Valid bool `json:"valid"` + Value uint64 `json:"value"` +} + +func New() *Pool { + return &Pool{ + throttler: time.Tick(time.Second * 5), //One request every five seconds + } +} + +func (p *Pool) Name() string { + return "moneroocean.stream" +} + +func (p *Pool) GetBlocks(token pool.Token) ([]pool.Block, pool.Token) { + + var t *pagingToken + var ok bool + + var page uint64 + + if t, ok = token.(*pagingToken); token != nil && ok { + page = t.page + } else { + t = &pagingToken{} + } + + <-p.throttler + response, err := http.DefaultClient.Get(fmt.Sprintf("https://api.moneroocean.stream/pool/blocks?page=%d&limit=500", page)) + if err != nil { + return nil, nil + } + defer response.Body.Close() + + blockData := make([]blockJson, 0, 500) + + if data, err := io.ReadAll(response.Body); err != nil { + return nil, nil + } else { + if err = json.Unmarshal(data, &blockData); err != nil { + return nil, nil + } + } + + var blocks []pool.Block + + start := t.id == pool.ZeroHash + for _, b := range blockData { + if b.Height < t.height { + start = true + } + if start && b.Valid { + blocks = append(blocks, pool.Block{ + Id: b.Hash, + Height: b.Height, + Reward: b.Value, + Timestamp: b.Ts, + }) + } + if b.Hash == t.id { + start = true + } + } + + if len(blocks) == 0 { + return nil, nil + } + + return blocks, &pagingToken{ + id: blocks[len(blocks)-1].Id, + page: page + 1, + height: blocks[len(blocks)-1].Height, + } +} diff --git a/pool/pool.go b/pool/pool.go new file mode 100644 index 0000000..4cf3c5c --- /dev/null +++ b/pool/pool.go @@ -0,0 +1,9 @@ +package pool + +type Pool interface { + Name() string + GetBlocks(token Token) ([]Block, Token) +} + +// Token Used to pass paging information between calls +type Token any diff --git a/pool/xmr.2miners.com/pool.go b/pool/xmr.2miners.com/pool.go new file mode 100644 index 0000000..8f3427a --- /dev/null +++ b/pool/xmr.2miners.com/pool.go @@ -0,0 +1,72 @@ +package xmr_2miners_com + +import ( + "encoding/json" + "io" + "monero-blocks/pool" + "net/http" + "time" +) + +type Pool struct { + throttler <-chan time.Time +} + +type blocksJson struct { + Matured []blockJson `json:"matured"` +} + +type blockJson struct { + Ts uint64 `json:"timestamp"` + Hash pool.Hash `json:"hash"` + Height uint64 `json:"height"` + Value uint64 `json:"reward"` +} + +func New() *Pool { + return &Pool{ + throttler: time.Tick(time.Second * 5), //One request every five seconds + } +} + +func (p *Pool) Name() string { + return "xmr.2miners.com" +} + +// GetBlocks does not support paging +func (p *Pool) GetBlocks(token pool.Token) ([]pool.Block, pool.Token) { + + <-p.throttler + response, err := http.DefaultClient.Get("https://xmr.2miners.com/api/blocks") + if err != nil { + return nil, nil + } + defer response.Body.Close() + + var blockData blocksJson + + if data, err := io.ReadAll(response.Body); err != nil { + return nil, nil + } else { + if err = json.Unmarshal(data, &blockData); err != nil { + return nil, nil + } + } + + var blocks []pool.Block + + for _, b := range blockData.Matured { + blocks = append(blocks, pool.Block{ + Id: b.Hash, + Height: b.Height, + Reward: b.Value, + Timestamp: b.Ts * 1000, + }) + } + + if len(blocks) == 0 { + return nil, nil + } + + return blocks, nil +} diff --git a/pool/xmr.nanopool.org/pool.go b/pool/xmr.nanopool.org/pool.go new file mode 100644 index 0000000..c6fa931 --- /dev/null +++ b/pool/xmr.nanopool.org/pool.go @@ -0,0 +1,102 @@ +package xmr_nanopool_org + +import ( + "encoding/json" + "fmt" + "io" + "monero-blocks/pool" + "net/http" + "time" +) + +type Pool struct { + throttler <-chan time.Time +} + +type pagingToken struct { + page uint64 + id pool.Hash + height uint64 +} + +type blocksJson struct { + Data []blockJson `json:"data"` +} + +type blockJson struct { + Ts uint64 `json:"date"` + Hash pool.Hash `json:"hash"` + Height uint64 `json:"block_number"` + Status int `json:"status"` + Value float64 `json:"value"` +} + +func New() *Pool { + return &Pool{ + throttler: time.Tick(time.Second * 5), //One request every five seconds + } +} + +func (p *Pool) Name() string { + return "xmr.nanopool.org" +} + +func (p *Pool) GetBlocks(token pool.Token) ([]pool.Block, pool.Token) { + var t *pagingToken + var ok bool + + var page uint64 + + if t, ok = token.(*pagingToken); token != nil && ok { + page = t.page + } else { + t = &pagingToken{} + } + + <-p.throttler + response, err := http.DefaultClient.Get(fmt.Sprintf("https://xmr.nanopool.org/api/v1/pool/blocks/%d/%d", page*500, 500)) + if err != nil { + return nil, nil + } + defer response.Body.Close() + + var blockData blocksJson + + if data, err := io.ReadAll(response.Body); err != nil { + return nil, nil + } else { + if err = json.Unmarshal(data, &blockData); err != nil { + return nil, nil + } + } + + var blocks []pool.Block + + start := t.id == pool.ZeroHash + for _, b := range blockData.Data { + if b.Height < t.height { + start = true + } + if start && b.Status == 1 { + blocks = append(blocks, pool.Block{ + Id: b.Hash, + Height: b.Height, + Reward: uint64(b.Value * 10000000000), + Timestamp: b.Ts * 1000, + }) + } + if b.Hash == t.id { + start = true + } + } + + if len(blocks) == 0 { + return nil, nil + } + + return blocks, &pagingToken{ + id: blocks[len(blocks)-1].Id, + page: page + 1, + height: blocks[len(blocks)-1].Height, + } +} diff --git a/pool/xmrpool.eu/pool.go b/pool/xmrpool.eu/pool.go new file mode 100644 index 0000000..3a50902 --- /dev/null +++ b/pool/xmrpool.eu/pool.go @@ -0,0 +1,117 @@ +package xmrpool_eu + +import ( + "encoding/json" + "fmt" + "io" + "math" + "monero-blocks/pool" + "net/http" + "strconv" + "strings" + "time" +) + +type Pool struct { + throttler <-chan time.Time +} + +type pagingToken struct { + height uint64 + id pool.Hash +} + +type blockJson struct { + Ts uint64 `json:"ts"` + Hash pool.Hash `json:"hash"` + Height uint64 `json:"height"` + Valid bool `json:"valid"` + Value uint64 `json:"value"` +} + +func New() *Pool { + return &Pool{ + throttler: time.Tick(time.Second * 5), //One request every five seconds + } +} + +func (p *Pool) Name() string { + return "xmrpool.eu" +} + +func (p *Pool) GetBlocks(token pool.Token) ([]pool.Block, pool.Token) { + + var t *pagingToken + var ok bool + + var height uint64 = math.MaxInt32 + if t, ok = token.(*pagingToken); token != nil && ok { + height = t.height + } else { + t = &pagingToken{} + } + + <-p.throttler + response, err := http.DefaultClient.Get(fmt.Sprintf("https://web.xmrpool.eu:8119/get_blocks?height=%d", height)) + if err != nil { + return nil, nil + } + defer response.Body.Close() + + var blockData []string + + if data, err := io.ReadAll(response.Body); err != nil { + return nil, nil + } else { + if err = json.Unmarshal(data, &blockData); err != nil || (len(blockData)%2 != 0) { + return nil, nil + } + } + + var blocks []pool.Block + + for i := 0; i < len(blockData); i += 2 { + pieces := strings.Split(blockData[i], ":") + if len(pieces) < 4 { + return nil, nil + } + if len(pieces) < 6 { + break + } + hash, _ := pool.HashFromString(pieces[0]) + ts, _ := strconv.ParseUint(pieces[1], 10, 0) + blockHeight, _ := strconv.ParseUint(blockData[i+1], 10, 0) + reward, _ := strconv.ParseUint(pieces[5], 10, 0) + blocks = append(blocks, pool.Block{ + Id: hash, + Height: blockHeight, + Reward: reward, + Timestamp: ts * 1000, + }) + } + + start := t.id == pool.ZeroHash + for i, b := range blocks { + if b.Height < t.height { + start = true + } + if start { + return blocks[i:], &pagingToken{ + id: blocks[len(blocks)-1].Id, + height: blocks[len(blocks)-1].Height, + } + } + if b.Id == t.id { + start = true + } + } + + if len(blocks) == 0 { + return nil, nil + } + + return nil, &pagingToken{ + id: blocks[len(blocks)-1].Id, + height: blocks[len(blocks)-1].Height, + } +}