remove crawler; add donate

well, that's not really what the library should be about, so, no reason
to have it here.

see https://github.com/cirocosta/monero-p2p-crawler

Signed-off-by: Ciro S. Costa <utxobr@protonmail.com>
This commit is contained in:
Ciro S. Costa 2021-06-12 09:07:14 -04:00
parent 6d98b46038
commit abdd05f886
9 changed files with 51 additions and 434 deletions

View file

@ -149,3 +149,9 @@ Big thanks to the Monero community and other projects around cryptonote:
- https://github.com/cdiv1e12/py-levin
- https://github.com/cryptonotefoundation/cryptonote
- https://github.com/LeTurt/turtlegod
## Donate
![xmr address](./assets/donate.png)
891B5keCnwXN14hA9FoAzGFtaWmcuLjTDT5aRTp65juBLkbNpEhLNfgcBn6aWdGuBqBnSThqMPsGRjWVQadCrhoAT6CnSL3

BIN
assets/donate.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.4 KiB

View file

@ -1,116 +0,0 @@
package main
import (
"context"
"fmt"
"net"
"os"
"strings"
"time"
"github.com/oschwald/geoip2-golang"
"golang.org/x/net/proxy"
"github.com/cirocosta/go-monero/pkg/crawler"
"github.com/cirocosta/go-monero/pkg/levin"
)
type CrawlCommand struct {
Ip string `long:"ip" default:"198.98.116.72" description:"p2p address"`
Port uint16 `long:"port" default:"18080" description:"p2p port"`
Timeout time.Duration `long:"timeout" default:"20s" description:"maximum execution time"`
Output string `long:"output" default:"nodes.csv" description:"file to write peers to"`
Proxy string `long:"proxy" short:"x" description:"socks5 proxy addr"`
GeoIpDatabase string `long:"geo-ip-db" description:"fpath of a mmdb geoip file"`
}
func (c *CrawlCommand) Execute(_ []string) error {
ctx, cancel := context.WithTimeout(context.Background(), c.Timeout)
defer cancel()
f, err := os.Create(c.Output)
if err != nil {
return fmt.Errorf("create '%s': %w", c.Output, err)
}
defer f.Close()
opts := []levin.ClientOption{}
if c.Proxy != "" {
dialer, err := proxy.SOCKS5("tcp", c.Proxy, nil, nil)
if err != nil {
return fmt.Errorf("socks5 '%s': %w", c.Proxy, err)
}
contextDialer, ok := dialer.(proxy.ContextDialer)
if !ok {
panic("can't cast proxy dialer to proxy context dialer")
}
opts = append(opts, levin.WithContextDialer(contextDialer))
}
ccrawler := crawler.NewCrawler(opts...)
ccrawler.TryPutForVisit(&levin.Peer{
Ip: c.Ip, Port: c.Port,
})
processingFuncs := []func(node *crawler.VisitedPeer) string{
func(node *crawler.VisitedPeer) string {
line := node.Addr() + ","
if node.Error != nil {
line += node.Error.Error()
}
return line
},
}
if c.GeoIpDatabase != "" {
db, err := geoip2.Open(c.GeoIpDatabase)
if err != nil {
return fmt.Errorf("geoip open: %w", err)
}
defer db.Close()
processingFuncs = append(processingFuncs, func(node *crawler.VisitedPeer) string {
ip := net.ParseIP(node.Ip())
record, err := db.Country(ip)
if err != nil {
panic(fmt.Errorf("db city '%s': %w", ip, err))
}
return record.Country.Names["en"]
})
}
go func() {
for node := range ccrawler.C {
columns := []string{}
for _, f := range processingFuncs {
columns = append(columns, f(node))
}
if _, err := f.WriteString(strings.Join(columns, ",") + "\n"); err != nil {
panic(err)
}
}
}()
if _, err := ccrawler.Run(ctx); err != nil {
return fmt.Errorf("crawler run: %w", err)
}
return nil
}
func init() {
parser.AddCommand("crawl",
"Crawl over the network to find all peers",
"Crawl over the network to find all peers",
&CrawlCommand{},
)
}

1
go.mod
View file

@ -4,7 +4,6 @@ go 1.16
require (
github.com/jessevdk/go-flags v1.5.0
github.com/oschwald/geoip2-golang v1.5.0
github.com/sclevine/spec v1.4.0
github.com/sirupsen/logrus v1.8.1
github.com/stretchr/testify v1.7.0

6
go.sum
View file

@ -3,10 +3,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
github.com/oschwald/geoip2-golang v1.5.0 h1:igg2yQIrrcRccB1ytFXqBfOHCjXWIoMv85lVJ1ONZzw=
github.com/oschwald/geoip2-golang v1.5.0/go.mod h1:xdvYt5xQzB8ORWFqPnqMwZpCpgNagttWdoZLlJQzg7s=
github.com/oschwald/maxminddb-golang v1.8.0 h1:Uh/DSnGoxsyp/KYbY1AuP0tYEwfs0sCph9p/UMXK/Hk=
github.com/oschwald/maxminddb-golang v1.8.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8=
@ -15,13 +11,11 @@ github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6 h1:0PC75Fz/kyMGhL0e1QnypqK2kQMqKt9csD1GnMJR+Zk=
golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

View file

@ -1,39 +0,0 @@
#!/bin/bash
set -o errexit
set -o nounset
readonly COMMAND={1:"none"}
main() {
case $COMMAND in
total)
cat ./nodes.csv | wc -l
;;
total-per-country)
cat ./nodes.csv | awk -F ',' '{print $$3}' | sort | uniq -c | sort
;;
reachable)
cat ./nodes.csv | grep -v 'dial' | grep -v 'net' | grep -v 'reset' | wc -l
;;
reachable-per-country)
cat ./nodes.csv | grep -v 'dial' | grep -v 'net' | grep -v 'reset' | awk -F ',' '{print $$3}' | sort | uniq -c | sort
;;
*)
help $0
exit 1
;;
esac
}
help () {
echo "usage: $1 <command>
Commands:
total
total-per-country
reachable
reachable-per-country
"
}
main "$@"

View file

@ -1,258 +0,0 @@
// crawler - crawls all over the p2p network to figure out all the nodes alive
// out there.
//
// the crawler starts from a root connected node and based on the
// `local_peerlist_new` that it receives from a handshake, figures out other
// peers and from that, goes on and on to find more new peers, eventually
// getting a good grasp of all the peers in the network.
//
package crawler
import (
"context"
"fmt"
"net"
"sync"
"time"
log "github.com/sirupsen/logrus"
"github.com/cirocosta/go-monero/pkg/levin"
)
const (
// CrawlerConcurrency determines how many goroutines are spawn to visit
// peers.
//
CrawlerConcurrency = 500
// CrawlerDiscoveryTimeout determines the maximum amount of time that a
// worker trying to visit a peer should take before timing out and
// giving up on it.
//
CrawlerDiscoveryTimeout = 30 * time.Second
)
type WorkerStatuses []byte
func (ws WorkerStatuses) Start(idx int) {
ws[idx] = 1
}
func (ws WorkerStatuses) Finish(idx int) {
ws[idx] = 0
}
func (ws WorkerStatuses) DoingWork() bool {
for _, b := range ws {
if b > 0 {
return true
}
}
return false
}
type VisitedPeer struct {
Peer *levin.Peer
Error error
}
func (n *VisitedPeer) String() string {
return n.Addr()
}
func (n *VisitedPeer) Ip() string {
return n.Peer.Ip
}
func (n *VisitedPeer) Addr() string {
return n.Peer.Addr()
}
type Crawler struct {
C chan *VisitedPeer
clientOptions []levin.ClientOption
visited map[string]*VisitedPeer
notVisited map[string]*levin.Peer
log *log.Entry
workerStatuses WorkerStatuses
sync.Mutex
}
func NewCrawler(clientOpts ...levin.ClientOption) *Crawler {
return &Crawler{
C: make(chan *VisitedPeer, 0),
clientOptions: clientOpts,
visited: map[string]*VisitedPeer{},
notVisited: map[string]*levin.Peer{},
workerStatuses: make(WorkerStatuses, CrawlerConcurrency),
log: log.WithFields(log.Fields{
"component": "crawler",
}),
}
}
func (c *Crawler) TakeNotVisited() *levin.Peer {
c.Lock()
defer c.Unlock()
for k, peer := range c.notVisited {
delete(c.notVisited, k)
return peer
}
return nil
}
func (c *Crawler) TryPutForVisit(peer *levin.Peer) {
c.Lock()
defer c.Unlock()
if _, alreadyVisited := c.visited[peer.Addr()]; alreadyVisited {
return
}
c.notVisited[peer.Addr()] = peer
return
}
func (c *Crawler) MarkVisited(node *VisitedPeer) (alreadyIn bool) {
c.Lock()
defer c.Unlock()
if _, alreadyIn = c.visited[node.Addr()]; alreadyIn {
return true
}
c.visited[node.Addr()] = node
return false
}
// Runs takes care of spinning up worker goroutines that will then do the job
// of communicating with the nodes and figuring out their peerlist.
//
func (c *Crawler) Run(ctx context.Context) (map[string]*VisitedPeer, error) {
var (
peersToVisitC = make(chan *levin.Peer, 0)
newPeersFoundC = make(chan *levin.Peer, 0)
peersVisitedC = make(chan *VisitedPeer, 0)
)
for workerIndex := 0; workerIndex < CrawlerConcurrency; workerIndex++ {
go c.Worker(workerIndex, peersToVisitC, newPeersFoundC, peersVisitedC)
}
go func() {
for peerFound := range newPeersFoundC {
c.TryPutForVisit(peerFound)
c.log.WithField("peerfound", peerFound).Debug("peer found, putting for visit")
}
}()
go func() {
for peerVisited := range peersVisitedC {
if alreadyIn := c.MarkVisited(peerVisited); alreadyIn {
continue
}
c.log.WithField("peervisited", peerVisited).Info("peer visited, marking visited")
c.C <- peerVisited
}
}()
for {
peer := c.TakeNotVisited()
if peer == nil {
c.log.Info("no one to visit, sleeping a bit")
time.Sleep(500 * time.Millisecond)
continue
}
peersToVisitC <- peer
}
close(peersToVisitC)
close(newPeersFoundC)
close(peersVisitedC)
return nil, nil
}
// Worker takes care of consuming a peer at a time and discovering their peers.
//
// Essentially: peerC -> Worker(peer) -> peer-peers...
//
func (c *Crawler) Worker(
workerIndex int,
peersToVisitC chan *levin.Peer,
newPeersFoundC chan *levin.Peer,
peersVisitedC chan *VisitedPeer,
) {
logger := c.log.WithField("worker-idx", workerIndex)
for peerToVisit := range peersToVisitC {
l := logger.WithField("peer", peerToVisit)
func() {
l.Debug("visiting")
defer l.Debug("visited")
c.workerStatuses.Start(workerIndex)
defer c.workerStatuses.Finish(workerIndex)
visitedPeer := &VisitedPeer{
Peer: peerToVisit,
Error: nil,
}
peersFound, err := c.Discover(context.Background(), peerToVisit)
if err != nil {
l.Error("errored discovering peers", err)
visitedPeer.Error = err
peersVisitedC <- visitedPeer
return
}
l.WithField("found", len(peersFound)).Info("peers found")
for _, peerFound := range peersFound {
newPeersFoundC <- peerFound
}
peersVisitedC <- visitedPeer
}()
}
logger.Info("peers to visit closed - bailing out")
}
func (c *Crawler) Discover(ctx context.Context, peer *levin.Peer) (map[string]*levin.Peer, error) {
client, err := levin.NewClient(ctx, peer.Addr(), c.clientOptions...)
if err != nil {
return nil, fmt.Errorf("new client: %w", err)
}
defer client.Close()
pl, err := client.Handshake(ctx)
if err != nil {
return nil, fmt.Errorf("handshake: %w", err)
}
return pl.Peers, nil
}
func IsOkError(err error) bool {
if err, ok := err.(net.Error); ok && err.Timeout() {
return true
}
return false
}

View file

@ -12,13 +12,34 @@ import (
)
const (
EndpointJsonRPC = "/json_rpc"
VersionJsonRPC = "2.0"
// endpointJsonRPC is the common endpoint used for all the RPC calls
// that make use of epee's JSONRPC invocation format for requests and
// responses.
//
endpointJsonRPC = "/json_rpc"
// versionJsonRPC is the version of the JsonRPC format.
//
versionJsonRPC = "2.0"
)
// Client is a wrapper over a plain HTTP client providing methods that
// correspond to all RPC invocations to a `monerod` daemon, including
// restricted and non-restricted ones.
//
type Client struct {
// http is the underlying http client that takes care of sending
// requests and receiving the responses.
//
// To provide your own, make use of `WithHTTPClient` when instantiating
// the client via the `NewClient` constructor.
//
http *http.Client
url *url.URL
// address is the address of the monerod instance serving the RPC
// endpoints.
//
address *url.URL
}
type ClientOptions struct {
@ -46,6 +67,12 @@ func NewHTTPClient(verbose bool) *http.Client {
}
// NewClient instantiates a new Client that is able to communicate with
// monerod's RPC endpoints.
//
// The `address` might be either restricted (typically <ip>:18089) or not
// (typically <ip>:18081).
//
func NewClient(address string, opts ...ClientOption) (*Client, error) {
options := &ClientOptions{
HTTPClient: NewHTTPClient(false),
@ -61,8 +88,8 @@ func NewClient(address string, opts ...ClientOption) (*Client, error) {
}
return &Client{
url: parsedAddress,
http: options.HTTPClient,
address: parsedAddress,
http: options.HTTPClient,
}, nil
}
@ -90,8 +117,8 @@ type RequestEnvelope struct {
// Other makes requests to any other endpoints that are not `/jsonrpc`.
//
func (c *Client) Other(ctx context.Context, endpoint string, params interface{}, response interface{}) error {
url := *c.url
url.Path = endpoint
address := *c.address
address.Path = endpoint
var body io.Reader
@ -104,9 +131,9 @@ func (c *Client) Other(ctx context.Context, endpoint string, params interface{},
body = bytes.NewReader(b)
}
req, err := http.NewRequestWithContext(ctx, "GET", url.String(), body)
req, err := http.NewRequestWithContext(ctx, "GET", address.String(), body)
if err != nil {
return fmt.Errorf("new req '%s': %w", url.String(), err)
return fmt.Errorf("new req '%s': %w", address.String(), err)
}
req.Header.Add("Content-Type", "application/json")
@ -119,12 +146,12 @@ func (c *Client) Other(ctx context.Context, endpoint string, params interface{},
}
func (c *Client) JsonRPC(ctx context.Context, method string, params interface{}, response interface{}) error {
url := *c.url
url.Path = EndpointJsonRPC
address := *c.address
address.Path = endpointJsonRPC
b, err := json.Marshal(&RequestEnvelope{
Id: "0",
JsonRPC: "2.0",
JsonRPC: versionJsonRPC,
Method: method,
Params: params,
})
@ -132,9 +159,9 @@ func (c *Client) JsonRPC(ctx context.Context, method string, params interface{},
return fmt.Errorf("marshal: %w", err)
}
req, err := http.NewRequestWithContext(ctx, "GET", url.String(), bytes.NewReader(b))
req, err := http.NewRequestWithContext(ctx, "GET", address.String(), bytes.NewReader(b))
if err != nil {
return fmt.Errorf("new req '%s': %w", url.String(), err)
return fmt.Errorf("new req '%s': %w", address.String(), err)
}
req.Header.Add("Content-Type", "application/json")

4
pkg/rpc/doc.go Normal file
View file

@ -0,0 +1,4 @@
// package rpc provides a client that's able to communicate with a `monerod`
// daemon via its RPC interfaces.
//
package rpc