From 4f6c3915adab19e81b1f174398617e102ebf5536 Mon Sep 17 00:00:00 2001 From: WeebDataHoarder <57538841+WeebDataHoarder@users.noreply.github.com> Date: Tue, 23 Apr 2024 14:51:33 +0200 Subject: [PATCH] Upgrade to go-randomx v3.1.0 with full JIT and full mode support --- .drone.yml | 6 +- README.md | 4 +- go.mod | 2 +- go.sum | 4 +- monero/randomx/randomx.go | 25 ++++ monero/randomx/randomx_cgo.go | 26 +--- monero/randomx/randomx_nocgo.go | 202 +++++++++++++++++++++++--------- 7 files changed, 184 insertions(+), 85 deletions(-) diff --git a/.drone.yml b/.drone.yml index 21c6b7e..6becf9b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -46,10 +46,11 @@ steps: from_secret: MONEROD_RPC_URL MONEROD_ZMQ_URL: from_secret: MONEROD_ZMQ_URL + CGO_ENABLED: "1" commands: - apk update - apk add --no-cache git gcc g++ musl-dev pkgconfig - - go list -f '{{.Dir}}/...' -m | xargs -n 1 sh -c 'go test -p 1 -failfast -timeout 20m -cover -gcflags=-d=checkptr -v $0 || exit 255' + - go list -f '{{.Dir}}/...' -m | xargs -n 1 sh -c 'go test -flags enable_randomx_library -p 1 -failfast -timeout 20m -cover -gcflags=-d=checkptr -v $0 || exit 255' - name: test-go-asm image: golang:1.22-alpine3.19 depends_on: @@ -125,10 +126,11 @@ steps: from_secret: MONEROD_RPC_URL MONEROD_ZMQ_URL: from_secret: MONEROD_ZMQ_URL + CGO_ENABLED: "1" commands: - apk update - apk add --no-cache git gcc g++ musl-dev pkgconfig - - go list -f '{{.Dir}}/...' -m | xargs -n 1 sh -c 'go test -p 1 -failfast -timeout 20m -cover -gcflags=-d=checkptr -v $0 || exit 255' + - go list -f '{{.Dir}}/...' -m | xargs -n 1 sh -c 'go test -flags enable_randomx_library -p 1 -failfast -timeout 20m -cover -gcflags=-d=checkptr -v $0 || exit 255' - name: test-go image: golang:1.22-alpine3.19 depends_on: diff --git a/README.md b/README.md index 955b7d8..e8b4d8c 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,9 @@ You can also use the OpenAlias `p2pool.observer` directly on the GUI. ### Development notes -Requires using CGO when running the main modes where RandomX hashes are used, but can be used with `CGO_ENABLED=0` specifically as a library. +This library supports both [Golang RandomX library](https://git.gammaspectra.live/P2Pool/go-randomx) and the [C++ RandomX counterpart](https://github.com/tevador/RandomX). -You can install the RandomX dependency via this command: +By default, the Golang library will be used. You can enable the C++ library if you can use CGO, use the compile flags `enable_randomx_library` and have it installed via the command below: ```bash $ git clone --depth 1 --branch master https://github.com/tevador/RandomX.git /tmp/RandomX && cd /tmp/RandomX && \ mkdir build && cd build && \ diff --git a/go.mod b/go.mod index b81e244..2b1ce8e 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22 require ( git.gammaspectra.live/P2Pool/edwards25519 v0.0.0-20240405085108-e2f706cb5c00 git.gammaspectra.live/P2Pool/go-json v0.99.0 - git.gammaspectra.live/P2Pool/go-randomx/v2 v2.2.0 + git.gammaspectra.live/P2Pool/go-randomx/v3 v3.1.0 git.gammaspectra.live/P2Pool/monero-base58 v1.0.0 git.gammaspectra.live/P2Pool/randomx-go-bindings v1.0.0 git.gammaspectra.live/P2Pool/sha3 v0.17.0 diff --git a/go.sum b/go.sum index 43286d8..c9b8e3d 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ git.gammaspectra.live/P2Pool/edwards25519 v0.0.0-20240405085108-e2f706cb5c00 h1: git.gammaspectra.live/P2Pool/edwards25519 v0.0.0-20240405085108-e2f706cb5c00/go.mod h1:FZsrMWGucMP3SZamzrd7m562geIs5zp1O/9MGoiAKH0= git.gammaspectra.live/P2Pool/go-json v0.99.0 h1:TbFOEWbbDLFzm1fM/2+WPUhhzSwJ501+otfrQ8jCP84= git.gammaspectra.live/P2Pool/go-json v0.99.0/go.mod h1:X9PvT0fmWrU1I+BiDUjMypUWdsWFm24QCW1sWxbzT8w= -git.gammaspectra.live/P2Pool/go-randomx/v2 v2.2.0 h1:ABLqnlKrv0pkSXyEWZ6C4PzLvQp/lkhTwKM21nZbN4Q= -git.gammaspectra.live/P2Pool/go-randomx/v2 v2.2.0/go.mod h1:eYjslaVjiP4C3mYGKgkjpseLgsD5kKBuOcZJy9KPJ78= +git.gammaspectra.live/P2Pool/go-randomx/v3 v3.1.0 h1:FEVDm+kMhiz/zEkjPcc6b/Pp9sKGR1KoAoejT1qgaXY= +git.gammaspectra.live/P2Pool/go-randomx/v3 v3.1.0/go.mod h1:esQjh/AvuhmoMeBv9MCZ2SqgU0vgWOk1u0l0zCTMiP0= git.gammaspectra.live/P2Pool/monero-base58 v1.0.0 h1:s8LZxVNc93YEs2NCCNWZ7CKr8RbEb031y6Wkvhn+TS4= git.gammaspectra.live/P2Pool/monero-base58 v1.0.0/go.mod h1:WWEJy/AdWKxKAruvlKI82brw+DtVlePy0ct3ZiBlc68= git.gammaspectra.live/P2Pool/randomx-go-bindings v1.0.0 h1:tajr4QFSPrb8NtHmU14JaXdhr+z+0RbOBLIgUDI5Tow= diff --git a/monero/randomx/randomx.go b/monero/randomx/randomx.go index 94f78a7..265b264 100644 --- a/monero/randomx/randomx.go +++ b/monero/randomx/randomx.go @@ -1,6 +1,8 @@ package randomx import ( + "crypto/subtle" + "git.gammaspectra.live/P2Pool/consensus/v3/monero/crypto" "git.gammaspectra.live/P2Pool/consensus/v3/types" ) @@ -35,3 +37,26 @@ const ( SeedHashEpochLag = 64 SeedHashEpochBlocks = 2048 ) + +func consensusHash(scratchpad []byte) types.Hash { + // Intentionally not a power of 2 + const ScratchpadSize = 1009 + + const RandomxArgonMemory = 262144 + n := RandomxArgonMemory * 1024 + + const Vec128Size = 128 / 8 + + cachePtr := scratchpad[ScratchpadSize*Vec128Size:] + scratchpadTopPtr := scratchpad[:ScratchpadSize*Vec128Size] + for i := ScratchpadSize * Vec128Size; i < n; i += ScratchpadSize * Vec128Size { + stride := ScratchpadSize * Vec128Size + if stride > len(cachePtr) { + stride = len(cachePtr) + } + subtle.XORBytes(scratchpadTopPtr, scratchpadTopPtr, cachePtr[:stride]) + cachePtr = cachePtr[stride:] + } + + return crypto.Keccak256(scratchpadTopPtr) +} diff --git a/monero/randomx/randomx_cgo.go b/monero/randomx/randomx_cgo.go index 6c69b26..b889a40 100644 --- a/monero/randomx/randomx_cgo.go +++ b/monero/randomx/randomx_cgo.go @@ -1,12 +1,10 @@ -//go:build cgo && !disable_randomx_library && !purego +//go:build cgo && enable_randomx_library && !purego package randomx import ( "bytes" - "crypto/subtle" "errors" - "git.gammaspectra.live/P2Pool/consensus/v3/monero/crypto" "git.gammaspectra.live/P2Pool/consensus/v3/types" "git.gammaspectra.live/P2Pool/consensus/v3/utils" "git.gammaspectra.live/P2Pool/randomx-go-bindings" @@ -104,29 +102,9 @@ func ConsensusHash(buf []byte) types.Hash { defer randomx.ReleaseCache(cache) randomx.InitCache(cache, buf) - // Intentionally not a power of 2 - const ScratchpadSize = 1009 - const RandomxArgonMemory = 262144 - n := RandomxArgonMemory * 1024 - - const Vec128Size = 128 / 8 - - type Vec128 [Vec128Size]byte scratchpad := unsafe.Slice((*byte)(randomx.GetCacheMemory(cache)), n) - - cachePtr := scratchpad[ScratchpadSize*Vec128Size:] - scratchpadTopPtr := scratchpad[:ScratchpadSize*Vec128Size] - for i := ScratchpadSize * Vec128Size; i < n; i += ScratchpadSize * Vec128Size { - stride := ScratchpadSize * Vec128Size - if stride > len(cachePtr) { - stride = len(cachePtr) - } - subtle.XORBytes(scratchpadTopPtr, scratchpadTopPtr, cachePtr[:stride]) - cachePtr = cachePtr[stride:] - } - - return crypto.Keccak256(scratchpadTopPtr) + return consensusHash(scratchpad) } func NewRandomX(n int, flags ...Flag) (Hasher, error) { diff --git a/monero/randomx/randomx_nocgo.go b/monero/randomx/randomx_nocgo.go index 27efb4b..c89f66e 100644 --- a/monero/randomx/randomx_nocgo.go +++ b/monero/randomx/randomx_nocgo.go @@ -1,89 +1,183 @@ -//go:build !cgo || disable_randomx_library || purego +//go:build !cgo || !enable_randomx_library || purego package randomx import ( "bytes" - "crypto/subtle" - "git.gammaspectra.live/P2Pool/consensus/v3/monero/crypto" + "errors" "git.gammaspectra.live/P2Pool/consensus/v3/types" - "git.gammaspectra.live/P2Pool/go-randomx/v2" + "git.gammaspectra.live/P2Pool/consensus/v3/utils" + "git.gammaspectra.live/P2Pool/go-randomx/v3" + fasthex "github.com/tmthrgd/go-hex" "runtime" + "slices" "sync" "unsafe" ) -type hasher struct { - cache *randomx.Randomx_Cache - lock sync.Mutex +type hasherCollection struct { + lock sync.RWMutex + index int flags []Flag + cache []*hasherState +} - key []byte +func (h *hasherCollection) Hash(key []byte, input []byte) (types.Hash, error) { + if hash, err := func() (types.Hash, error) { + h.lock.RLock() + defer h.lock.RUnlock() + for _, c := range h.cache { + if len(c.key) > 0 && bytes.Compare(c.key, key) == 0 { + return c.Hash(input), nil + } + } + + return types.ZeroHash, errors.New("no hasher") + }(); err == nil && hash != types.ZeroHash { + return hash, nil + } else { + h.lock.Lock() + defer h.lock.Unlock() + index := h.index + h.index = (h.index + 1) % len(h.cache) + if err = h.cache[index].Init(key); err != nil { + return types.ZeroHash, err + } + return h.cache[index].Hash(input), nil + } +} + +func (h *hasherCollection) initStates(size int) (err error) { + for _, c := range h.cache { + c.Close() + } + h.cache = make([]*hasherState, size) + for i := range h.cache { + if h.cache[i], err = newRandomXState(h.flags...); err != nil { + return err + } + } + return nil +} + +func (h *hasherCollection) OptionFlags(flags ...Flag) error { + h.lock.Lock() + defer h.lock.Unlock() + if slices.Compare(h.flags, flags) != 0 { + h.flags = flags + return h.initStates(len(h.cache)) + } + return nil +} +func (h *hasherCollection) OptionNumberOfCachedStates(n int) error { + h.lock.Lock() + defer h.lock.Unlock() + if len(h.cache) != n { + return h.initStates(n) + } + return nil +} + +func (h *hasherCollection) Close() { + h.lock.Lock() + defer h.lock.Unlock() + for _, c := range h.cache { + c.Close() + } +} + +type hasherState struct { + lock sync.Mutex + cache *randomx.Cache + dataset randomx.Dataset + vm *randomx.VM + flags uint64 + key []byte } func ConsensusHash(buf []byte) types.Hash { - cache := randomx.Randomx_alloc_cache(0) + cache := randomx.NewCache(0) + defer cache.Close() + cache.Init(buf) scratchpad := unsafe.Slice((*byte)(unsafe.Pointer(unsafe.SliceData(cache.Blocks))), len(cache.Blocks)*len(cache.Blocks[0])*int(unsafe.Sizeof(uint64(0)))) defer runtime.KeepAlive(cache) - // Intentionally not a power of 2 - const ScratchpadSize = 1009 - - const RandomxArgonMemory = 262144 - n := RandomxArgonMemory * 1024 - - const Vec128Size = 128 / 8 - - type Vec128 [Vec128Size]byte - - cachePtr := scratchpad[ScratchpadSize*Vec128Size:] - scratchpadTopPtr := scratchpad[:ScratchpadSize*Vec128Size] - for i := ScratchpadSize * Vec128Size; i < n; i += ScratchpadSize * Vec128Size { - stride := ScratchpadSize * Vec128Size - if stride > len(cachePtr) { - stride = len(cachePtr) - } - subtle.XORBytes(scratchpadTopPtr, scratchpadTopPtr, cachePtr[:stride]) - cachePtr = cachePtr[stride:] - } - - return crypto.Keccak256(scratchpadTopPtr) -} - -func (h *hasher) OptionFlags(flags ...Flag) error { - return nil -} -func (h *hasher) OptionNumberOfCachedStates(n int) error { - return nil + return consensusHash(scratchpad) } func NewRandomX(n int, flags ...Flag) (Hasher, error) { - return &hasher{ + collection := &hasherCollection{ flags: flags, - cache: randomx.Randomx_alloc_cache(randomx.RANDOMX_FLAG_JIT), - }, nil + } + + if err := collection.initStates(n); err != nil { + return nil, err + } + return collection, nil } -func (h *hasher) Hash(key []byte, input []byte) (output types.Hash, err error) { - vm := func() *randomx.VM { - h.lock.Lock() - defer h.lock.Unlock() +func newRandomXState(flags ...Flag) (*hasherState, error) { - if h.key == nil || bytes.Compare(h.key, key) != 0 { - h.key = make([]byte, len(key)) - copy(h.key, key) - - h.cache.Init(h.key) + applyFlags := randomx.GetFlags() + for _, f := range flags { + if f == FlagLargePages { + applyFlags |= randomx.RANDOMX_FLAG_LARGE_PAGES + } else if f == FlagFullMemory { + applyFlags |= randomx.RANDOMX_FLAG_FULL_MEM + } else if f == FlagSecure { + applyFlags |= randomx.RANDOMX_FLAG_SECURE } - return h.cache.VM_Initialize() - }() + } + h := &hasherState{ + flags: applyFlags, + } + h.cache = randomx.NewCache(h.flags) - vm.CalculateHash(input, (*[32]byte)(&output)) + if dataset := randomx.NewDataset(h.cache); dataset == nil { + h.cache.Close() + return nil, errors.New("couldn't initialize dataset") + } else { + h.dataset = dataset + } + + if vm := randomx.NewVM(h.dataset); vm == nil { + h.cache.Close() + return nil, errors.New("couldn't initialize dataset") + } else { + h.vm = vm + } + + return h, nil +} + +func (h *hasherState) Init(key []byte) (err error) { + h.lock.Lock() + defer h.lock.Unlock() + h.key = make([]byte, len(key)) + copy(h.key, key) + + utils.Logf("RandomX", "Initializing to seed %s", fasthex.EncodeToString(h.key)) + h.cache.Init(h.key) + randomx.InitDatasetParallel(h.dataset, runtime.NumCPU()) + + utils.Logf("RandomX", "Initialized to seed %s", fasthex.EncodeToString(h.key)) + + return nil +} + +func (h *hasherState) Hash(input []byte) (output types.Hash) { + h.lock.Lock() + defer h.lock.Unlock() + h.vm.CalculateHash(input, (*[32]byte)(&output)) + runtime.KeepAlive(input) return } -func (h *hasher) Close() { +func (h *hasherState) Close() { + h.lock.Lock() + defer h.lock.Unlock() + h.vm.Close() h.cache.Close() }