Upgrade to go-randomx v3.1.0 with full JIT and full mode support

.drone.yml

@@ -46,10 +46,11 @@ steps:
         from_secret: MONEROD_RPC_URL
       MONEROD_ZMQ_URL:
         from_secret: MONEROD_ZMQ_URL
+      CGO_ENABLED: "1"
     commands:
       - apk update
       - apk add --no-cache git gcc g++ musl-dev pkgconfig
-      - go list -f '{{.Dir}}/...' -m | xargs -n 1 sh -c 'go test -p 1 -failfast -timeout 20m -cover -gcflags=-d=checkptr -v $0 || exit 255'
+      - go list -f '{{.Dir}}/...' -m | xargs -n 1 sh -c 'go test -tags enable_randomx_library -p 1 -failfast -timeout 20m -cover -gcflags=-d=checkptr -v $0 || exit 255'
   - name: test-go-asm
     image: golang:1.22-alpine3.19
     depends_on:
@@ -125,10 +126,11 @@ steps:
         from_secret: MONEROD_RPC_URL
       MONEROD_ZMQ_URL:
         from_secret: MONEROD_ZMQ_URL
+      CGO_ENABLED: "1"
     commands:
       - apk update
       - apk add --no-cache git gcc g++ musl-dev pkgconfig
-      - go list -f '{{.Dir}}/...' -m | xargs -n 1 sh -c 'go test -p 1 -failfast -timeout 20m -cover -gcflags=-d=checkptr -v $0 || exit 255'
+      - go list -f '{{.Dir}}/...' -m | xargs -n 1 sh -c 'go test -tags enable_randomx_library -p 1 -failfast -timeout 20m -cover -gcflags=-d=checkptr -v $0 || exit 255'
   - name: test-go
     image: golang:1.22-alpine3.19
     depends_on:

README.md

@@ -21,9 +21,9 @@ You can also use the OpenAlias `p2pool.observer` directly on the GUI.
 
 ### Development notes
 
-Requires using CGO when running the main modes where RandomX hashes are used, but can be used with `CGO_ENABLED=0` specifically as a library.
+This library supports both [Golang RandomX library](https://git.gammaspectra.live/P2Pool/go-randomx) and the [C++ RandomX counterpart](https://github.com/tevador/RandomX).
 
-You can install the RandomX dependency via this command:
+By default, the Golang library will be used. You can enable the C++ library if by using CGO and the compile tag `enable_randomx_library` and have it installed via the command below:
 ```bash
 $ git clone --depth 1 --branch master https://github.com/tevador/RandomX.git /tmp/RandomX && cd /tmp/RandomX && \
     mkdir build && cd build && \

go.mod

@@ -5,7 +5,7 @@ go 1.22
 require (
 	git.gammaspectra.live/P2Pool/edwards25519 v0.0.0-20240405085108-e2f706cb5c00
 	git.gammaspectra.live/P2Pool/go-json v0.99.0
-	git.gammaspectra.live/P2Pool/go-randomx/v2 v2.2.0
+	git.gammaspectra.live/P2Pool/go-randomx/v3 v3.1.0
 	git.gammaspectra.live/P2Pool/monero-base58 v1.0.0
 	git.gammaspectra.live/P2Pool/randomx-go-bindings v1.0.0
 	git.gammaspectra.live/P2Pool/sha3 v0.17.0

go.sum

@@ -2,8 +2,8 @@ git.gammaspectra.live/P2Pool/edwards25519 v0.0.0-20240405085108-e2f706cb5c00 h1:
 git.gammaspectra.live/P2Pool/edwards25519 v0.0.0-20240405085108-e2f706cb5c00/go.mod h1:FZsrMWGucMP3SZamzrd7m562geIs5zp1O/9MGoiAKH0=
 git.gammaspectra.live/P2Pool/go-json v0.99.0 h1:TbFOEWbbDLFzm1fM/2+WPUhhzSwJ501+otfrQ8jCP84=
 git.gammaspectra.live/P2Pool/go-json v0.99.0/go.mod h1:X9PvT0fmWrU1I+BiDUjMypUWdsWFm24QCW1sWxbzT8w=
-git.gammaspectra.live/P2Pool/go-randomx/v2 v2.2.0 h1:ABLqnlKrv0pkSXyEWZ6C4PzLvQp/lkhTwKM21nZbN4Q=
-git.gammaspectra.live/P2Pool/go-randomx/v2 v2.2.0/go.mod h1:eYjslaVjiP4C3mYGKgkjpseLgsD5kKBuOcZJy9KPJ78=
+git.gammaspectra.live/P2Pool/go-randomx/v3 v3.1.0 h1:FEVDm+kMhiz/zEkjPcc6b/Pp9sKGR1KoAoejT1qgaXY=
+git.gammaspectra.live/P2Pool/go-randomx/v3 v3.1.0/go.mod h1:esQjh/AvuhmoMeBv9MCZ2SqgU0vgWOk1u0l0zCTMiP0=
 git.gammaspectra.live/P2Pool/monero-base58 v1.0.0 h1:s8LZxVNc93YEs2NCCNWZ7CKr8RbEb031y6Wkvhn+TS4=
 git.gammaspectra.live/P2Pool/monero-base58 v1.0.0/go.mod h1:WWEJy/AdWKxKAruvlKI82brw+DtVlePy0ct3ZiBlc68=
 git.gammaspectra.live/P2Pool/randomx-go-bindings v1.0.0 h1:tajr4QFSPrb8NtHmU14JaXdhr+z+0RbOBLIgUDI5Tow=

monero/randomx/randomx.go

@@ -1,6 +1,8 @@
 package randomx
 
 import (
+	"crypto/subtle"
+	"git.gammaspectra.live/P2Pool/consensus/v3/monero/crypto"
 	"git.gammaspectra.live/P2Pool/consensus/v3/types"
 )
 
@@ -35,3 +37,26 @@ const (
 	SeedHashEpochLag    = 64
 	SeedHashEpochBlocks = 2048
 )
+
+func consensusHash(scratchpad []byte) types.Hash {
+	// Intentionally not a power of 2
+	const ScratchpadSize = 1009
+
+	const RandomxArgonMemory = 262144
+	n := RandomxArgonMemory * 1024
+
+	const Vec128Size = 128 / 8
+
+	cachePtr := scratchpad[ScratchpadSize*Vec128Size:]
+	scratchpadTopPtr := scratchpad[:ScratchpadSize*Vec128Size]
+	for i := ScratchpadSize * Vec128Size; i < n; i += ScratchpadSize * Vec128Size {
+		stride := ScratchpadSize * Vec128Size
+		if stride > len(cachePtr) {
+			stride = len(cachePtr)
+		}
+		subtle.XORBytes(scratchpadTopPtr, scratchpadTopPtr, cachePtr[:stride])
+		cachePtr = cachePtr[stride:]
+	}
+
+	return crypto.Keccak256(scratchpadTopPtr)
+}

monero/randomx/randomx_cgo.go

@@ -1,12 +1,10 @@
-//go:build cgo && !disable_randomx_library && !purego
+//go:build cgo && enable_randomx_library && !purego
 
 package randomx
 
 import (
 	"bytes"
-	"crypto/subtle"
 	"errors"
-	"git.gammaspectra.live/P2Pool/consensus/v3/monero/crypto"
 	"git.gammaspectra.live/P2Pool/consensus/v3/types"
 	"git.gammaspectra.live/P2Pool/consensus/v3/utils"
 	"git.gammaspectra.live/P2Pool/randomx-go-bindings"
@@ -104,29 +102,12 @@ func ConsensusHash(buf []byte) types.Hash {
 	defer randomx.ReleaseCache(cache)
 
 	randomx.InitCache(cache, buf)
-	// Intentionally not a power of 2
-	const ScratchpadSize = 1009
 
 	const RandomxArgonMemory = 262144
 	n := RandomxArgonMemory * 1024
 
-	const Vec128Size = 128 / 8
-
-	type Vec128 [Vec128Size]byte
 	scratchpad := unsafe.Slice((*byte)(randomx.GetCacheMemory(cache)), n)
-
-	cachePtr := scratchpad[ScratchpadSize*Vec128Size:]
-	scratchpadTopPtr := scratchpad[:ScratchpadSize*Vec128Size]
-	for i := ScratchpadSize * Vec128Size; i < n; i += ScratchpadSize * Vec128Size {
-		stride := ScratchpadSize * Vec128Size
-		if stride > len(cachePtr) {
-			stride = len(cachePtr)
-		}
-		subtle.XORBytes(scratchpadTopPtr, scratchpadTopPtr, cachePtr[:stride])
-		cachePtr = cachePtr[stride:]
-	}
-
-	return crypto.Keccak256(scratchpadTopPtr)
+	return consensusHash(scratchpad)
 }
 
 func NewRandomX(n int, flags ...Flag) (Hasher, error) {

monero/randomx/randomx_nocgo.go

@@ -1,89 +1,183 @@
-//go:build !cgo || disable_randomx_library || purego
+//go:build !cgo || !enable_randomx_library || purego
 
 package randomx
 
 import (
 	"bytes"
-	"crypto/subtle"
-	"git.gammaspectra.live/P2Pool/consensus/v3/monero/crypto"
+	"errors"
 	"git.gammaspectra.live/P2Pool/consensus/v3/types"
-	"git.gammaspectra.live/P2Pool/go-randomx/v2"
+	"git.gammaspectra.live/P2Pool/consensus/v3/utils"
+	"git.gammaspectra.live/P2Pool/go-randomx/v3"
+	fasthex "github.com/tmthrgd/go-hex"
 	"runtime"
+	"slices"
 	"sync"
 	"unsafe"
 )
 
-type hasher struct {
-	cache *randomx.Randomx_Cache
-	lock  sync.Mutex
+type hasherCollection struct {
+	lock  sync.RWMutex
+	index int
 	flags []Flag
+	cache []*hasherState
+}
 
-	key []byte
+func (h *hasherCollection) Hash(key []byte, input []byte) (types.Hash, error) {
+	if hash, err := func() (types.Hash, error) {
+		h.lock.RLock()
+		defer h.lock.RUnlock()
+		for _, c := range h.cache {
+			if len(c.key) > 0 && bytes.Compare(c.key, key) == 0 {
+				return c.Hash(input), nil
+			}
+		}
+
+		return types.ZeroHash, errors.New("no hasher")
+	}(); err == nil && hash != types.ZeroHash {
+		return hash, nil
+	} else {
+		h.lock.Lock()
+		defer h.lock.Unlock()
+		index := h.index
+		h.index = (h.index + 1) % len(h.cache)
+		if err = h.cache[index].Init(key); err != nil {
+			return types.ZeroHash, err
+		}
+		return h.cache[index].Hash(input), nil
+	}
+}
+
+func (h *hasherCollection) initStates(size int) (err error) {
+	for _, c := range h.cache {
+		c.Close()
+	}
+	h.cache = make([]*hasherState, size)
+	for i := range h.cache {
+		if h.cache[i], err = newRandomXState(h.flags...); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (h *hasherCollection) OptionFlags(flags ...Flag) error {
+	h.lock.Lock()
+	defer h.lock.Unlock()
+	if slices.Compare(h.flags, flags) != 0 {
+		h.flags = flags
+		return h.initStates(len(h.cache))
+	}
+	return nil
+}
+func (h *hasherCollection) OptionNumberOfCachedStates(n int) error {
+	h.lock.Lock()
+	defer h.lock.Unlock()
+	if len(h.cache) != n {
+		return h.initStates(n)
+	}
+	return nil
+}
+
+func (h *hasherCollection) Close() {
+	h.lock.Lock()
+	defer h.lock.Unlock()
+	for _, c := range h.cache {
+		c.Close()
+	}
+}
+
+type hasherState struct {
+	lock    sync.Mutex
+	cache   *randomx.Cache
+	dataset randomx.Dataset
+	vm      *randomx.VM
+	flags   uint64
+	key     []byte
 }
 
 func ConsensusHash(buf []byte) types.Hash {
-	cache := randomx.Randomx_alloc_cache(0)
+	cache := randomx.NewCache(0)
+	defer cache.Close()
+
 	cache.Init(buf)
 
 	scratchpad := unsafe.Slice((*byte)(unsafe.Pointer(unsafe.SliceData(cache.Blocks))), len(cache.Blocks)*len(cache.Blocks[0])*int(unsafe.Sizeof(uint64(0))))
 	defer runtime.KeepAlive(cache)
 
-	// Intentionally not a power of 2
-	const ScratchpadSize = 1009
-
-	const RandomxArgonMemory = 262144
-	n := RandomxArgonMemory * 1024
-
-	const Vec128Size = 128 / 8
-
-	type Vec128 [Vec128Size]byte
-
-	cachePtr := scratchpad[ScratchpadSize*Vec128Size:]
-	scratchpadTopPtr := scratchpad[:ScratchpadSize*Vec128Size]
-	for i := ScratchpadSize * Vec128Size; i < n; i += ScratchpadSize * Vec128Size {
-		stride := ScratchpadSize * Vec128Size
-		if stride > len(cachePtr) {
-			stride = len(cachePtr)
-		}
-		subtle.XORBytes(scratchpadTopPtr, scratchpadTopPtr, cachePtr[:stride])
-		cachePtr = cachePtr[stride:]
-	}
-
-	return crypto.Keccak256(scratchpadTopPtr)
-}
-
-func (h *hasher) OptionFlags(flags ...Flag) error {
-	return nil
-}
-func (h *hasher) OptionNumberOfCachedStates(n int) error {
-	return nil
+	return consensusHash(scratchpad)
 }
 
 func NewRandomX(n int, flags ...Flag) (Hasher, error) {
-	return &hasher{
+	collection := &hasherCollection{
 		flags: flags,
-		cache: randomx.Randomx_alloc_cache(randomx.RANDOMX_FLAG_JIT),
-	}, nil
+	}
+
+	if err := collection.initStates(n); err != nil {
+		return nil, err
+	}
+	return collection, nil
 }
 
-func (h *hasher) Hash(key []byte, input []byte) (output types.Hash, err error) {
-	vm := func() *randomx.VM {
-		h.lock.Lock()
-		defer h.lock.Unlock()
+func newRandomXState(flags ...Flag) (*hasherState, error) {
 
-		if h.key == nil || bytes.Compare(h.key, key) != 0 {
-			h.key = make([]byte, len(key))
-			copy(h.key, key)
-
-			h.cache.Init(h.key)
+	applyFlags := randomx.GetFlags()
+	for _, f := range flags {
+		if f == FlagLargePages {
+			applyFlags |= randomx.RANDOMX_FLAG_LARGE_PAGES
+		} else if f == FlagFullMemory {
+			applyFlags |= randomx.RANDOMX_FLAG_FULL_MEM
+		} else if f == FlagSecure {
+			applyFlags |= randomx.RANDOMX_FLAG_SECURE
 		}
-		return h.cache.VM_Initialize()
-	}()
+	}
+	h := &hasherState{
+		flags: applyFlags,
+	}
+	h.cache = randomx.NewCache(h.flags)
 
-	vm.CalculateHash(input, (*[32]byte)(&output))
+	if dataset := randomx.NewDataset(h.cache); dataset == nil {
+		h.cache.Close()
+		return nil, errors.New("couldn't initialize dataset")
+	} else {
+		h.dataset = dataset
+	}
+
+	if vm := randomx.NewVM(h.dataset); vm == nil {
+		h.cache.Close()
+		return nil, errors.New("couldn't initialize dataset")
+	} else {
+		h.vm = vm
+	}
+
+	return h, nil
+}
+
+func (h *hasherState) Init(key []byte) (err error) {
+	h.lock.Lock()
+	defer h.lock.Unlock()
+	h.key = make([]byte, len(key))
+	copy(h.key, key)
+
+	utils.Logf("RandomX", "Initializing to seed %s", fasthex.EncodeToString(h.key))
+	h.cache.Init(h.key)
+	randomx.InitDatasetParallel(h.dataset, runtime.NumCPU())
+
+	utils.Logf("RandomX", "Initialized to seed %s", fasthex.EncodeToString(h.key))
+
+	return nil
+}
+
+func (h *hasherState) Hash(input []byte) (output types.Hash) {
+	h.lock.Lock()
+	defer h.lock.Unlock()
+	h.vm.CalculateHash(input, (*[32]byte)(&output))
+	runtime.KeepAlive(input)
 	return
 }
 
-func (h *hasher) Close() {
+func (h *hasherState) Close() {
+	h.lock.Lock()
+	defer h.lock.Unlock()
+	h.vm.Close()
 	h.cache.Close()
 }