Version v3.0.0, support full datataset mode, modified api

README.md

@@ -14,6 +14,8 @@ This package implements RandomX without CGO, using only Golang code, native floa
 
 All test cases pass properly.
 
+Supports Full mode and Light mode.
+
 For the C++ implementation and design of RandomX, see [github.com/tevador/RandomX](https://github.com/tevador/RandomX)
 
 |           Feature            | 386 | amd64 | arm | arm64 | mips | mips64 | riscv64 | wasm |

aes/hash.go

@@ -30,7 +30,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 package aes
 
 import (
-	"git.gammaspectra.live/P2Pool/go-randomx/v2/keys"
+	"git.gammaspectra.live/P2Pool/go-randomx/v3/keys"
 	"unsafe"
 )
 

aes/round_amd64.go

@@ -3,29 +3,29 @@
 package aes
 
 import (
-	_ "git.gammaspectra.live/P2Pool/go-randomx/v2/asm"
+	_ "git.gammaspectra.live/P2Pool/go-randomx/v3/asm"
 	"golang.org/x/sys/cpu"
 	_ "unsafe"
 )
 
 //go:noescape
-//go:linkname hard_aesdec git.gammaspectra.live/P2Pool/go-randomx/v2/asm.aesdec
+//go:linkname hard_aesdec git.gammaspectra.live/P2Pool/go-randomx/v3/asm.aesdec
 func hard_aesdec(state *[4]uint32, key *[4]uint32)
 
 //go:noescape
-//go:linkname hard_aesenc git.gammaspectra.live/P2Pool/go-randomx/v2/asm.aesenc
+//go:linkname hard_aesenc git.gammaspectra.live/P2Pool/go-randomx/v3/asm.aesenc
 func hard_aesenc(state *[4]uint32, key *[4]uint32)
 
 //go:noescape
-//go:linkname hard_aesroundtrip_decenc git.gammaspectra.live/P2Pool/go-randomx/v2/asm.aesroundtrip_decenc
+//go:linkname hard_aesroundtrip_decenc git.gammaspectra.live/P2Pool/go-randomx/v3/asm.aesroundtrip_decenc
 func hard_aesroundtrip_decenc(states *[4][4]uint32, keys *[4][4]uint32)
 
 //go:noescape
-//go:linkname hard_aesroundtrip_encdec git.gammaspectra.live/P2Pool/go-randomx/v2/asm.aesroundtrip_encdec
+//go:linkname hard_aesroundtrip_encdec git.gammaspectra.live/P2Pool/go-randomx/v3/asm.aesroundtrip_encdec
 func hard_aesroundtrip_encdec(states *[4][4]uint32, keys *[4][4]uint32)
 
 //go:noescape
-//go:linkname hard_aesroundtrip_encdec1 git.gammaspectra.live/P2Pool/go-randomx/v2/asm.aesroundtrip_encdec1
+//go:linkname hard_aesroundtrip_encdec1 git.gammaspectra.live/P2Pool/go-randomx/v3/asm.aesroundtrip_encdec1
 func hard_aesroundtrip_encdec1(states *[4][4]uint32, key *[4]uint32)
 
 var supportsAES = cpu.X86.HasAES

cache.go

@@ -1,8 +1,8 @@
 package randomx
 
 import (
-	"git.gammaspectra.live/P2Pool/go-randomx/v2/argon2"
-	"git.gammaspectra.live/P2Pool/go-randomx/v2/keys"
+	"git.gammaspectra.live/P2Pool/go-randomx/v3/argon2"
+	"git.gammaspectra.live/P2Pool/go-randomx/v3/keys"
 	"runtime"
 	"slices"
 	"unsafe"
@@ -15,7 +15,7 @@ func (m *MemoryBlock) GetLine(addr uint64) *RegisterLine {
 	return (*RegisterLine)(unsafe.Pointer(unsafe.SliceData(m[addr : addr+8 : addr+8])))
 }
 
-type Randomx_Cache struct {
+type Cache struct {
 	Blocks []MemoryBlock
 
 	Programs [RANDOMX_PROGRAM_COUNT]SuperScalarProgram
@@ -25,36 +25,20 @@ type Randomx_Cache struct {
 	Flags uint64
 }
 
-func Randomx_alloc_cache(flags uint64) *Randomx_Cache {
+func NewCache(flags uint64) *Cache {
 	if flags == RANDOMX_FLAG_DEFAULT {
 		flags = RANDOMX_FLAG_JIT
 	}
-	return &Randomx_Cache{
+	return &Cache{
 		Flags: flags,
 	}
 }
 
-func (cache *Randomx_Cache) HasJIT() bool {
+func (cache *Cache) HasJIT() bool {
 	return cache.Flags&RANDOMX_FLAG_JIT > 0 && cache.JitPrograms[0] != nil
 }
 
-func (cache *Randomx_Cache) VM_Initialize() *VM {
-
-	vm := &VM{
-		Dataset: &Randomx_DatasetLight{
-			Cache: cache,
-		},
-	}
-	if cache.HasJIT() {
-		vm.JITProgram = mapProgram(nil, int(RandomXCodeSize))
-		if cache.Flags&RANDOMX_FLAG_SECURE == 0 {
-			mapProgramRWX(vm.JITProgram)
-		}
-	}
-	return vm
-}
-
-func (cache *Randomx_Cache) Close() error {
+func (cache *Cache) Close() error {
 	for _, p := range cache.JitPrograms {
 		if p != nil {
 			err := p.Close()
@@ -66,10 +50,12 @@ func (cache *Randomx_Cache) Close() error {
 	return nil
 }
 
-func (cache *Randomx_Cache) Init(key []byte) {
-	// Lock due to external JIT madness
-	runtime.LockOSThread()
-	defer runtime.UnlockOSThread()
+func (cache *Cache) Init(key []byte) {
+	if cache.Flags&RANDOMX_FLAG_JIT > 0 {
+		// Lock due to external JIT madness
+		runtime.LockOSThread()
+		defer runtime.UnlockOSThread()
+	}
 
 	kkey := slices.Clone(key)
 
@@ -93,7 +79,7 @@ func (cache *Randomx_Cache) Init(key []byte) {
 const Mask = CacheSize/CacheLineSize - 1
 
 // GetMixBlock fetch a 64 byte block in uint64 form
-func (cache *Randomx_Cache) GetMixBlock(addr uint64) *RegisterLine {
+func (cache *Cache) GetMixBlock(addr uint64) *RegisterLine {
 
 	addr = (addr & Mask) * CacheLineSize
 
@@ -101,7 +87,7 @@ func (cache *Randomx_Cache) GetMixBlock(addr uint64) *RegisterLine {
 	return cache.Blocks[block].GetLine(addr % 1024)
 }
 
-func (cache *Randomx_Cache) InitDatasetItem(rl *RegisterLine, itemNumber uint64) {
+func (cache *Cache) InitDatasetItem(rl *RegisterLine, itemNumber uint64) {
 	registerValue := itemNumber
 
 	rl[0] = (itemNumber + 1) * keys.SuperScalar_Constants[0]
@@ -129,7 +115,7 @@ func (cache *Randomx_Cache) InitDatasetItem(rl *RegisterLine, itemNumber uint64)
 	}
 }
 
-func (cache *Randomx_Cache) InitDatasetItemJIT(rl *RegisterLine, itemNumber uint64) {
+func (cache *Cache) InitDatasetItemJIT(rl *RegisterLine, itemNumber uint64) {
 	registerValue := itemNumber
 
 	rl[0] = (itemNumber + 1) * keys.SuperScalar_Constants[0]
@@ -155,9 +141,12 @@ func (cache *Randomx_Cache) InitDatasetItemJIT(rl *RegisterLine, itemNumber uint
 	}
 }
 
-func (cache *Randomx_Cache) initDataset(dataset []RegisterLine, startItem, endItem uint64) {
-	panic("todo")
+func (cache *Cache) InitDataset(dataset []RegisterLine, startItem, endItem uint64) {
 	for itemNumber := startItem; itemNumber < endItem; itemNumber, dataset = itemNumber+1, dataset[1:] {
-		cache.InitDatasetItem(&dataset[0], itemNumber)
+		if cache.HasJIT() {
+			cache.InitDatasetItemJIT(&dataset[0], itemNumber)
+		} else {
+			cache.InitDatasetItem(&dataset[0], itemNumber)
+		}
 	}
 }

config.go

@@ -29,7 +29,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 package randomx
 
-import "git.gammaspectra.live/P2Pool/go-randomx/v2/argon2"
+import "git.gammaspectra.live/P2Pool/go-randomx/v3/argon2"
 
 // see reference configuration.h
 // Cache size in KiB. Must be a power of 2.
@@ -81,7 +81,7 @@ const RANDOMX_JUMP_BITS = 8
 // Jump condition mask offset in bits. The sum of RANDOMX_JUMP_BITS and RANDOMX_JUMP_OFFSET must not exceed 16.
 const RANDOMX_JUMP_OFFSET = 8
 
-const DATASETEXTRAITEMS = RANDOMX_DATASET_EXTRA_SIZE / RANDOMX_DATASET_ITEM_SIZE
+const DatasetExtraItems = RANDOMX_DATASET_EXTRA_SIZE / RANDOMX_DATASET_ITEM_SIZE
 
 const SuperscalarMaxSize = 3*RANDOMX_SUPERSCALAR_LATENCY + 2
 const RANDOMX_DATASET_ITEM_SIZE uint64 = 64

dataset.go

@@ -1,8 +1,30 @@
 package randomx
 
-type Randomx_Dataset interface {
-	InitDataset(startItem, endItem uint64)
-	ReadDataset(address uint64, r, cache *RegisterLine)
+import "sync"
+
+type Dataset interface {
+	InitDataset(startItem, itemCount uint64)
+	ReadDataset(address uint64, r *RegisterLine)
 	PrefetchDataset(address uint64)
 	Flags() uint64
+	Cache() *Cache
+}
+
+func InitDatasetParallel(dataset Dataset, n int) {
+	n = max(1, n)
+
+	var wg sync.WaitGroup
+	for i := uint64(1); i < uint64(n); i++ {
+		a := (DatasetItemCount * i) / uint64(n)
+		b := (DatasetItemCount * (i + 1)) / uint64(n)
+
+		wg.Add(1)
+		go func(a, b uint64) {
+			defer wg.Done()
+			dataset.InitDataset(a, b-a)
+		}(a, b)
+	}
+
+	dataset.InitDataset(0, DatasetItemCount/uint64(n))
+	wg.Wait()
 }

dataset_full.go

@@ -0,0 +1,46 @@
+package randomx
+
+const DatasetSize = RANDOMX_DATASET_BASE_SIZE + RANDOMX_DATASET_EXTRA_SIZE
+
+const DatasetItemCount = DatasetSize / CacheLineSize
+
+type DatasetFull struct {
+	cache  *Cache
+	Memory [DatasetItemCount]RegisterLine
+}
+
+func NewFullDataset(cache *Cache) *DatasetFull {
+	return &DatasetFull{
+		cache: cache,
+	}
+}
+
+func (d *DatasetFull) PrefetchDataset(address uint64) {
+
+}
+
+func (d *DatasetFull) ReadDataset(address uint64, r *RegisterLine) {
+	cache := &d.Memory[address/CacheLineSize]
+
+	for i := range r {
+		r[i] ^= cache[i]
+	}
+}
+
+func (d *DatasetFull) Cache() *Cache {
+	return d.cache
+}
+
+func (d *DatasetFull) Flags() uint64 {
+	return d.cache.Flags
+}
+
+func (d *DatasetFull) InitDataset(startItem, itemCount uint64) {
+	if startItem >= DatasetItemCount || itemCount > DatasetItemCount {
+		panic("out of range")
+	}
+	if startItem+itemCount > DatasetItemCount {
+		panic("out of range")
+	}
+	d.cache.InitDataset(d.Memory[startItem:startItem+itemCount], startItem, startItem+itemCount)
+}

dataset_light.go

@@ -1,19 +1,25 @@
 package randomx
 
-type Randomx_DatasetLight struct {
-	Cache  *Randomx_Cache
-	Memory []uint64
+type DatasetLight struct {
+	cache *Cache
 }
 
-func (d *Randomx_DatasetLight) PrefetchDataset(address uint64) {
+func NewLightDataset(cache *Cache) *DatasetLight {
+	return &DatasetLight{
+		cache: cache,
+	}
+}
+
+func (d *DatasetLight) PrefetchDataset(address uint64) {
 
 }
 
-func (d *Randomx_DatasetLight) ReadDataset(address uint64, r, cache *RegisterLine) {
-	if d.Cache.HasJIT() {
-		d.Cache.InitDatasetItemJIT(cache, address/CacheLineSize)
+func (d *DatasetLight) ReadDataset(address uint64, r *RegisterLine) {
+	var cache RegisterLine
+	if d.cache.HasJIT() {
+		d.cache.InitDatasetItemJIT(&cache, address/CacheLineSize)
 	} else {
-		d.Cache.InitDatasetItem(cache, address/CacheLineSize)
+		d.cache.InitDatasetItem(&cache, address/CacheLineSize)
 	}
 
 	for i := range r {
@@ -21,10 +27,14 @@ func (d *Randomx_DatasetLight) ReadDataset(address uint64, r, cache *RegisterLin
 	}
 }
 
-func (d *Randomx_DatasetLight) Flags() uint64 {
-	return d.Cache.Flags
+func (d *DatasetLight) Flags() uint64 {
+	return d.cache.Flags
 }
 
-func (d *Randomx_DatasetLight) InitDataset(startItem, endItem uint64) {
-	//d.Cache.initDataset(d.Cache.Programs)
+func (d *DatasetLight) Cache() *Cache {
+	return d.cache
+}
+
+func (d *DatasetLight) InitDataset(startItem, itemCount uint64) {
+
 }

go.mod

@@ -1,4 +1,4 @@
-module git.gammaspectra.live/P2Pool/go-randomx/v2
+module git.gammaspectra.live/P2Pool/go-randomx/v3
 
 go 1.21
 

jit_amd64.go

@@ -5,7 +5,7 @@ package randomx
 import (
 	"bytes"
 	"encoding/binary"
-	"git.gammaspectra.live/P2Pool/go-randomx/v2/asm"
+	"git.gammaspectra.live/P2Pool/go-randomx/v3/asm"
 )
 
 /*

randomx_test.go

@@ -31,7 +31,9 @@ package randomx
 
 import (
 	"fmt"
+	"os"
 	"runtime"
+	"slices"
 )
 import "testing"
 
@@ -47,9 +49,9 @@ var Tests = []struct {
 	{[]byte("test key 001"), []byte("sed do eiusmod tempor incididunt ut labore et dolore magna aliqua"), "e9ff4503201c0c2cca26d285c93ae883f9b1d30c9eb240b820756f2d5a7905fc"}, // test d
 }
 
-func Test_Randomx(t *testing.T) {
+func Test_RandomXLight(t *testing.T) {
 
-	c := Randomx_alloc_cache(0)
+	c := NewCache(0)
 
 	for ix, tt := range Tests {
 
@@ -62,7 +64,10 @@ func Test_Randomx(t *testing.T) {
 				}
 			}()
 
-			vm := c.VM_Initialize()
+			dataset := NewLightDataset(c)
+			dataset.InitDataset(0, DatasetItemCount)
+
+			vm := NewVM(dataset)
 			defer vm.Close()
 
 			var output_hash [32]byte
@@ -74,57 +79,119 @@ func Test_Randomx(t *testing.T) {
 			}
 		})
 	}
-
 }
 
-func Benchmark_RandomX(b *testing.B) {
+func Test_RandomXFull(t *testing.T) {
+
+	c := NewCache(0)
+
+	for ix, tt := range Tests {
+
+		t.Run(string(tt.key)+"_____"+string(tt.input), func(t *testing.T) {
+			c.Init(tt.key)
+			defer func() {
+				err := c.Close()
+				if err != nil {
+					t.Error(err)
+				}
+			}()
+
+			dataset := NewFullDataset(c)
+			InitDatasetParallel(dataset, runtime.NumCPU())
+
+			vm := NewVM(dataset)
+			defer vm.Close()
+
+			var output_hash [32]byte
+			vm.CalculateHash(tt.input, &output_hash)
+
+			actual := fmt.Sprintf("%x", output_hash)
+			if actual != tt.expected {
+				t.Errorf("#%d Fib(%v): expected %s, actual %s", ix, tt.key, tt.expected, actual)
+			}
+		})
+
+		// cleanup 2GiB between runs
+		runtime.GC()
+	}
+}
+
+var BenchmarkTest = Tests[0]
+var BenchmarkCache *Cache
+var BenchmarkDatasetLight *DatasetLight
+var BenchmarkDatasetFull *DatasetFull
+
+func TestMain(m *testing.M) {
+	if slices.Contains(os.Args, "-test.bench") {
+		//init light and full dataset
+		BenchmarkCache = NewCache(0)
+		BenchmarkCache.Init(BenchmarkTest.key)
+		BenchmarkDatasetLight = NewLightDataset(BenchmarkCache)
+		BenchmarkDatasetLight.InitDataset(0, DatasetItemCount)
+		BenchmarkDatasetFull = NewFullDataset(BenchmarkCache)
+		InitDatasetParallel(BenchmarkDatasetFull, runtime.NumCPU())
+		defer BenchmarkCache.Close()
+	}
+	os.Exit(m.Run())
+}
+
+func Benchmark_RandomXLight(b *testing.B) {
 	b.ReportAllocs()
 
-	tt := Tests[0]
-
-	c := Randomx_alloc_cache(0)
-
-	c.Init(tt.key)
-	defer func() {
-		err := c.Close()
-		if err != nil {
-			b.Error(err)
-		}
-	}()
-
-	vm := c.VM_Initialize()
+	vm := NewVM(BenchmarkDatasetLight)
 	defer vm.Close()
+
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		var output_hash [32]byte
-		vm.CalculateHash(tt.input, &output_hash)
+		vm.CalculateHash(BenchmarkTest.input, &output_hash)
 		runtime.KeepAlive(output_hash)
 	}
 }
 
-func Benchmark_RandomXParallel(b *testing.B) {
+func Benchmark_RandomXFull(b *testing.B) {
 	b.ReportAllocs()
 
-	tt := Tests[0]
-
-	c := Randomx_alloc_cache(0)
-
-	c.Init(tt.key)
-	defer func() {
-		err := c.Close()
-		if err != nil {
-			b.Error(err)
-		}
-	}()
+	vm := NewVM(BenchmarkDatasetFull)
+	defer vm.Close()
 
 	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		var output_hash [32]byte
+		vm.CalculateHash(BenchmarkTest.input, &output_hash)
+		runtime.KeepAlive(output_hash)
+	}
+}
+
+func Benchmark_RandomXLight_Parallel(b *testing.B) {
+	b.ReportAllocs()
+	b.ResetTimer()
+
 	b.RunParallel(func(pb *testing.PB) {
 		var output_hash [32]byte
-		vm := c.VM_Initialize()
+
+		vm := NewVM(BenchmarkDatasetLight)
 		defer vm.Close()
 
 		for pb.Next() {
-			vm.CalculateHash(tt.input, &output_hash)
+			vm.CalculateHash(BenchmarkTest.input, &output_hash)
+			runtime.KeepAlive(output_hash)
+		}
+	})
+}
+
+func Benchmark_RandomXFull_Parallel(b *testing.B) {
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		var output_hash [32]byte
+
+		vm := NewVM(BenchmarkDatasetFull)
+		defer vm.Close()
+
+		for pb.Next() {
+			vm.CalculateHash(BenchmarkTest.input, &output_hash)
 			runtime.KeepAlive(output_hash)
 		}
 	})

vm.go

@@ -30,7 +30,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 package randomx
 
 import (
-	"git.gammaspectra.live/P2Pool/go-randomx/v2/aes"
+	"git.gammaspectra.live/P2Pool/go-randomx/v3/aes"
 	"math"
 	"runtime"
 	"unsafe"
@@ -45,11 +45,25 @@ type REG struct {
 type VM struct {
 	ScratchPad ScratchPad
 
-	Dataset Randomx_Dataset
+	Dataset Dataset
 
 	JITProgram VMProgramFunc
 }
 
+func NewVM(dataset Dataset) *VM {
+
+	vm := &VM{
+		Dataset: dataset,
+	}
+	if dataset.Cache().HasJIT() {
+		vm.JITProgram = mapProgram(nil, int(RandomXCodeSize))
+		if dataset.Flags()&RANDOMX_FLAG_SECURE == 0 {
+			mapProgramRWX(vm.JITProgram)
+		}
+	}
+	return vm
+}
+
 // Run calculate hash based on input
 // Warning: Underlying callers will run float64 SetRoundingMode directly
 // It is the caller's responsibility to set and restore the mode to IEEE 754 roundTiesToEven between full executions
@@ -86,7 +100,7 @@ func (vm *VM) Run(inputHash [64]byte, roundingMode uint8) (reg RegisterFile) {
 		addressRegisters >>= 1
 	}
 
-	datasetOffset := (entropy[13] % (DATASETEXTRAITEMS + 1)) * CacheLineSize
+	datasetOffset := (entropy[13] % (DatasetExtraItems + 1)) * CacheLineSize
 
 	eMask := [2]uint64{EMask(entropy[14]), EMask(entropy[15])}
 
@@ -95,8 +109,6 @@ func (vm *VM) Run(inputHash [64]byte, roundingMode uint8) (reg RegisterFile) {
 	spAddr0 := mem.mx
 	spAddr1 := mem.ma
 
-	var rlCache RegisterLine
-
 	if vm.JITProgram != nil {
 		if vm.Dataset.Flags()&RANDOMX_FLAG_SECURE > 0 {
 			mapProgramRW(vm.JITProgram)
@@ -143,7 +155,7 @@ func (vm *VM) Run(inputHash [64]byte, roundingMode uint8) (reg RegisterFile) {
 
 		vm.Dataset.PrefetchDataset(datasetOffset + mem.mx)
 		// execute diffuser superscalar program to get dataset 64 bytes
-		vm.Dataset.ReadDataset(datasetOffset+mem.ma, &reg.R, &rlCache)
+		vm.Dataset.ReadDataset(datasetOffset+mem.ma, &reg.R)
 
 		// swap the elements
 		mem.mx, mem.ma = mem.ma, mem.mx

vm_bytecode_native.go

@@ -3,7 +3,7 @@
 package randomx
 
 import (
-	"git.gammaspectra.live/P2Pool/go-randomx/v2/asm"
+	"git.gammaspectra.live/P2Pool/go-randomx/v3/asm"
 	"math"
 	"math/bits"
 )

vm_instruction.go

@@ -30,7 +30,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 package randomx
 
 import (
-	"git.gammaspectra.live/P2Pool/go-randomx/v2/aes"
+	"git.gammaspectra.live/P2Pool/go-randomx/v3/aes"
 	"unsafe"
 )
 import "encoding/binary"