Adjust go.mod

.drone.yml

@@ -7,6 +7,7 @@ steps:
   - name: backend
     image: golang:1.19-alpine
     commands:
+      - apk update && apk add --no-cache gcc musl-dev
       - GOAMD64=v2 go build -v -o srg .
 
 ...

SynchRoGazer.go

@@ -2,11 +2,14 @@ package main
 
 import (
 	"bufio"
+	"crypto/md5"
+	sha256_stl "crypto/sha256"
 	"database/sql"
 	"encoding/hex"
 	"encoding/json"
 	"flag"
 	"fmt"
+	"github.com/klauspost/cpuid"
 	"github.com/lib/pq"
 	"github.com/minio/md5-simd"
 	"github.com/minio/sha256-simd"
@@ -16,6 +19,7 @@ import (
 	"os"
 	"runtime"
 	"sync/atomic"
+	"time"
 )
 
 type HashFileResult struct {
@@ -26,7 +30,7 @@ type HashFileResult struct {
 	Size   uint64
 }
 
-func HashFile(results chan<- HashFileResult, md5hasher *md5simd.Hasher, sha256hasher *hash.Hash, path string) {
+func HashFile(results chan<- HashFileResult, md5hasher hash.Hash, sha256hasher hash.Hash, path string) {
 	fi, err := os.Stat(path)
 	if err != nil {
 		results <- HashFileResult{
@@ -54,13 +58,13 @@ func HashFile(results chan<- HashFileResult, md5hasher *md5simd.Hasher, sha256ha
 	}
 	defer fh.Close()
 
-	io.Copy(io.MultiWriter(*sha256hasher, *md5hasher), fh)
+	io.Copy(io.MultiWriter(sha256hasher, md5hasher), fh)
 
 	results <- HashFileResult{
 		Error:  nil,
 		Path:   path,
-		SHA256: hex.EncodeToString((*sha256hasher).Sum(nil)),
-		MD5:    hex.EncodeToString((*md5hasher).Sum(nil)),
+		SHA256: hex.EncodeToString(sha256hasher.Sum(nil)),
+		MD5:    hex.EncodeToString(md5hasher.Sum(nil)),
 		Size:   uint64(fi.Size()),
 	}
 }
@@ -241,9 +245,28 @@ func main() {
 	defer os.Stdin.Close()
 
 	var md5servers []md5simd.Server
-	md5hashers := make(chan md5simd.Hasher, *taskLimit)
+	md5hashers := make(chan hash.Hash, *taskLimit)
 	sha256hashers := make(chan hash.Hash, *taskLimit)
 
+	if !cpuid.CPU.Supports(cpuid.AVX2) {
+		for j := 0; j < *taskLimit; j++ {
+			md5hashers <- md5.New()
+			sha256hashers <- sha256_stl.New()
+		}
+	} else {
+		for j := 0; j < *taskLimit; j++ {
+			serverIndex := j / 16
+
+			if (serverIndex + 1) > len(md5servers) {
+				md5servers = append(md5servers, md5simd.NewServer())
+			}
+
+			hasher := md5servers[serverIndex].NewHash()
+			md5hashers <- hasher
+			sha256hashers <- sha256.New()
+		}
+	}
+
 	for j := 0; j < *taskLimit; j++ {
 		serverIndex := j / 16
 
@@ -265,6 +288,11 @@ func main() {
 			path := scanner.Text()
 			atomic.AddInt64(&entries, 1)
 
+			for atomic.LoadInt64(&taskCount) >= (int64(*taskLimit) + 1) {
+				runtime.Gosched()
+				time.Sleep(time.Millisecond * 20)
+			}
+
 			atomic.AddInt64(&taskCount, 1)
 			go func() {
 				defer atomic.AddInt64(&taskCount, -1)
@@ -272,7 +300,7 @@ func main() {
 				sha256hasher := <-sha256hashers
 				md5hasher.Reset()
 				sha256hasher.Reset()
-				HashFile(resultChannel, &md5hasher, &sha256hasher, path)
+				HashFile(resultChannel, md5hasher, sha256hasher, path)
 				md5hashers <- md5hasher
 				sha256hashers <- sha256hasher
 			}()
@@ -299,7 +327,9 @@ func main() {
 	close(sha256hashers)
 
 	for md5hasher := range md5hashers {
-		md5hasher.Close()
+		if h, ok := md5hasher.(md5simd.Hasher); ok {
+			h.Close()
+		}
 	}
 
 	for _, md5server := range md5servers {

go.mod

@@ -3,6 +3,7 @@ module git.gammaspectra.live/S.O.N.G/SynchRoGazer
 go 1.17
 
 require (
+	github.com/klauspost/cpuid v1.3.1
 	github.com/lib/pq v1.10.6
 	github.com/minio/md5-simd v1.1.2
 	github.com/minio/sha256-simd v1.0.0

go.sum

@@ -1,3 +1,5 @@
+github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
+github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0=