Limit initial file loading

SynchRoGazer.go

@@ -2,14 +2,11 @@ package main
 
 import (
 	"bufio"
-	"crypto/md5"
-	sha256_stl "crypto/sha256"
 	"database/sql"
 	"encoding/hex"
 	"encoding/json"
 	"flag"
 	"fmt"
-	"github.com/klauspost/cpuid"
 	"github.com/lib/pq"
 	"github.com/minio/md5-simd"
 	"github.com/minio/sha256-simd"
@@ -30,7 +27,7 @@ type HashFileResult struct {
 	Size   uint64
 }
 
-func HashFile(results chan<- HashFileResult, md5hasher hash.Hash, sha256hasher hash.Hash, path string) {
+func HashFile(results chan<- HashFileResult, md5hasher *md5simd.Hasher, sha256hasher *hash.Hash, path string) {
 	fi, err := os.Stat(path)
 	if err != nil {
 		results <- HashFileResult{
@@ -58,13 +55,13 @@ func HashFile(results chan<- HashFileResult, md5hasher hash.Hash, sha256hasher h
 	}
 	defer fh.Close()
 
-	io.Copy(io.MultiWriter(sha256hasher, md5hasher), fh)
+	io.Copy(io.MultiWriter(*sha256hasher, *md5hasher), fh)
 
 	results <- HashFileResult{
 		Error:  nil,
 		Path:   path,
-		SHA256: hex.EncodeToString(sha256hasher.Sum(nil)),
-		MD5:    hex.EncodeToString(md5hasher.Sum(nil)),
+		SHA256: hex.EncodeToString((*sha256hasher).Sum(nil)),
+		MD5:    hex.EncodeToString((*md5hasher).Sum(nil)),
 		Size:   uint64(fi.Size()),
 	}
 }
@@ -245,28 +242,9 @@ func main() {
 	defer os.Stdin.Close()
 
 	var md5servers []md5simd.Server
-	md5hashers := make(chan hash.Hash, *taskLimit)
+	md5hashers := make(chan md5simd.Hasher, *taskLimit)
 	sha256hashers := make(chan hash.Hash, *taskLimit)
 
-	if !cpuid.CPU.Supports(cpuid.AVX2) {
-		for j := 0; j < *taskLimit; j++ {
-			md5hashers <- md5.New()
-			sha256hashers <- sha256_stl.New()
-		}
-	} else {
-		for j := 0; j < *taskLimit; j++ {
-			serverIndex := j / 16
-
-			if (serverIndex + 1) > len(md5servers) {
-				md5servers = append(md5servers, md5simd.NewServer())
-			}
-
-			hasher := md5servers[serverIndex].NewHash()
-			md5hashers <- hasher
-			sha256hashers <- sha256.New()
-		}
-	}
-
 	for j := 0; j < *taskLimit; j++ {
 		serverIndex := j / 16
 
@@ -300,7 +278,7 @@ func main() {
 				sha256hasher := <-sha256hashers
 				md5hasher.Reset()
 				sha256hasher.Reset()
-				HashFile(resultChannel, md5hasher, sha256hasher, path)
+				HashFile(resultChannel, &md5hasher, &sha256hasher, path)
 				md5hashers <- md5hasher
 				sha256hashers <- sha256hasher
 			}()
@@ -327,9 +305,7 @@ func main() {
 	close(sha256hashers)
 
 	for md5hasher := range md5hashers {
-		if h, ok := md5hasher.(md5simd.Hasher); ok {
-			h.Close()
-		}
+		md5hasher.Close()
 	}
 
 	for _, md5server := range md5servers {

go.mod

@@ -3,7 +3,6 @@ module git.gammaspectra.live/S.O.N.G/SynchRoGazer
 go 1.17
 
 require (
-	github.com/klauspost/cpuid v1.3.1
 	github.com/lib/pq v1.10.6
 	github.com/minio/md5-simd v1.1.2
 	github.com/minio/sha256-simd v1.0.0

go.sum

@@ -1,5 +1,3 @@
-github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
-github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0=