From b2bd300ba2a9a97a60aba2757ff5e621fa68ddc1 Mon Sep 17 00:00:00 2001
From: WeebDataHoarder <57538841+WeebDataHoarder@users.noreply.github.com>
Date: Tue, 7 Dec 2021 13:38:54 +0100
Subject: [PATCH] Initial version

---
 README.md       |  10 ++++-
 SynchRoGazer.go | 105 ++++++++++++++++++++++++++++++++++++++++++++++++
 go.mod          |   8 ++++
 go.sum          |   7 ++++
 4 files changed, 129 insertions(+), 1 deletion(-)
 create mode 100644 SynchRoGazer.go
 create mode 100644 go.mod
 create mode 100644 go.sum

diff --git a/README.md b/README.md
index 0518d10..ee43d76 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,11 @@
 # SynchRoGazer
 
-Mass static file serving tool indexed via hashes
\ No newline at end of file
+Mass static file indexing via hashes
+
+
+## Usage
+`$ go run .`
+
+To set a larger task limit (mind fdlimit): `$ go run . -tasklimit=128`
+
+Output on stdout. Errors on stderr.
\ No newline at end of file
diff --git a/SynchRoGazer.go b/SynchRoGazer.go
new file mode 100644
index 0000000..c6763f2
--- /dev/null
+++ b/SynchRoGazer.go
@@ -0,0 +1,105 @@
+package main
+
+import (
+	"bufio"
+	"encoding/hex"
+	"flag"
+	"fmt"
+	"github.com/minio/md5-simd"
+	"github.com/minio/sha256-simd"
+	"io"
+	"math"
+	"os"
+	"runtime"
+	"strings"
+	"sync/atomic"
+)
+
+type HashFileResult struct {
+	error  error
+	path   string
+	sha256 string
+	md5    string
+}
+
+func HashFile(results chan<- HashFileResult, jobs chan string, path string, md5server *md5simd.Server, taskCount *int64) {
+	defer atomic.AddInt64(taskCount, -1)
+	fh, err := os.Open(path)
+	if err != nil {
+		results <- HashFileResult{
+			error: err,
+			path:  path,
+		}
+		<-jobs
+		return
+	}
+	defer fh.Close()
+
+	sha256sum := sha256.New()
+	md5sum := (*md5server).NewHash()
+	defer md5sum.Close()
+
+	io.Copy(io.MultiWriter(sha256sum, md5sum), fh)
+
+	results <- HashFileResult{
+		error:  nil,
+		path:   path,
+		sha256: hex.EncodeToString(sha256sum.Sum(nil)),
+		md5:    hex.EncodeToString(md5sum.Sum(nil)),
+	}
+
+	<-jobs
+}
+
+func PrintHashFileResult(result *HashFileResult) {
+	if result.error != nil {
+		fmt.Fprintln(os.Stderr, result.path, "error: ", result.error)
+	} else {
+		fmt.Println(result.sha256, result.md5, result.path)
+	}
+}
+
+func main() {
+	taskLimit := flag.Int("tasklimit", int(math.Ceil(float64(runtime.NumCPU())*1.5)), "Maximum number of concurrent hashing tasks. Change to avoid fdlimit issues. Defaults to number of CPU cores * 1.5")
+
+	flag.Parse()
+
+	var taskCount int64
+
+	scanner := bufio.NewScanner(os.Stdin)
+	defer os.Stdin.Close()
+
+	md5server := md5simd.NewServer()
+	defer md5server.Close()
+
+	resultChannel := make(chan HashFileResult)
+
+	jobs := make(chan string, *taskLimit)
+
+	atomic.AddInt64(&taskCount, 1)
+	go func() {
+		defer atomic.AddInt64(&taskCount, -1)
+		for scanner.Scan() {
+			text := scanner.Text()
+
+			path := strings.TrimRight(text, "\n\r")
+
+			jobs <- path
+			atomic.AddInt64(&taskCount, 1)
+			go HashFile(resultChannel, jobs, path, &md5server, &taskCount)
+		}
+
+	}()
+
+	//Already print before finishing, use atomic ints instead of a WaitGroup
+	for atomic.LoadInt64(&taskCount) > 0 {
+		result := <-resultChannel
+		PrintHashFileResult(&result)
+	}
+
+	close(resultChannel)
+
+	for result := range resultChannel {
+		PrintHashFileResult(&result)
+	}
+}
diff --git a/go.mod b/go.mod
new file mode 100644
index 0000000..b3a8ae3
--- /dev/null
+++ b/go.mod
@@ -0,0 +1,8 @@
+module git.gammaspectra.live/S.O.N.G./SynchRoGazer
+
+go 1.14
+
+require (
+	github.com/minio/md5-simd v1.1.2
+	github.com/minio/sha256-simd v1.0.0
+)
diff --git a/go.sum b/go.sum
new file mode 100644
index 0000000..2e4b6db
--- /dev/null
+++ b/go.sum
@@ -0,0 +1,7 @@
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw=
+github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
+github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=