Initial version
This commit is contained in:
parent
6645cbcaf4
commit
b2bd300ba2
10
README.md
10
README.md
|
@ -1,3 +1,11 @@
|
|||
# SynchRoGazer
|
||||
|
||||
Mass static file serving tool indexed via hashes
|
||||
Mass static file indexing via hashes
|
||||
|
||||
|
||||
## Usage
|
||||
`$ go run .`
|
||||
|
||||
To set a larger task limit (mind fdlimit): `$ go run . -tasklimit=128`
|
||||
|
||||
Output on stdout. Errors on stderr.
|
105
SynchRoGazer.go
Normal file
105
SynchRoGazer.go
Normal file
|
@ -0,0 +1,105 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/hex"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/minio/md5-simd"
|
||||
"github.com/minio/sha256-simd"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
type HashFileResult struct {
|
||||
error error
|
||||
path string
|
||||
sha256 string
|
||||
md5 string
|
||||
}
|
||||
|
||||
func HashFile(results chan<- HashFileResult, jobs chan string, path string, md5server *md5simd.Server, taskCount *int64) {
|
||||
defer atomic.AddInt64(taskCount, -1)
|
||||
fh, err := os.Open(path)
|
||||
if err != nil {
|
||||
results <- HashFileResult{
|
||||
error: err,
|
||||
path: path,
|
||||
}
|
||||
<-jobs
|
||||
return
|
||||
}
|
||||
defer fh.Close()
|
||||
|
||||
sha256sum := sha256.New()
|
||||
md5sum := (*md5server).NewHash()
|
||||
defer md5sum.Close()
|
||||
|
||||
io.Copy(io.MultiWriter(sha256sum, md5sum), fh)
|
||||
|
||||
results <- HashFileResult{
|
||||
error: nil,
|
||||
path: path,
|
||||
sha256: hex.EncodeToString(sha256sum.Sum(nil)),
|
||||
md5: hex.EncodeToString(md5sum.Sum(nil)),
|
||||
}
|
||||
|
||||
<-jobs
|
||||
}
|
||||
|
||||
func PrintHashFileResult(result *HashFileResult) {
|
||||
if result.error != nil {
|
||||
fmt.Fprintln(os.Stderr, result.path, "error: ", result.error)
|
||||
} else {
|
||||
fmt.Println(result.sha256, result.md5, result.path)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
taskLimit := flag.Int("tasklimit", int(math.Ceil(float64(runtime.NumCPU())*1.5)), "Maximum number of concurrent hashing tasks. Change to avoid fdlimit issues. Defaults to number of CPU cores * 1.5")
|
||||
|
||||
flag.Parse()
|
||||
|
||||
var taskCount int64
|
||||
|
||||
scanner := bufio.NewScanner(os.Stdin)
|
||||
defer os.Stdin.Close()
|
||||
|
||||
md5server := md5simd.NewServer()
|
||||
defer md5server.Close()
|
||||
|
||||
resultChannel := make(chan HashFileResult)
|
||||
|
||||
jobs := make(chan string, *taskLimit)
|
||||
|
||||
atomic.AddInt64(&taskCount, 1)
|
||||
go func() {
|
||||
defer atomic.AddInt64(&taskCount, -1)
|
||||
for scanner.Scan() {
|
||||
text := scanner.Text()
|
||||
|
||||
path := strings.TrimRight(text, "\n\r")
|
||||
|
||||
jobs <- path
|
||||
atomic.AddInt64(&taskCount, 1)
|
||||
go HashFile(resultChannel, jobs, path, &md5server, &taskCount)
|
||||
}
|
||||
|
||||
}()
|
||||
|
||||
//Already print before finishing, use atomic ints instead of a WaitGroup
|
||||
for atomic.LoadInt64(&taskCount) > 0 {
|
||||
result := <-resultChannel
|
||||
PrintHashFileResult(&result)
|
||||
}
|
||||
|
||||
close(resultChannel)
|
||||
|
||||
for result := range resultChannel {
|
||||
PrintHashFileResult(&result)
|
||||
}
|
||||
}
|
8
go.mod
Normal file
8
go.mod
Normal file
|
@ -0,0 +1,8 @@
|
|||
module git.gammaspectra.live/S.O.N.G./SynchRoGazer
|
||||
|
||||
go 1.14
|
||||
|
||||
require (
|
||||
github.com/minio/md5-simd v1.1.2
|
||||
github.com/minio/sha256-simd v1.0.0
|
||||
)
|
7
go.sum
Normal file
7
go.sum
Normal file
|
@ -0,0 +1,7 @@
|
|||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw=
|
||||
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
|
||||
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
|
Loading…
Reference in a new issue