Use md5server pool as each one is good for 16 tasks, use channels for md5/sha256 hashers for better reuse
This commit is contained in:
parent
b2bd300ba2
commit
b951d638c0
|
@ -7,6 +7,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/minio/md5-simd"
|
"github.com/minio/md5-simd"
|
||||||
"github.com/minio/sha256-simd"
|
"github.com/minio/sha256-simd"
|
||||||
|
"hash"
|
||||||
"io"
|
"io"
|
||||||
"math"
|
"math"
|
||||||
"os"
|
"os"
|
||||||
|
@ -22,33 +23,25 @@ type HashFileResult struct {
|
||||||
md5 string
|
md5 string
|
||||||
}
|
}
|
||||||
|
|
||||||
func HashFile(results chan<- HashFileResult, jobs chan string, path string, md5server *md5simd.Server, taskCount *int64) {
|
func HashFile(results chan<- HashFileResult, md5hasher *md5simd.Hasher, sha256hasher *hash.Hash, path string) {
|
||||||
defer atomic.AddInt64(taskCount, -1)
|
|
||||||
fh, err := os.Open(path)
|
fh, err := os.Open(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
results <- HashFileResult{
|
results <- HashFileResult{
|
||||||
error: err,
|
error: err,
|
||||||
path: path,
|
path: path,
|
||||||
}
|
}
|
||||||
<-jobs
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer fh.Close()
|
defer fh.Close()
|
||||||
|
|
||||||
sha256sum := sha256.New()
|
io.Copy(io.MultiWriter(*sha256hasher, *md5hasher), fh)
|
||||||
md5sum := (*md5server).NewHash()
|
|
||||||
defer md5sum.Close()
|
|
||||||
|
|
||||||
io.Copy(io.MultiWriter(sha256sum, md5sum), fh)
|
|
||||||
|
|
||||||
results <- HashFileResult{
|
results <- HashFileResult{
|
||||||
error: nil,
|
error: nil,
|
||||||
path: path,
|
path: path,
|
||||||
sha256: hex.EncodeToString(sha256sum.Sum(nil)),
|
sha256: hex.EncodeToString((*sha256hasher).Sum(nil)),
|
||||||
md5: hex.EncodeToString(md5sum.Sum(nil)),
|
md5: hex.EncodeToString((*md5hasher).Sum(nil)),
|
||||||
}
|
}
|
||||||
|
|
||||||
<-jobs
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func PrintHashFileResult(result *HashFileResult) {
|
func PrintHashFileResult(result *HashFileResult) {
|
||||||
|
@ -69,13 +62,24 @@ func main() {
|
||||||
scanner := bufio.NewScanner(os.Stdin)
|
scanner := bufio.NewScanner(os.Stdin)
|
||||||
defer os.Stdin.Close()
|
defer os.Stdin.Close()
|
||||||
|
|
||||||
md5server := md5simd.NewServer()
|
var md5servers []md5simd.Server
|
||||||
defer md5server.Close()
|
md5hashers := make(chan md5simd.Hasher, *taskLimit)
|
||||||
|
sha256hashers := make(chan hash.Hash, *taskLimit)
|
||||||
|
|
||||||
|
for j := 0; j < *taskLimit; j++ {
|
||||||
|
serverIndex := j / 16
|
||||||
|
|
||||||
|
if (serverIndex + 1) > len(md5servers) {
|
||||||
|
md5servers = append(md5servers, md5simd.NewServer())
|
||||||
|
}
|
||||||
|
|
||||||
|
hasher := md5servers[serverIndex].NewHash()
|
||||||
|
md5hashers <- hasher
|
||||||
|
sha256hashers <- sha256.New()
|
||||||
|
}
|
||||||
|
|
||||||
resultChannel := make(chan HashFileResult)
|
resultChannel := make(chan HashFileResult)
|
||||||
|
|
||||||
jobs := make(chan string, *taskLimit)
|
|
||||||
|
|
||||||
atomic.AddInt64(&taskCount, 1)
|
atomic.AddInt64(&taskCount, 1)
|
||||||
go func() {
|
go func() {
|
||||||
defer atomic.AddInt64(&taskCount, -1)
|
defer atomic.AddInt64(&taskCount, -1)
|
||||||
|
@ -84,9 +88,17 @@ func main() {
|
||||||
|
|
||||||
path := strings.TrimRight(text, "\n\r")
|
path := strings.TrimRight(text, "\n\r")
|
||||||
|
|
||||||
jobs <- path
|
|
||||||
atomic.AddInt64(&taskCount, 1)
|
atomic.AddInt64(&taskCount, 1)
|
||||||
go HashFile(resultChannel, jobs, path, &md5server, &taskCount)
|
go func() {
|
||||||
|
defer atomic.AddInt64(&taskCount, -1)
|
||||||
|
md5hasher := <-md5hashers
|
||||||
|
sha256hasher := <-sha256hashers
|
||||||
|
md5hasher.Reset()
|
||||||
|
sha256hasher.Reset()
|
||||||
|
HashFile(resultChannel, &md5hasher, &sha256hasher, path)
|
||||||
|
md5hashers <- md5hasher
|
||||||
|
sha256hashers <- sha256hasher
|
||||||
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
}()
|
}()
|
||||||
|
@ -102,4 +114,15 @@ func main() {
|
||||||
for result := range resultChannel {
|
for result := range resultChannel {
|
||||||
PrintHashFileResult(&result)
|
PrintHashFileResult(&result)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
close(md5hashers)
|
||||||
|
close(sha256hashers)
|
||||||
|
|
||||||
|
for md5hasher := range md5hashers {
|
||||||
|
md5hasher.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, md5server := range md5servers {
|
||||||
|
md5server.Close()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue