|
|
|
@ -2,11 +2,14 @@ package main
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"bufio"
|
|
|
|
|
"crypto/md5"
|
|
|
|
|
sha256_stl "crypto/sha256"
|
|
|
|
|
"database/sql"
|
|
|
|
|
"encoding/hex"
|
|
|
|
|
"encoding/json"
|
|
|
|
|
"flag"
|
|
|
|
|
"fmt"
|
|
|
|
|
"github.com/klauspost/cpuid"
|
|
|
|
|
"github.com/lib/pq"
|
|
|
|
|
"github.com/minio/md5-simd"
|
|
|
|
|
"github.com/minio/sha256-simd"
|
|
|
|
@ -26,7 +29,7 @@ type HashFileResult struct {
|
|
|
|
|
Size uint64
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func HashFile(results chan<- HashFileResult, md5hasher *md5simd.Hasher, sha256hasher *hash.Hash, path string) {
|
|
|
|
|
func HashFile(results chan<- HashFileResult, md5hasher hash.Hash, sha256hasher hash.Hash, path string) {
|
|
|
|
|
fi, err := os.Stat(path)
|
|
|
|
|
if err != nil {
|
|
|
|
|
results <- HashFileResult{
|
|
|
|
@ -54,13 +57,13 @@ func HashFile(results chan<- HashFileResult, md5hasher *md5simd.Hasher, sha256ha
|
|
|
|
|
}
|
|
|
|
|
defer fh.Close()
|
|
|
|
|
|
|
|
|
|
io.Copy(io.MultiWriter(*sha256hasher, *md5hasher), fh)
|
|
|
|
|
io.Copy(io.MultiWriter(sha256hasher, md5hasher), fh)
|
|
|
|
|
|
|
|
|
|
results <- HashFileResult{
|
|
|
|
|
Error: nil,
|
|
|
|
|
Path: path,
|
|
|
|
|
SHA256: hex.EncodeToString((*sha256hasher).Sum(nil)),
|
|
|
|
|
MD5: hex.EncodeToString((*md5hasher).Sum(nil)),
|
|
|
|
|
SHA256: hex.EncodeToString(sha256hasher.Sum(nil)),
|
|
|
|
|
MD5: hex.EncodeToString(md5hasher.Sum(nil)),
|
|
|
|
|
Size: uint64(fi.Size()),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@ -241,9 +244,28 @@ func main() {
|
|
|
|
|
defer os.Stdin.Close()
|
|
|
|
|
|
|
|
|
|
var md5servers []md5simd.Server
|
|
|
|
|
md5hashers := make(chan md5simd.Hasher, *taskLimit)
|
|
|
|
|
md5hashers := make(chan hash.Hash, *taskLimit)
|
|
|
|
|
sha256hashers := make(chan hash.Hash, *taskLimit)
|
|
|
|
|
|
|
|
|
|
if !cpuid.CPU.Supports(cpuid.AVX2) {
|
|
|
|
|
for j := 0; j < *taskLimit; j++ {
|
|
|
|
|
md5hashers <- md5.New()
|
|
|
|
|
sha256hashers <- sha256_stl.New()
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
for j := 0; j < *taskLimit; j++ {
|
|
|
|
|
serverIndex := j / 16
|
|
|
|
|
|
|
|
|
|
if (serverIndex + 1) > len(md5servers) {
|
|
|
|
|
md5servers = append(md5servers, md5simd.NewServer())
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
hasher := md5servers[serverIndex].NewHash()
|
|
|
|
|
md5hashers <- hasher
|
|
|
|
|
sha256hashers <- sha256.New()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for j := 0; j < *taskLimit; j++ {
|
|
|
|
|
serverIndex := j / 16
|
|
|
|
|
|
|
|
|
@ -272,7 +294,7 @@ func main() {
|
|
|
|
|
sha256hasher := <-sha256hashers
|
|
|
|
|
md5hasher.Reset()
|
|
|
|
|
sha256hasher.Reset()
|
|
|
|
|
HashFile(resultChannel, &md5hasher, &sha256hasher, path)
|
|
|
|
|
HashFile(resultChannel, md5hasher, sha256hasher, path)
|
|
|
|
|
md5hashers <- md5hasher
|
|
|
|
|
sha256hashers <- sha256hasher
|
|
|
|
|
}()
|
|
|
|
@ -299,7 +321,9 @@ func main() {
|
|
|
|
|
close(sha256hashers)
|
|
|
|
|
|
|
|
|
|
for md5hasher := range md5hashers {
|
|
|
|
|
md5hasher.Close()
|
|
|
|
|
if h, ok := md5hasher.(md5simd.Hasher); ok {
|
|
|
|
|
h.Close()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for _, md5server := range md5servers {
|
|
|
|
|