2021-12-07 12:38:54 +00:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bufio"
|
|
|
|
"encoding/hex"
|
2021-12-07 15:10:46 +00:00
|
|
|
"encoding/json"
|
2021-12-07 12:38:54 +00:00
|
|
|
"flag"
|
|
|
|
"fmt"
|
|
|
|
"github.com/minio/md5-simd"
|
|
|
|
"github.com/minio/sha256-simd"
|
2021-12-07 13:50:34 +00:00
|
|
|
"hash"
|
2021-12-07 12:38:54 +00:00
|
|
|
"io"
|
|
|
|
"os"
|
|
|
|
"runtime"
|
|
|
|
"sync/atomic"
|
|
|
|
)
|
|
|
|
|
|
|
|
type HashFileResult struct {
|
2021-12-07 15:10:46 +00:00
|
|
|
Error error
|
|
|
|
Path string
|
|
|
|
SHA256 string
|
|
|
|
MD5 string
|
2021-12-07 12:38:54 +00:00
|
|
|
}
|
|
|
|
|
2021-12-07 13:50:34 +00:00
|
|
|
func HashFile(results chan<- HashFileResult, md5hasher *md5simd.Hasher, sha256hasher *hash.Hash, path string) {
|
2021-12-07 12:38:54 +00:00
|
|
|
fh, err := os.Open(path)
|
|
|
|
if err != nil {
|
|
|
|
results <- HashFileResult{
|
2021-12-07 15:10:46 +00:00
|
|
|
Error: err,
|
|
|
|
Path: path,
|
2021-12-07 12:38:54 +00:00
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
defer fh.Close()
|
|
|
|
|
2021-12-07 13:50:34 +00:00
|
|
|
io.Copy(io.MultiWriter(*sha256hasher, *md5hasher), fh)
|
2021-12-07 12:38:54 +00:00
|
|
|
|
|
|
|
results <- HashFileResult{
|
2021-12-07 15:10:46 +00:00
|
|
|
Error: nil,
|
|
|
|
Path: path,
|
|
|
|
SHA256: hex.EncodeToString((*sha256hasher).Sum(nil)),
|
|
|
|
MD5: hex.EncodeToString((*md5hasher).Sum(nil)),
|
2021-12-07 12:38:54 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-12-07 15:10:46 +00:00
|
|
|
func PrintHashFileResult(result *HashFileResult, format string) {
|
|
|
|
switch {
|
|
|
|
case format == "json":
|
|
|
|
var jsonData []byte
|
|
|
|
jsonData, err := json.Marshal(*result)
|
|
|
|
if err != nil {
|
|
|
|
fmt.Fprintln(os.Stderr, result.Path, "Error: ", result.Error)
|
|
|
|
} else {
|
|
|
|
if result.Error != nil {
|
|
|
|
fmt.Fprintln(os.Stderr, string(jsonData))
|
|
|
|
} else {
|
|
|
|
fmt.Println(string(jsonData))
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
case format == "text":
|
|
|
|
if result.Error != nil {
|
|
|
|
fmt.Fprintln(os.Stderr, result.Path, "Error: ", result.Error)
|
|
|
|
} else {
|
|
|
|
fmt.Println(result.SHA256, result.MD5, result.Path)
|
|
|
|
}
|
|
|
|
|
2021-12-07 12:38:54 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func main() {
|
2021-12-07 13:54:32 +00:00
|
|
|
taskLimit := flag.Int("tasklimit", func() int {
|
|
|
|
result := runtime.NumCPU() * 16
|
|
|
|
if result > 128 {
|
|
|
|
return 128
|
|
|
|
}
|
|
|
|
return result
|
|
|
|
}(), "Maximum number of concurrent hashing tasks. Change to avoid fdlimit issues. Defaults to number of min(128, CPU cores * 16)")
|
2021-12-07 12:38:54 +00:00
|
|
|
|
2021-12-07 15:10:46 +00:00
|
|
|
outputFormat := flag.String("format", "text", "Output formats. Allowed: text, json")
|
|
|
|
|
2021-12-07 12:38:54 +00:00
|
|
|
flag.Parse()
|
|
|
|
|
|
|
|
var taskCount int64
|
|
|
|
|
|
|
|
scanner := bufio.NewScanner(os.Stdin)
|
|
|
|
defer os.Stdin.Close()
|
|
|
|
|
2021-12-07 13:50:34 +00:00
|
|
|
var md5servers []md5simd.Server
|
|
|
|
md5hashers := make(chan md5simd.Hasher, *taskLimit)
|
|
|
|
sha256hashers := make(chan hash.Hash, *taskLimit)
|
2021-12-07 12:38:54 +00:00
|
|
|
|
2021-12-07 13:50:34 +00:00
|
|
|
for j := 0; j < *taskLimit; j++ {
|
|
|
|
serverIndex := j / 16
|
2021-12-07 12:38:54 +00:00
|
|
|
|
2021-12-07 13:50:34 +00:00
|
|
|
if (serverIndex + 1) > len(md5servers) {
|
|
|
|
md5servers = append(md5servers, md5simd.NewServer())
|
|
|
|
}
|
|
|
|
|
|
|
|
hasher := md5servers[serverIndex].NewHash()
|
|
|
|
md5hashers <- hasher
|
|
|
|
sha256hashers <- sha256.New()
|
|
|
|
}
|
|
|
|
|
|
|
|
resultChannel := make(chan HashFileResult)
|
2021-12-07 12:38:54 +00:00
|
|
|
|
|
|
|
atomic.AddInt64(&taskCount, 1)
|
|
|
|
go func() {
|
|
|
|
defer atomic.AddInt64(&taskCount, -1)
|
|
|
|
for scanner.Scan() {
|
2021-12-07 14:01:14 +00:00
|
|
|
path := scanner.Text()
|
2021-12-07 12:38:54 +00:00
|
|
|
|
|
|
|
atomic.AddInt64(&taskCount, 1)
|
2021-12-07 13:50:34 +00:00
|
|
|
go func() {
|
|
|
|
defer atomic.AddInt64(&taskCount, -1)
|
|
|
|
md5hasher := <-md5hashers
|
|
|
|
sha256hasher := <-sha256hashers
|
|
|
|
md5hasher.Reset()
|
|
|
|
sha256hasher.Reset()
|
|
|
|
HashFile(resultChannel, &md5hasher, &sha256hasher, path)
|
|
|
|
md5hashers <- md5hasher
|
|
|
|
sha256hashers <- sha256hasher
|
|
|
|
}()
|
2021-12-07 12:38:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
}()
|
|
|
|
|
|
|
|
//Already print before finishing, use atomic ints instead of a WaitGroup
|
|
|
|
for atomic.LoadInt64(&taskCount) > 0 {
|
|
|
|
result := <-resultChannel
|
2021-12-07 15:10:46 +00:00
|
|
|
PrintHashFileResult(&result, *outputFormat)
|
2021-12-07 12:38:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
close(resultChannel)
|
|
|
|
|
|
|
|
for result := range resultChannel {
|
2021-12-07 15:10:46 +00:00
|
|
|
PrintHashFileResult(&result, *outputFormat)
|
2021-12-07 12:38:54 +00:00
|
|
|
}
|
2021-12-07 13:50:34 +00:00
|
|
|
|
|
|
|
close(md5hashers)
|
|
|
|
close(sha256hashers)
|
|
|
|
|
|
|
|
for md5hasher := range md5hashers {
|
|
|
|
md5hasher.Close()
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, md5server := range md5servers {
|
|
|
|
md5server.Close()
|
|
|
|
}
|
2021-12-07 12:38:54 +00:00
|
|
|
}
|