Compare commits

...

2 commits

Author SHA1 Message Date
DataHoarder 80f91c85f1
Add gcc to test
Some checks failed
continuous-integration/drone/push Build is failing
2022-08-04 18:06:52 +02:00
DataHoarder 5e17d0bc5d
Do not use SIMD when not supported 2022-08-04 18:05:45 +02:00
2 changed files with 32 additions and 7 deletions

View file

@ -7,6 +7,7 @@ steps:
- name: backend
image: golang:1.19-alpine
commands:
- apk update && apk add --no-cache gcc musl-dev
- GOAMD64=v2 go build -v -o srg .
...

View file

@ -2,11 +2,14 @@ package main
import (
"bufio"
"crypto/md5"
sha256_stl "crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"flag"
"fmt"
"github.com/klauspost/cpuid"
"github.com/lib/pq"
"github.com/minio/md5-simd"
"github.com/minio/sha256-simd"
@ -26,7 +29,7 @@ type HashFileResult struct {
Size uint64
}
func HashFile(results chan<- HashFileResult, md5hasher *md5simd.Hasher, sha256hasher *hash.Hash, path string) {
func HashFile(results chan<- HashFileResult, md5hasher hash.Hash, sha256hasher hash.Hash, path string) {
fi, err := os.Stat(path)
if err != nil {
results <- HashFileResult{
@ -54,13 +57,13 @@ func HashFile(results chan<- HashFileResult, md5hasher *md5simd.Hasher, sha256ha
}
defer fh.Close()
io.Copy(io.MultiWriter(*sha256hasher, *md5hasher), fh)
io.Copy(io.MultiWriter(sha256hasher, md5hasher), fh)
results <- HashFileResult{
Error: nil,
Path: path,
SHA256: hex.EncodeToString((*sha256hasher).Sum(nil)),
MD5: hex.EncodeToString((*md5hasher).Sum(nil)),
SHA256: hex.EncodeToString(sha256hasher.Sum(nil)),
MD5: hex.EncodeToString(md5hasher.Sum(nil)),
Size: uint64(fi.Size()),
}
}
@ -241,9 +244,28 @@ func main() {
defer os.Stdin.Close()
var md5servers []md5simd.Server
md5hashers := make(chan md5simd.Hasher, *taskLimit)
md5hashers := make(chan hash.Hash, *taskLimit)
sha256hashers := make(chan hash.Hash, *taskLimit)
if !cpuid.CPU.Supports(cpuid.AVX2) {
for j := 0; j < *taskLimit; j++ {
md5hashers <- md5.New()
sha256hashers <- sha256_stl.New()
}
} else {
for j := 0; j < *taskLimit; j++ {
serverIndex := j / 16
if (serverIndex + 1) > len(md5servers) {
md5servers = append(md5servers, md5simd.NewServer())
}
hasher := md5servers[serverIndex].NewHash()
md5hashers <- hasher
sha256hashers <- sha256.New()
}
}
for j := 0; j < *taskLimit; j++ {
serverIndex := j / 16
@ -272,7 +294,7 @@ func main() {
sha256hasher := <-sha256hashers
md5hasher.Reset()
sha256hasher.Reset()
HashFile(resultChannel, &md5hasher, &sha256hasher, path)
HashFile(resultChannel, md5hasher, sha256hasher, path)
md5hashers <- md5hasher
sha256hashers <- sha256hasher
}()
@ -299,7 +321,9 @@ func main() {
close(sha256hashers)
for md5hasher := range md5hashers {
md5hasher.Close()
if h, ok := md5hasher.(md5simd.Hasher); ok {
h.Close()
}
}
for _, md5server := range md5servers {