Compare commits

...

2 commits

Author SHA1 Message Date
DataHoarder 80f91c85f1
Add gcc to test
Some checks failed
continuous-integration/drone/push Build is failing
2022-08-04 18:06:52 +02:00
DataHoarder 5e17d0bc5d
Do not use SIMD when not supported 2022-08-04 18:05:45 +02:00
2 changed files with 32 additions and 7 deletions

View file

@ -7,6 +7,7 @@ steps:
- name: backend - name: backend
image: golang:1.19-alpine image: golang:1.19-alpine
commands: commands:
- apk update && apk add --no-cache gcc musl-dev
- GOAMD64=v2 go build -v -o srg . - GOAMD64=v2 go build -v -o srg .
... ...

View file

@ -2,11 +2,14 @@ package main
import ( import (
"bufio" "bufio"
"crypto/md5"
sha256_stl "crypto/sha256"
"database/sql" "database/sql"
"encoding/hex" "encoding/hex"
"encoding/json" "encoding/json"
"flag" "flag"
"fmt" "fmt"
"github.com/klauspost/cpuid"
"github.com/lib/pq" "github.com/lib/pq"
"github.com/minio/md5-simd" "github.com/minio/md5-simd"
"github.com/minio/sha256-simd" "github.com/minio/sha256-simd"
@ -26,7 +29,7 @@ type HashFileResult struct {
Size uint64 Size uint64
} }
func HashFile(results chan<- HashFileResult, md5hasher *md5simd.Hasher, sha256hasher *hash.Hash, path string) { func HashFile(results chan<- HashFileResult, md5hasher hash.Hash, sha256hasher hash.Hash, path string) {
fi, err := os.Stat(path) fi, err := os.Stat(path)
if err != nil { if err != nil {
results <- HashFileResult{ results <- HashFileResult{
@ -54,13 +57,13 @@ func HashFile(results chan<- HashFileResult, md5hasher *md5simd.Hasher, sha256ha
} }
defer fh.Close() defer fh.Close()
io.Copy(io.MultiWriter(*sha256hasher, *md5hasher), fh) io.Copy(io.MultiWriter(sha256hasher, md5hasher), fh)
results <- HashFileResult{ results <- HashFileResult{
Error: nil, Error: nil,
Path: path, Path: path,
SHA256: hex.EncodeToString((*sha256hasher).Sum(nil)), SHA256: hex.EncodeToString(sha256hasher.Sum(nil)),
MD5: hex.EncodeToString((*md5hasher).Sum(nil)), MD5: hex.EncodeToString(md5hasher.Sum(nil)),
Size: uint64(fi.Size()), Size: uint64(fi.Size()),
} }
} }
@ -241,9 +244,28 @@ func main() {
defer os.Stdin.Close() defer os.Stdin.Close()
var md5servers []md5simd.Server var md5servers []md5simd.Server
md5hashers := make(chan md5simd.Hasher, *taskLimit) md5hashers := make(chan hash.Hash, *taskLimit)
sha256hashers := make(chan hash.Hash, *taskLimit) sha256hashers := make(chan hash.Hash, *taskLimit)
if !cpuid.CPU.Supports(cpuid.AVX2) {
for j := 0; j < *taskLimit; j++ {
md5hashers <- md5.New()
sha256hashers <- sha256_stl.New()
}
} else {
for j := 0; j < *taskLimit; j++ {
serverIndex := j / 16
if (serverIndex + 1) > len(md5servers) {
md5servers = append(md5servers, md5simd.NewServer())
}
hasher := md5servers[serverIndex].NewHash()
md5hashers <- hasher
sha256hashers <- sha256.New()
}
}
for j := 0; j < *taskLimit; j++ { for j := 0; j < *taskLimit; j++ {
serverIndex := j / 16 serverIndex := j / 16
@ -272,7 +294,7 @@ func main() {
sha256hasher := <-sha256hashers sha256hasher := <-sha256hashers
md5hasher.Reset() md5hasher.Reset()
sha256hasher.Reset() sha256hasher.Reset()
HashFile(resultChannel, &md5hasher, &sha256hasher, path) HashFile(resultChannel, md5hasher, sha256hasher, path)
md5hashers <- md5hasher md5hashers <- md5hasher
sha256hashers <- sha256hasher sha256hashers <- sha256hasher
}() }()
@ -299,7 +321,9 @@ func main() {
close(sha256hashers) close(sha256hashers)
for md5hasher := range md5hashers { for md5hasher := range md5hashers {
md5hasher.Close() if h, ok := md5hasher.(md5simd.Hasher); ok {
h.Close()
}
} }
for _, md5server := range md5servers { for _, md5server := range md5servers {