From c000a83682c586e82bbd8709fe766e9257951218 Mon Sep 17 00:00:00 2001 From: WeebDataHoarder <57538841+WeebDataHoarder@users.noreply.github.com> Date: Mon, 17 Jan 2022 14:12:05 +0100 Subject: [PATCH] Use Cloudflare Go fork, fixed caches not working, use faster signature verification --- Dockerfile | 21 ++++- OrbitalBeat.go | 207 ++++++++++++++++++++++++++++++++----------------- go.mod | 3 +- go.sum | 12 ++- 4 files changed, 162 insertions(+), 81 deletions(-) diff --git a/Dockerfile b/Dockerfile index e6e66ab..f3a6f2f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,22 @@ -FROM golang:1.18-rc-bullseye +FROM debian:bullseye -COPY . /src +RUN DEBIAN_FRONTEND=noninteractive apt update && \ + DEBIAN_FRONTEND=noninteractive apt install -y \ + git build-essential \ + golang -WORKDIR /src -RUN go build -o orbeat . && mv orbeat /usr/bin && rm -rf /src +RUN git clone --depth 1 https://github.com/cloudflare/go.git /src/go && cd /src/go && \ + cd src && \ + ./all.bash +RUN DEBIAN_FRONTEND=noninteractive apt remove -y golang && \ + DEBIAN_FRONTEND=noninteractive apt autoremove -y + +ENV PATH "$PATH:/src/go/bin" + +COPY . /src/orbeat + +WORKDIR /src/orbeat +RUN go build -o orbeat . && mv orbeat /usr/bin && rm -rf /src/orbeat WORKDIR / ENTRYPOINT ["/usr/bin/orbeat"] \ No newline at end of file diff --git a/OrbitalBeat.go b/OrbitalBeat.go index 2d59f99..40eea84 100644 --- a/OrbitalBeat.go +++ b/OrbitalBeat.go @@ -3,7 +3,6 @@ package main import ( "bytes" "crypto/ecdsa" - "crypto/ed25519" "crypto/elliptic" "crypto/rand" "crypto/tls" @@ -17,6 +16,7 @@ import ( "encoding/pem" "flag" "fmt" + "github.com/cloudflare/circl/sign/ed25519" "github.com/ipfs/go-cid" _ "github.com/lib/pq" "github.com/multiformats/go-multihash" @@ -29,39 +29,14 @@ import ( "path" "strings" "sync" - "sync/atomic" "time" ) type ContentCacheEntry struct { Entry ContentEntry - RefCount int64 AccessTime time.Time } -func (e *ContentCacheEntry) valid() bool { - return atomic.LoadInt64(&e.RefCount) > 0 -} - -func (e *ContentCacheEntry) borrow() *ContentCacheEntry { - if e.valid() { - if atomic.AddInt64(&e.RefCount, 1) > 1 { - return e - } else { //Has already been deallocated or is in the process - atomic.AddInt64(&e.RefCount, -1) - } - } - return nil -} - -func (e *ContentCacheEntry) release() { - objectCacheMutex.Lock() - if atomic.AddInt64(&e.RefCount, -1) == 0 { - delete(objectCache, e.Entry.Identifier) - } - objectCacheMutex.Unlock() -} - var base32Encoding = base32.NewEncoding("qpzry9x8gf2tvdw0s3jn54khce6mua7l").WithPadding(base32.NoPadding) var dbHandle *sql.DB @@ -72,7 +47,7 @@ var fdlimit int var fsHandler fasthttp.RequestHandler var objectCacheMutex sync.RWMutex -var objectCache = make(map[cid.Cid]*ContentCacheEntry) +var objectCache = make(map[string]*ContentCacheEntry) var messageCacheLimit int var messageCacheMutex sync.RWMutex @@ -97,8 +72,36 @@ func getFirstValidContentEntry(entries *[]ContentEntry) *ContentEntry { return nil } -func handleQueryRequest(ctx *fasthttp.RequestCtx, identifier cid.Cid, extraArguments []string) { +func guessMimeType(p string) string { + //Need a few extra types from base Go + ext := strings.ToLower(path.Ext(p)) + switch ext { + case ".flac": + fallthrough + case ".ogg": + fallthrough + case ".opus": + fallthrough + case ".tta": + fallthrough + case ".aac": + fallthrough + case ".wav": + fallthrough + case ".alac": + return "audio/" + ext[1:] + case ".m4a": + return "audio/mp4" + case ".mp3": + return "audio/mpeg;codecs=mp3" + } + + return "" +} + +func handleQueryRequest(ctx *fasthttp.RequestCtx, identifier cid.Cid, extraArguments []string) { + cTime := time.Now() var cacheEntry = tryGetCacheEntryForIdentifier(identifier) if cacheEntry == nil { @@ -107,8 +110,16 @@ func handleQueryRequest(ctx *fasthttp.RequestCtx, identifier cid.Cid, extraArgum entry := getFirstValidContentEntry(&result) if entry != nil { - cacheEntry = getCacheEntryForContentEntry(entry) + cacheEntry = getCacheEntryForContentEntry(entry, identifier) } + + pTime := cTime + cTime = time.Now() + addServerTimingMetric(ctx, "e", "Content Entry (MISS)", cTime.Sub(pTime)) + } else { + pTime := cTime + cTime = time.Now() + addServerTimingMetric(ctx, "e", "Content Entry (HIT)", cTime.Sub(pTime)) } if cacheEntry == nil { @@ -137,7 +148,6 @@ func handleQueryRequest(ctx *fasthttp.RequestCtx, identifier cid.Cid, extraArgum ctx.SetStatusCode(fasthttp.StatusNotFound) return } - defer cacheEntry.release() mh, _ := multihash.Decode(cacheEntry.Entry.Identifier.Hash()) @@ -157,6 +167,15 @@ func handleQueryRequest(ctx *fasthttp.RequestCtx, identifier cid.Cid, extraArgum ctx.Request.URI().Reset() ctx.Request.URI().SetPath(cacheEntry.Entry.Path) + + pTime := cTime + cTime = time.Now() + addServerTimingMetric(ctx, "s", "Content Serve", cTime.Sub(pTime)) + + mime := guessMimeType(cacheEntry.Entry.Path) + if len(mime) > 0 { + ctx.Response.Header.SetContentType(mime) + } fsHandler(ctx) } @@ -180,7 +199,7 @@ func setCORSHeaders(ctx *fasthttp.RequestCtx) { ctx.Response.Header.Set("Cross-Origin-Opener-Policy", "unsafe-none") } -func getCacheEntryForContentEntry(entry *ContentEntry) *ContentCacheEntry { +func getCacheEntryForContentEntry(entry *ContentEntry, originalIdentifier cid.Cid) *ContentCacheEntry { cacheEntry := tryGetCacheEntryForIdentifier(entry.Identifier) @@ -200,18 +219,22 @@ func getCacheEntryForContentEntry(entry *ContentEntry) *ContentCacheEntry { } if item != nil { - delete(objectCache, item.Entry.Identifier) - item.release() + delete(objectCache, item.Entry.Identifier.String()) } } - objectCache[entry.Identifier] = &ContentCacheEntry{ + c := &ContentCacheEntry{ Entry: *entry, - RefCount: 1, AccessTime: time.Now(), } - return objectCache[entry.Identifier].borrow() + objectCache[entry.Identifier.String()] = c + + if originalIdentifier.String() != entry.Identifier.String() { + objectCache[originalIdentifier.String()] = c + } + + return c } func tryGetCacheEntryForIdentifier(identifier cid.Cid) *ContentCacheEntry { @@ -219,9 +242,8 @@ func tryGetCacheEntryForIdentifier(identifier cid.Cid) *ContentCacheEntry { objectCacheMutex.RLock() defer objectCacheMutex.RUnlock() - cacheEntry, ok := objectCache[identifier] + cacheEntry, ok := objectCache[identifier.String()] if ok { - cacheEntry := cacheEntry.borrow() cacheEntry.AccessTime = time.Now().UTC() return cacheEntry } @@ -238,12 +260,25 @@ func IsTrustedPublicKey(key ed25519.PublicKey) bool { return false } +func addServerTimingMetric(ctx *fasthttp.RequestCtx, name string, desc string, d time.Duration) { + v := string(ctx.Response.Header.Peek("Server-Timing")) + if len(v) > 0 { + ctx.Response.Header.Set("Server-Timing", fmt.Sprintf("%s, %s;desc=\"%s\";dur=%.6F", v, name, desc, float64(d.Nanoseconds())/1e6)) + } else { + ctx.Response.Header.Set("Server-Timing", fmt.Sprintf("%s;desc=\"%s\";dur=%.6F", name, desc, float64(d.Nanoseconds())/1e6)) + } +} + func handle(ctx *fasthttp.RequestCtx) { if len(ctx.Host()) > 0 && string(ctx.Host()) == ctx.TLSConnectionState().ServerName { //Prevents rebinding / DNS stuff ctx.SetStatusCode(fasthttp.StatusNotFound) return } + cTime := time.Now() + addServerTimingMetric(ctx, "c", "Connection", ctx.Time().Sub(ctx.ConnTime())) + addServerTimingMetric(ctx, "r", "Request Handler", cTime.Sub(ctx.Time())) + if ctx.IsGet() || ctx.IsHead() { log.Printf("Serve %s", string(ctx.Path())) setOtherHeaders(ctx) @@ -270,7 +305,21 @@ func handle(ctx *fasthttp.RequestCtx) { return } - if !message.verify() { + pTime := cTime + cTime := time.Now() + addServerTimingMetric(ctx, "d", "Decode", cTime.Sub(pTime)) + + result, cacheHit := message.verify() + + pTime = cTime + cTime = time.Now() + if cacheHit { + addServerTimingMetric(ctx, "v", "Ed25519 Verify (HIT)", cTime.Sub(pTime)) + } else { + addServerTimingMetric(ctx, "v", "Ed25519 Verify (MISS)", cTime.Sub(pTime)) + } + + if !result { ctx.SetStatusCode(fasthttp.StatusForbidden) return } @@ -304,7 +353,7 @@ func (s *ContentMessage) sign(privateKey ed25519.PrivateKey) { s.VerificationResult = nil } -func (s *ContentMessage) verify() bool { +func (s *ContentMessage) verify() (bool, bool) { currentTime := time.Now() issueTime := time.Unix(s.IssueTime, 0) @@ -312,11 +361,11 @@ func (s *ContentMessage) verify() bool { validityEnd := issueTime.Add(time.Hour * 24) //Only 24 hours after time if validityStart.After(currentTime) { - return false + return false, false } if validityEnd.Before(currentTime) { - return false + return false, false } messageCacheMutex.RLock() @@ -324,7 +373,7 @@ func (s *ContentMessage) verify() bool { cachedMessage, ok := messageCache[k] messageCacheMutex.RUnlock() if ok { - return *cachedMessage.VerificationResult + return *cachedMessage.VerificationResult, true } messageCacheMutex.Lock() @@ -351,7 +400,7 @@ func (s *ContentMessage) verify() bool { messageCache[k] = s - return *s.VerificationResult + return *s.VerificationResult, false } func (s *ContentMessage) encodeMessage() []byte { @@ -531,6 +580,16 @@ func createSelfSignedCertificate() ([]byte, []byte) { }) } +func processCertificate(c *tls.Certificate) { + if c.Leaf == nil { + leaf, err := x509.ParseCertificate(c.Certificate[0]) + if err != nil { + return + } + c.Leaf = leaf + } +} + func main() { //TODO: OCSP certificatePath := flag.String("certificate", "", "Path to SSL certificate file.") @@ -544,7 +603,7 @@ func main() { sniAddressOption := flag.String("sni", "", "Define SNI address if desired. Empty will serve any requests regardless.") - fdLimitOption := flag.Int("fdlimit", 128, "Maximum number of lingering cached open files.") + fdLimitOption := flag.Int("fdlimit", 512, "Maximum number of lingering cached open files.") signatureCacheLimitOption := flag.Int("siglimit", 4096, "Maximum number of lingering valid signature cache results.") @@ -609,6 +668,8 @@ func main() { } else { serverCertificate = bogusCertificate } + processCertificate(&bogusCertificate) + processCertificate(&serverCertificate) fs := fasthttp.FS{ Root: "/", @@ -623,6 +684,34 @@ func main() { fsHandler = fs.NewRequestHandler() + tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + MaxVersion: 0, //max supported, currently TLS 1.3 + CurvePreferences: []tls.CurveID{ + tls.X25519, + tls.CurveP256, + tls.CurveP384, + }, + CipherSuites: []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + }, + PreferServerCipherSuites: false, + SessionTicketsDisabled: false, + Renegotiation: tls.RenegotiateFreelyAsClient, + NextProtos: []string{ + //"h2", + "http/1.1", + }, + GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { + if isSNIAllowed(info.ServerName) { + return &serverCertificate, nil + } + return &bogusCertificate, nil + }, + } + server := &fasthttp.Server{ ReadTimeout: 5 * time.Second, IdleTimeout: 15 * time.Second, @@ -631,33 +720,7 @@ func main() { NoDefaultDate: true, TCPKeepalive: false, DisableKeepalive: true, - TLSConfig: &tls.Config{ - MinVersion: tls.VersionTLS12, - MaxVersion: 0, //max supported, currently TLS 1.3 - CurvePreferences: []tls.CurveID{ - tls.X25519, - tls.CurveP256, - tls.CurveP384, - }, - CipherSuites: []uint16{ - tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - }, - PreferServerCipherSuites: false, - SessionTicketsDisabled: false, - Renegotiation: tls.RenegotiateFreelyAsClient, - NextProtos: []string{ - //"h2", - "http/1.1", - }, - GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { - if isSNIAllowed(info.ServerName) { - return &serverCertificate, nil - } - return &bogusCertificate, nil - }, - }, + TLSConfig: tlsConfig, } log.Printf("Serving on %s", *listenAddress) diff --git a/go.mod b/go.mod index 1a114ef..a10592e 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,9 @@ module git.gammaspectra.live/S.O.N.G/OrbitalBeat -go 1.14 +go 1.16 require ( + github.com/cloudflare/circl v1.1.0 github.com/ipfs/go-cid v0.1.0 github.com/lib/pq v1.10.4 github.com/multiformats/go-multihash v0.0.15 diff --git a/go.sum b/go.sum index 38bada1..2ad9b38 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,8 @@ github.com/andybalholm/brotli v1.0.2 h1:JKnhI/XQ75uFBTiuzXpzFrUriDPiZjlOSzh6wXogP0E= github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/ipfs/go-cid v0.1.0 h1:YN33LQulcRHjfom/i25yoOZR4Telp1Hr/2RU3d0PnC0= github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= @@ -33,21 +36,22 @@ github.com/valyala/fasthttp v1.32.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf h1:B2n+Zi5QeYRDAEodEu72OS36gmTWjgpXr2+cWcBW90o= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 h1:hZR0X1kPW+nwyJ9xRxqZk1vx5RUObAPBdKVvXPDUH/E= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac h1:oN6lz7iLW/YC7un8pq+9bOLyXrprv2+DKfkJY+2LJJw= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=