Added basic statistics under /stats
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
This commit is contained in:
parent
82375a7b0e
commit
6617760c7f
137
OrbitalBeat.go
137
OrbitalBeat.go
|
@ -5,6 +5,7 @@ import (
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"git.gammaspectra.live/S.O.N.G/MakyuuIchaival"
|
"git.gammaspectra.live/S.O.N.G/MakyuuIchaival"
|
||||||
|
@ -20,8 +21,10 @@ import (
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -33,6 +36,7 @@ type ContentCacheEntry struct {
|
||||||
var dbHandle *sql.DB
|
var dbHandle *sql.DB
|
||||||
var sha256Statement *sql.Stmt
|
var sha256Statement *sql.Stmt
|
||||||
var md5Statement *sql.Stmt
|
var md5Statement *sql.Stmt
|
||||||
|
var statsStatement *sql.Stmt
|
||||||
var fdlimit int
|
var fdlimit int
|
||||||
|
|
||||||
var objectCacheMutex sync.RWMutex
|
var objectCacheMutex sync.RWMutex
|
||||||
|
@ -42,6 +46,41 @@ var trustedPublicKeys []ed25519.PublicKey
|
||||||
|
|
||||||
var debugOutput = false
|
var debugOutput = false
|
||||||
|
|
||||||
|
type statistics struct {
|
||||||
|
Served servedStatistics `json:"served"`
|
||||||
|
ContentCache contentCacheStatistics `json:"content_cache"`
|
||||||
|
SignatureCache signatureCacheStatistics `json:"signature_cache"`
|
||||||
|
HttpCode httpCodeStatistics `json:"http_code"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type servedStatistics struct {
|
||||||
|
TotalBytesServed uint64 `json:"bytes"`
|
||||||
|
TotalHashesServed uint64 `json:"hashes"`
|
||||||
|
TotalContentRequestsServed uint64 `json:"content_requests"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type contentCacheStatistics struct {
|
||||||
|
ContentCacheHit uint64 `json:"cache_hit"`
|
||||||
|
ContentCacheMiss uint64 `json:"cache_miss"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type signatureCacheStatistics struct {
|
||||||
|
SignatureCacheHit uint64 `json:"cache_hit"`
|
||||||
|
SignatureCacheMiss uint64 `json:"cache_miss"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type httpCodeStatistics struct {
|
||||||
|
Code200 uint64 `json:"200"`
|
||||||
|
Code204 uint64 `json:"204"`
|
||||||
|
Code302 uint64 `json:"302"`
|
||||||
|
Code400 uint64 `json:"400"`
|
||||||
|
Code403 uint64 `json:"403"`
|
||||||
|
Code404 uint64 `json:"404"`
|
||||||
|
Code501 uint64 `json:"501"`
|
||||||
|
}
|
||||||
|
|
||||||
|
var globalStatistics statistics
|
||||||
|
|
||||||
func getFirstValidContentEntry(entries *[]ContentEntry) *ContentEntry {
|
func getFirstValidContentEntry(entries *[]ContentEntry) *ContentEntry {
|
||||||
for _, entry := range *entries {
|
for _, entry := range *entries {
|
||||||
stat, err := os.Stat(entry.Path)
|
stat, err := os.Stat(entry.Path)
|
||||||
|
@ -101,19 +140,13 @@ func GetMimeTypeFromExtension(ext string) string {
|
||||||
//Image types
|
//Image types
|
||||||
case "png":
|
case "png":
|
||||||
return "image/png"
|
return "image/png"
|
||||||
case "jfif":
|
case "jpg", "jpeg", "jfif":
|
||||||
fallthrough
|
|
||||||
case "jpeg":
|
|
||||||
fallthrough
|
|
||||||
case "jpg":
|
|
||||||
return "image/jpeg"
|
return "image/jpeg"
|
||||||
case "gif":
|
case "gif":
|
||||||
return "image/gif"
|
return "image/gif"
|
||||||
case "svg":
|
case "svg":
|
||||||
return "image/svg+xml"
|
return "image/svg+xml"
|
||||||
case "tiff":
|
case "tif", "tiff":
|
||||||
fallthrough
|
|
||||||
case "tif":
|
|
||||||
return "image/tiff"
|
return "image/tiff"
|
||||||
case "webp":
|
case "webp":
|
||||||
return "image/webp"
|
return "image/webp"
|
||||||
|
@ -178,8 +211,10 @@ func handleQueryRequest(ctx httputils.RequestContext, identifier cid.Cid, extraA
|
||||||
if entry != nil {
|
if entry != nil {
|
||||||
cacheEntry = getCacheEntryForContentEntry(entry, identifier)
|
cacheEntry = getCacheEntryForContentEntry(entry, identifier)
|
||||||
}
|
}
|
||||||
|
atomic.AddUint64(&globalStatistics.ContentCache.ContentCacheMiss, 1)
|
||||||
ctx.AddTimingInformational("ec", "Content Cache MISS")
|
ctx.AddTimingInformational("ec", "Content Cache MISS")
|
||||||
} else {
|
} else {
|
||||||
|
atomic.AddUint64(&globalStatistics.ContentCache.ContentCacheHit, 1)
|
||||||
ctx.AddTimingInformational("ec", "Content Cache HIT")
|
ctx.AddTimingInformational("ec", "Content Cache HIT")
|
||||||
}
|
}
|
||||||
pTime := cTime
|
pTime := cTime
|
||||||
|
@ -207,10 +242,12 @@ func handleQueryRequest(ctx httputils.RequestContext, identifier cid.Cid, extraA
|
||||||
kind = "md5"
|
kind = "md5"
|
||||||
}
|
}
|
||||||
if kind != "" {
|
if kind != "" {
|
||||||
|
atomic.AddUint64(&globalStatistics.HttpCode.Code302, 1)
|
||||||
ctx.DoRedirect(fmt.Sprintf("%s/%s/%s/%s", origin, kind, hex.EncodeToString(mh.Digest), strings.Join(extraArguments, "/")), http.StatusFound)
|
ctx.DoRedirect(fmt.Sprintf("%s/%s/%s/%s", origin, kind, hex.EncodeToString(mh.Digest), strings.Join(extraArguments, "/")), http.StatusFound)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
atomic.AddUint64(&globalStatistics.HttpCode.Code404, 1)
|
||||||
ctx.SetResponseCode(http.StatusNotFound)
|
ctx.SetResponseCode(http.StatusNotFound)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -219,6 +256,7 @@ func handleQueryRequest(ctx httputils.RequestContext, identifier cid.Cid, extraA
|
||||||
|
|
||||||
file, err := os.Open(cacheEntry.Entry.Path)
|
file, err := os.Open(cacheEntry.Entry.Path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
atomic.AddUint64(&globalStatistics.HttpCode.Code404, 1)
|
||||||
ctx.SetResponseCode(http.StatusNotFound)
|
ctx.SetResponseCode(http.StatusNotFound)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -244,7 +282,41 @@ func handleQueryRequest(ctx httputils.RequestContext, identifier cid.Cid, extraA
|
||||||
ctx.SetResponseHeader("Content-Type", mime)
|
ctx.SetResponseHeader("Content-Type", mime)
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx.ServeStream(httputils.NewStreamFromFile(file))
|
atomic.AddUint64(&globalStatistics.Served.TotalContentRequestsServed, 1)
|
||||||
|
ss := &statsStream{
|
||||||
|
stream: httputils.NewStreamFromFile(file),
|
||||||
|
}
|
||||||
|
ctx.ServeStream(ss)
|
||||||
|
runtime.SetFinalizer(ss, (*statsStream).Close)
|
||||||
|
}
|
||||||
|
|
||||||
|
type statsStream struct {
|
||||||
|
stream httputils.DefinedStream
|
||||||
|
byteCount int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ss *statsStream) Read(p []byte) (n int, err error) {
|
||||||
|
ss.byteCount += int64(len(p))
|
||||||
|
return ss.stream.Read(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ss *statsStream) Seek(offset int64, whence int) (int64, error) {
|
||||||
|
return ss.stream.Seek(offset, whence)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ss *statsStream) Close() error {
|
||||||
|
atomic.AddUint64(&globalStatistics.Served.TotalBytesServed, uint64(ss.byteCount))
|
||||||
|
ss.byteCount = 0
|
||||||
|
|
||||||
|
return ss.stream.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ss *statsStream) Size() int64 {
|
||||||
|
return ss.stream.Size()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ss *statsStream) ModTime() time.Time {
|
||||||
|
return ss.stream.ModTime()
|
||||||
}
|
}
|
||||||
|
|
||||||
func setOtherHeaders(ctx httputils.RequestContext) {
|
func setOtherHeaders(ctx httputils.RequestContext) {
|
||||||
|
@ -300,12 +372,15 @@ func getCacheEntryForContentEntry(entry *ContentEntry, originalIdentifier cid.Ci
|
||||||
AccessTime: time.Now(),
|
AccessTime: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//TODO: make it not require a mutex
|
||||||
objectCache[entry.Identifier.String()] = c
|
objectCache[entry.Identifier.String()] = c
|
||||||
|
|
||||||
if originalIdentifier.String() != entry.Identifier.String() {
|
if originalIdentifier.String() != entry.Identifier.String() {
|
||||||
objectCache[originalIdentifier.String()] = c
|
objectCache[originalIdentifier.String()] = c
|
||||||
}
|
}
|
||||||
|
|
||||||
|
atomic.AddUint64(&globalStatistics.Served.TotalHashesServed, 1)
|
||||||
|
|
||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -334,6 +409,7 @@ func IsTrustedPublicKey(key ed25519.PublicKey) bool {
|
||||||
|
|
||||||
func handle(ctx httputils.RequestContext) {
|
func handle(ctx httputils.RequestContext) {
|
||||||
if len(ctx.GetHost()) > 0 && len(ctx.GetTLSServerName()) > 0 && strings.Split(ctx.GetHost(), ":")[0] != ctx.GetTLSServerName() { //Prevents rebinding / DNS stuff
|
if len(ctx.GetHost()) > 0 && len(ctx.GetTLSServerName()) > 0 && strings.Split(ctx.GetHost(), ":")[0] != ctx.GetTLSServerName() { //Prevents rebinding / DNS stuff
|
||||||
|
atomic.AddUint64(&globalStatistics.HttpCode.Code404, 1)
|
||||||
ctx.SetResponseCode(http.StatusNotFound)
|
ctx.SetResponseCode(http.StatusNotFound)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -350,24 +426,55 @@ func handle(ctx httputils.RequestContext) {
|
||||||
}
|
}
|
||||||
setOtherHeaders(ctx)
|
setOtherHeaders(ctx)
|
||||||
setCORSHeaders(ctx)
|
setCORSHeaders(ctx)
|
||||||
|
|
||||||
|
if ctx.GetPath() == "/stats" {
|
||||||
|
ctx.SetResponseHeader("Content-Type", "application/json")
|
||||||
|
atomic.AddUint64(&globalStatistics.HttpCode.Code200, 1)
|
||||||
|
ctx.SetResponseCode(http.StatusOK)
|
||||||
|
|
||||||
|
statsStruct := struct {
|
||||||
|
Statistics *statistics `json:"statistics"`
|
||||||
|
TotalEntries uint64 `json:"total_entries"`
|
||||||
|
TotalSize uint64 `json:"total_size"`
|
||||||
|
}{
|
||||||
|
Statistics: &globalStatistics,
|
||||||
|
}
|
||||||
|
|
||||||
|
if rows, err := statsStatement.Query(); err == nil {
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
if rows.Next() {
|
||||||
|
rows.Scan(&statsStruct.TotalEntries, &statsStruct.TotalSize)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
statBytes, _ := json.MarshalIndent(statsStruct, "", " ")
|
||||||
|
ctx.ServeBytes(statBytes)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
pathElements := strings.Split(ctx.GetPath(), "/")
|
pathElements := strings.Split(ctx.GetPath(), "/")
|
||||||
if len(pathElements) < 2 {
|
if len(pathElements) < 2 {
|
||||||
|
atomic.AddUint64(&globalStatistics.HttpCode.Code400, 1)
|
||||||
ctx.SetResponseCode(http.StatusBadRequest)
|
ctx.SetResponseCode(http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
messageBytes, err := MakyuuIchaival.Bech32Encoding.DecodeString(pathElements[1])
|
messageBytes, err := MakyuuIchaival.Bech32Encoding.DecodeString(pathElements[1])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
atomic.AddUint64(&globalStatistics.HttpCode.Code400, 1)
|
||||||
ctx.SetResponseCode(http.StatusBadRequest)
|
ctx.SetResponseCode(http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
message := contentmessage.DecodeContentMessage(messageBytes)
|
message := contentmessage.DecodeContentMessage(messageBytes)
|
||||||
if message == nil {
|
if message == nil {
|
||||||
|
atomic.AddUint64(&globalStatistics.HttpCode.Code400, 1)
|
||||||
ctx.SetResponseCode(http.StatusBadRequest)
|
ctx.SetResponseCode(http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if !IsTrustedPublicKey(message.PublicKey) {
|
if !IsTrustedPublicKey(message.PublicKey) {
|
||||||
|
atomic.AddUint64(&globalStatistics.HttpCode.Code403, 1)
|
||||||
ctx.SetResponseCode(http.StatusForbidden)
|
ctx.SetResponseCode(http.StatusForbidden)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -381,13 +488,16 @@ func handle(ctx httputils.RequestContext) {
|
||||||
pTime = cTime
|
pTime = cTime
|
||||||
cTime = time.Now()
|
cTime = time.Now()
|
||||||
if cacheHit {
|
if cacheHit {
|
||||||
|
atomic.AddUint64(&globalStatistics.SignatureCache.SignatureCacheHit, 1)
|
||||||
ctx.AddTimingInformational("vc", "Ed25519 Cache HIT")
|
ctx.AddTimingInformational("vc", "Ed25519 Cache HIT")
|
||||||
} else {
|
} else {
|
||||||
|
atomic.AddUint64(&globalStatistics.SignatureCache.SignatureCacheMiss, 1)
|
||||||
ctx.AddTimingInformational("vc", "Ed25519 Cache MISS")
|
ctx.AddTimingInformational("vc", "Ed25519 Cache MISS")
|
||||||
}
|
}
|
||||||
ctx.AddTiming("v", "Ed25519 Verify", cTime.Sub(pTime))
|
ctx.AddTiming("v", "Ed25519 Verify", cTime.Sub(pTime))
|
||||||
|
|
||||||
if !result {
|
if !result {
|
||||||
|
atomic.AddUint64(&globalStatistics.HttpCode.Code403, 1)
|
||||||
ctx.SetResponseCode(http.StatusForbidden)
|
ctx.SetResponseCode(http.StatusForbidden)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -400,8 +510,11 @@ func handle(ctx httputils.RequestContext) {
|
||||||
} else if ctx.IsOptions() {
|
} else if ctx.IsOptions() {
|
||||||
setOtherHeaders(ctx)
|
setOtherHeaders(ctx)
|
||||||
setCORSHeaders(ctx)
|
setCORSHeaders(ctx)
|
||||||
|
|
||||||
|
atomic.AddUint64(&globalStatistics.HttpCode.Code204, 1)
|
||||||
ctx.SetResponseCode(http.StatusNoContent)
|
ctx.SetResponseCode(http.StatusNoContent)
|
||||||
} else {
|
} else {
|
||||||
|
atomic.AddUint64(&globalStatistics.HttpCode.Code501, 1)
|
||||||
ctx.SetResponseCode(http.StatusNotImplemented)
|
ctx.SetResponseCode(http.StatusNotImplemented)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -518,6 +631,12 @@ func main() {
|
||||||
}
|
}
|
||||||
defer md5Statement.Close()
|
defer md5Statement.Close()
|
||||||
|
|
||||||
|
statsStatement, err = dbHandle.Prepare("SELECT COUNT(*) AS count, SUM(size) as size FROM entries;")
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
defer statsStatement.Close()
|
||||||
|
|
||||||
tlsConfiguration, err := tlsutils.NewTLSConfiguration(*certificatePath, *keypairPath, strings.ToLower(*sniAddressOption))
|
tlsConfiguration, err := tlsutils.NewTLSConfiguration(*certificatePath, *keypairPath, strings.ToLower(*sniAddressOption))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
|
|
Loading…
Reference in a new issue