2022-01-16 12:49:45 +00:00
package main
import (
"crypto/rand"
"encoding/hex"
2022-01-19 20:46:05 +00:00
"encoding/json"
2022-01-16 12:49:45 +00:00
"flag"
2022-01-19 20:46:05 +00:00
"git.gammaspectra.live/S.O.N.G/FinalCommander/content"
"git.gammaspectra.live/S.O.N.G/FinalCommander/utilities"
2022-01-18 18:54:39 +00:00
"git.gammaspectra.live/S.O.N.G/MakyuuIchaival"
"git.gammaspectra.live/S.O.N.G/MakyuuIchaival/httputils"
"git.gammaspectra.live/S.O.N.G/MakyuuIchaival/tlsutils"
"github.com/cloudflare/circl/sign/ed25519"
2022-01-16 12:49:45 +00:00
"github.com/mroth/weightedrand"
"github.com/multiformats/go-multihash"
"log"
"net/http"
"os"
2022-03-09 19:37:33 +00:00
"path"
2022-01-16 12:49:45 +00:00
"strings"
"time"
)
2022-06-01 22:13:13 +00:00
var publicKey ed25519 . PublicKey
2022-01-16 12:49:45 +00:00
2022-01-18 18:54:39 +00:00
var debugOutput = false
2022-01-19 20:46:05 +00:00
var contentServers [ ] * content . Server
2022-01-16 12:49:45 +00:00
2022-01-19 20:46:05 +00:00
var db * content . Database
2022-01-16 12:49:45 +00:00
2022-01-19 22:42:37 +00:00
var redirectListenAddress string
2022-01-19 20:46:05 +00:00
func selectNextContentServer ( skip [ ] int ) * content . Server {
2022-01-16 12:49:45 +00:00
inSkip := func ( i int ) bool {
for _ , c := range skip {
if i == c {
return true
}
}
return false
}
var choosingList [ ] weightedrand . Choice
for _ , c := range contentServers {
2022-01-19 20:46:05 +00:00
if ! inSkip ( c . Index ) && c . GetCheckResult ( ) {
2022-01-16 12:49:45 +00:00
choosingList = append ( choosingList , weightedrand . NewChoice ( c , c . Weight ) )
}
}
chooser , err := weightedrand . NewChooser ( choosingList ... )
if err != nil {
return nil
}
2022-01-19 20:46:05 +00:00
return chooser . Pick ( ) . ( * content . Server )
2022-01-16 12:49:45 +00:00
}
2022-06-06 22:20:18 +00:00
func setOtherHeaders ( ctx httputils . RequestContext ) {
ctx . SetResponseHeader ( "Cache-Control" , "no-store" )
2022-01-18 18:54:39 +00:00
ctx . SetResponseHeader ( "Server" , "FinalCommander" )
ctx . SetResponseHeader ( "Vary" , "Content-Encoding" )
ctx . SetResponseHeader ( "X-Content-Type-Options" , "nosniff" )
ctx . SetResponseHeader ( "X-Robots-Tags" , "noindex, nofollow, notranslate" )
ctx . SetResponseHeader ( "Referrer-Policy" , "origin" )
2022-01-16 12:49:45 +00:00
}
2022-06-06 22:20:18 +00:00
func setCORSHeaders ( ctx httputils . RequestContext ) {
2022-01-18 18:54:39 +00:00
ctx . SetResponseHeader ( "Access-Control-Allow-Credentials" , "true" )
ctx . SetResponseHeader ( "Access-Control-Max-Age" , "7200" ) //Firefox caps this to 86400, Chrome to 7200. Default is 5 seconds (!!!)
ctx . SetResponseHeader ( "Access-Control-Allow-Methods" , "GET,HEAD,OPTIONS" )
ctx . SetResponseHeader ( "Access-Control-Allow-Headers" , "DNT,ETag,Origin,Accept,Accept-Language,X-Requested-With,Range" )
ctx . SetResponseHeader ( "Access-Control-Allow-Origin" , "*" )
ctx . SetResponseHeader ( "Access-Control-Expose-Headers" , "*" )
2022-01-16 12:49:45 +00:00
//CORP, COEP, COOP
2022-01-18 18:54:39 +00:00
ctx . SetResponseHeader ( "Cross-Origin-Embedder-Policy" , "require-corp" )
ctx . SetResponseHeader ( "Cross-Origin-Resource-Policy" , "cross-origin" )
ctx . SetResponseHeader ( "Cross-Origin-Opener-Policy" , "unsafe-none" )
2022-01-16 12:49:45 +00:00
}
2022-06-06 22:20:18 +00:00
func handleHexHash ( pathElements [ ] string , ctx httputils . RequestContext , host string ) {
2022-02-08 17:27:27 +00:00
if len ( pathElements ) < 3 {
ctx . SetResponseCode ( http . StatusBadRequest )
2022-01-16 12:49:45 +00:00
return
}
2022-02-08 17:27:27 +00:00
hashType := strings . ToLower ( pathElements [ 1 ] )
2022-01-19 22:42:37 +00:00
2022-03-09 19:37:33 +00:00
hashString := pathElements [ 2 ]
if len ( path . Ext ( hashString ) ) > 0 {
hashString = strings . TrimSuffix ( hashString , path . Ext ( hashString ) )
}
hash , err := hex . DecodeString ( hashString )
2022-02-08 17:27:27 +00:00
if err != nil {
ctx . SetResponseCode ( http . StatusBadRequest )
return
2022-01-19 22:42:37 +00:00
}
2022-02-08 17:27:27 +00:00
var mh multihash . Multihash
if hashType == "sha256" && len ( hash ) == 32 {
mh , _ = multihash . Encode ( hash , multihash . SHA2_256 )
} else if hashType == "md5" && len ( hash ) == 16 {
mh , _ = multihash . Encode ( hash , multihash . MD5 )
} else {
ctx . SetResponseCode ( http . StatusNotImplemented )
return
}
key := content . NewHashIdentifierFromMultihash ( mh )
var skip [ ] int
entry := getContentEntry ( key )
if len ( pathElements ) > 3 {
if pathElements [ 3 ] == "information" {
ctx . SetResponseCode ( http . StatusOK )
ctx . SetResponseHeader ( "Content-Type" , "application/json" )
if entry != nil {
b , _ := json . Marshal ( struct {
2022-06-01 22:13:13 +00:00
Known bool ` json:"known" `
CID string ` json:"cid" `
AccessTime int64 ` json:"accessTime" `
CheckTime int64 ` json:"checkTime" `
2022-06-01 22:18:35 +00:00
Invalid [ ] int ` json:"invalid,omitempty" `
2022-02-08 17:27:27 +00:00
} {
2022-06-01 22:13:13 +00:00
Known : true ,
CID : entry . CID ( ) . String ( ) ,
AccessTime : entry . AccessTime ,
CheckTime : entry . CheckTime ,
Invalid : entry . InvalidList ,
2022-02-08 17:27:27 +00:00
} )
ctx . ServeBytes ( b )
} else {
b , _ := json . Marshal ( struct {
Known bool ` json:"known" `
CID string ` json:"cid" `
} {
Known : false ,
CID : key . CID ( ) . String ( ) ,
} )
ctx . ServeBytes ( b )
}
return
} else if pathElements [ 3 ] == "drop" { //Drop key from cache
ctx . SetResponseCode ( http . StatusOK )
ctx . SetResponseHeader ( "Content-Type" , "text/plain" )
_ = db . RemoveEntry ( key )
if entry != nil && ! entry . Key . Equals ( key ) {
_ = db . RemoveEntry ( & entry . Key )
}
ctx . ServeBytes ( [ ] byte { } )
2022-01-16 12:49:45 +00:00
return
}
2022-02-08 17:27:27 +00:00
//TODO: update entries with these instant returns
data , err := MakyuuIchaival . Bech32Encoding . DecodeString ( pathElements [ 3 ] )
2022-01-16 12:49:45 +00:00
if err != nil {
2022-01-18 18:54:39 +00:00
ctx . SetResponseCode ( http . StatusBadRequest )
2022-01-16 12:49:45 +00:00
return
}
2022-02-08 17:27:27 +00:00
skip = utilities . DecodeIntegerList ( data )
}
if entry != nil {
oldSkip := skip
skip = entry . InvalidList
for _ , ci := range oldSkip {
if ! entry . InInvalidList ( ci ) {
skip = append ( skip , ci )
}
}
contentServer := selectNextContentServer ( skip )
if contentServer == nil {
ctx . SetResponseCode ( http . StatusNotFound )
return
}
2022-01-16 12:49:45 +00:00
2022-06-06 18:14:34 +00:00
ctx . DoRedirect ( contentServer . GetContentURL ( entry , skip ) + host , http . StatusFound )
2022-02-08 17:27:27 +00:00
} else {
contentServer := selectNextContentServer ( skip )
if contentServer == nil {
ctx . SetResponseCode ( http . StatusNotFound )
2022-01-16 12:49:45 +00:00
return
}
2022-01-19 20:46:05 +00:00
2022-02-08 17:27:27 +00:00
//TODO: only trigger this when we don't get a 404
go func ( ) {
var newInvalidList [ ] int
var e * content . Entry
2022-01-19 20:46:05 +00:00
2022-02-08 17:27:27 +00:00
for _ , c := range contentServers {
if ! c . GetCheckResult ( ) {
continue
2022-01-19 20:46:05 +00:00
}
2022-02-08 17:27:27 +00:00
2022-06-06 18:14:34 +00:00
result , err := c . CheckEntryKey ( key )
2022-02-08 17:27:27 +00:00
if result != nil {
if e == nil {
e = & content . Entry {
Key : * result ,
Version : 0 ,
AccessTime : time . Now ( ) . UTC ( ) . Unix ( ) ,
CheckTime : time . Now ( ) . UTC ( ) . Unix ( ) + 3600 * 24 * 3 , // Check sooner after addition, not all servers might have it yet
}
}
} else if err == nil {
newInvalidList = append ( newInvalidList , c . Index )
2022-01-19 23:37:15 +00:00
}
2022-01-19 20:46:05 +00:00
}
2022-02-08 17:27:27 +00:00
if e != nil {
if ! key . IsKey ( ) { //Check for entry already existing
entry := getContentEntry ( & e . Key )
2022-01-16 12:49:45 +00:00
2022-02-08 17:27:27 +00:00
if entry != nil {
e = entry
}
//Add alias mapping
_ = db . SetAlias ( & content . Alias {
Key : * key ,
Identifier : e . Key ,
} )
2022-01-19 20:46:05 +00:00
}
2022-01-16 12:49:45 +00:00
2022-02-08 17:27:27 +00:00
e . AccessTime = time . Now ( ) . UTC ( ) . Unix ( )
e . InvalidList = newInvalidList
_ = db . SetEntry ( e )
2022-01-19 20:46:05 +00:00
}
2022-02-08 17:27:27 +00:00
} ( )
2022-01-16 12:49:45 +00:00
2022-06-06 18:14:34 +00:00
ctx . DoRedirect ( contentServer . GetHashURL ( mh , skip ) + host , http . StatusFound )
2022-02-08 17:27:27 +00:00
}
}
2022-01-19 20:46:05 +00:00
2022-06-06 22:20:18 +00:00
func handle ( ctx httputils . RequestContext ) {
2022-06-06 23:13:56 +00:00
if len ( ctx . GetHost ( ) ) > 0 && len ( ctx . GetTLSServerName ( ) ) > 0 && strings . Split ( ctx . GetHost ( ) , ":" ) [ 0 ] != ctx . GetTLSServerName ( ) { //Prevents rebinding / DNS stuff
2022-02-08 17:27:27 +00:00
ctx . SetResponseCode ( http . StatusNotFound )
return
}
2022-01-19 20:46:05 +00:00
2022-06-06 23:11:52 +00:00
host := ctx . GetHost ( )
2022-02-08 17:27:27 +00:00
if host == "" && ctx . GetTLSServerName ( ) != "" {
host = ctx . GetTLSServerName ( ) + redirectListenAddress
}
2022-01-19 20:46:05 +00:00
2022-02-08 17:27:27 +00:00
if host != "" {
host = "/" + host
}
2022-01-19 20:46:05 +00:00
2022-02-08 17:27:27 +00:00
if ctx . IsGet ( ) || ctx . IsHead ( ) {
if debugOutput {
log . Printf ( "Serve %s" , ctx . GetPath ( ) )
2022-01-19 20:46:05 +00:00
}
2022-02-08 17:27:27 +00:00
setOtherHeaders ( ctx )
setCORSHeaders ( ctx )
2022-06-01 22:13:13 +00:00
if ctx . GetPath ( ) == "/status" {
ctx . SetResponseCode ( http . StatusOK )
ctx . SetResponseHeader ( "Content-Type" , "application/json" )
b , _ := json . MarshalIndent ( struct {
PublicKey string ` json:"public_key,omitempty" `
Servers [ ] * content . Server ` json:"servers" json:"servers,omitempty" `
} {
PublicKey : MakyuuIchaival . Bech32Encoding . EncodeToString ( publicKey ) ,
Servers : contentServers ,
} , "" , " " )
ctx . ServeBytes ( b )
return
}
2022-02-08 17:27:27 +00:00
pathElements := strings . Split ( ctx . GetPath ( ) , "/" )
//TODO: handle ni RFC 6920
handleHexHash ( pathElements , ctx , host )
2022-01-18 18:54:39 +00:00
} else if ctx . IsOptions ( ) {
setOtherHeaders ( ctx )
setCORSHeaders ( ctx )
ctx . SetResponseCode ( http . StatusNoContent )
2022-01-16 12:49:45 +00:00
} else {
2022-01-18 18:54:39 +00:00
ctx . SetResponseCode ( http . StatusNotImplemented )
2022-01-16 12:49:45 +00:00
}
}
2022-01-19 20:46:05 +00:00
func getContentEntry ( key * content . HashIdentifier ) * content . Entry {
entry := db . GetEntry ( * key )
if entry != nil {
entry . UpdateAccessTime ( db )
if entry . NeedsCheck ( ) {
go func ( ) {
b , _ := entry . Encode ( ) //Encode/decode to copy object
e := content . DecodeEntry ( entry . Key , b )
2022-01-19 23:28:45 +00:00
e . CheckTime = time . Now ( ) . UTC ( ) . Unix ( ) + 3600 * 24 * 30 //force a check every 30 days as they get fetched
2022-01-19 20:46:05 +00:00
var newInvalidList [ ] int
for _ , c := range contentServers {
if ! c . GetCheckResult ( ) {
if e . InInvalidList ( c . Index ) { //Keep old result if down
newInvalidList = append ( newInvalidList , c . Index )
}
continue
}
2022-01-20 09:38:20 +00:00
2022-06-06 18:14:34 +00:00
h , err := c . CheckEntryKey ( & e . Key )
2022-01-20 09:38:20 +00:00
if h == nil && err == nil {
2022-01-19 20:46:05 +00:00
newInvalidList = append ( newInvalidList , c . Index )
}
}
e . InvalidList = newInvalidList
_ = db . SetEntry ( e )
} ( )
}
}
return entry
}
2022-01-16 12:49:45 +00:00
func checkContentServers ( ) {
for _ , c := range contentServers {
2022-01-19 20:46:05 +00:00
c . Check ( )
2022-01-16 12:49:45 +00:00
}
}
func main ( ) {
2022-06-06 18:14:34 +00:00
debugOption := flag . Bool ( "debug" , false , "Enable debug output." )
2022-01-16 14:25:17 +00:00
certificatePath := flag . String ( "certificate" , "" , "Path to SSL certificate file." )
keypairPath := flag . String ( "keypair" , "" , "Path to SSL key file." )
2022-01-19 20:46:05 +00:00
databasePath := flag . String ( "dbpath" , "database" , "Path to key/value database." )
2022-01-16 12:49:45 +00:00
2022-06-06 18:14:34 +00:00
listenAddress := flag . String ( "listen" , ":7777" , "address/port to listen on." )
2022-01-16 12:49:45 +00:00
2022-06-06 18:14:34 +00:00
weightedServerList := flag . String ( "servers" , "" , "Weighted list of servers to use. All will use HTTPs. Allowed protocols: orbt. Format [protocol=]address:PORT/WEIGHT,[...]" )
2022-01-16 12:49:45 +00:00
sniAddressOption := flag . String ( "sni" , "" , "Define SNI address if desired. Empty will serve any requests regardless." )
flag . Parse ( )
var err error
2022-06-06 18:14:34 +00:00
debugOutput = * debugOption
2022-01-16 12:49:45 +00:00
privateKeyEnv := os . Getenv ( "PRIVATE_KEY" )
2022-01-19 20:46:05 +00:00
if privateKeyEnv != "" {
log . Print ( "PRIVATE_KEY is deprecated. Use PRIVATE_SEED instead with seed" )
privateKey , _ := MakyuuIchaival . Bech32Encoding . DecodeString ( privateKeyEnv )
log . Printf ( "Private Ed25519 seed (keep safe!): %s" , MakyuuIchaival . Bech32Encoding . EncodeToString ( ed25519 . PrivateKey ( privateKey ) . Seed ( ) ) )
return
}
privateSeedEnv := os . Getenv ( "PRIVATE_SEED" )
if privateSeedEnv == "" {
log . Print ( "No PRIVATE_SEED environment variable specified, generating new identity" )
2022-01-16 12:49:45 +00:00
publicKey , privateKey , _ := ed25519 . GenerateKey ( rand . Reader )
2022-01-18 18:54:39 +00:00
log . Printf ( "Public Ed25519 key (share this): %s" , MakyuuIchaival . Bech32Encoding . EncodeToString ( publicKey ) )
2022-01-19 20:46:05 +00:00
log . Printf ( "Private Ed25519 seed (keep safe!): %s" , MakyuuIchaival . Bech32Encoding . EncodeToString ( privateKey . Seed ( ) ) )
2022-01-16 12:49:45 +00:00
return
}
2022-01-19 20:46:05 +00:00
privateSeed , err := MakyuuIchaival . Bech32Encoding . DecodeString ( privateSeedEnv )
2022-01-16 12:49:45 +00:00
if err != nil {
log . Fatal ( err )
}
2022-01-19 20:46:05 +00:00
if len ( privateSeed ) != ed25519 . SeedSize {
log . Fatal ( "Wrong Private key length" )
}
2022-06-06 18:14:34 +00:00
privateKey := ed25519 . NewKeyFromSeed ( privateSeed )
2022-06-01 22:13:13 +00:00
publicKey = make ( [ ] byte , ed25519 . PublicKeySize )
2022-01-19 20:46:05 +00:00
copy ( publicKey , privateKey [ ed25519 . PublicKeySize : ] )
2022-01-18 18:54:39 +00:00
log . Printf ( "Loaded Private Ed25519 key, Public %s" , MakyuuIchaival . Bech32Encoding . EncodeToString ( publicKey ) )
2022-01-16 12:49:45 +00:00
2022-01-19 20:46:05 +00:00
db , err = content . OpenDatabase ( * databasePath )
if err != nil {
log . Fatal ( err )
2022-01-16 12:49:45 +00:00
}
2022-01-19 20:46:05 +00:00
defer db . Close ( )
2022-01-16 12:49:45 +00:00
for i , s := range strings . Split ( * weightedServerList , "," ) {
2022-06-06 18:14:34 +00:00
cs , err := content . NewContentServerFromArgument ( s , i , privateKey )
2022-01-16 12:49:45 +00:00
if err != nil {
log . Fatal ( err )
}
2022-01-19 20:46:05 +00:00
contentServers = append ( contentServers , cs )
2022-01-16 12:49:45 +00:00
}
checkContentServers ( )
2022-01-17 09:48:32 +00:00
go func ( ) {
ticker := time . NewTicker ( 1 * time . Minute )
2022-01-20 09:38:20 +00:00
for range ticker . C {
2022-01-17 09:48:32 +00:00
checkContentServers ( )
}
} ( )
2022-01-18 18:54:39 +00:00
tlsConfiguration , err := tlsutils . NewTLSConfiguration ( * certificatePath , * keypairPath , * sniAddressOption )
2022-01-16 12:49:45 +00:00
if err != nil {
log . Fatal ( err )
}
2022-01-16 14:25:17 +00:00
2022-01-18 18:54:39 +00:00
server := & httputils . Server {
ListenAddress : * listenAddress ,
TLSConfig : tlsConfiguration ,
EnableHTTP2 : true ,
EnableHTTP3 : true ,
Handler : handle ,
Debug : debugOutput ,
2022-01-16 12:49:45 +00:00
}
2022-01-19 22:42:37 +00:00
redirectListenAddress = * listenAddress
2022-01-18 18:54:39 +00:00
server . Serve ( )
2022-01-16 12:49:45 +00:00
}