commit f5df661a53a9e10091f8b3da9b45f1818a4de832 Author: WeebDataHoarder <57538841+WeebDataHoarder@users.noreply.github.com> Date: Sun Jan 16 02:06:19 2022 +0100 Initial commit. Working file serve, and signatures! diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..7865c2a --- /dev/null +++ b/.env.example @@ -0,0 +1,8 @@ +DATA_MOUNT_PATH=/mnt/storage +CERTIFICATE_PATH=/example/server.crt +KEYPAIR_PATH=/example/server.key + +TRUSTED_KEYS= + +# Leave empty to disable +SNI= \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1ebdff1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +/.idea +.env + +/*.key +/*.crt \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..334cccb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +FROM golang:1.17-bullseye + +COPY . /src + +WORKDIR /src +RUN go build -o orbeat . && mv orbeat /usr/bin && rm -rf /src +WORKDIR / + +ENTRYPOINT ["/usr/bin/orbeat"] \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5da1a02 --- /dev/null +++ b/LICENSE @@ -0,0 +1,9 @@ +Copyright (c) 2021 OrbitalBeat Contributors All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/OrbitalBeat.go b/OrbitalBeat.go new file mode 100644 index 0000000..7824952 --- /dev/null +++ b/OrbitalBeat.go @@ -0,0 +1,633 @@ +package main + +import ( + "bytes" + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "database/sql" + "encoding/base32" + "encoding/binary" + "encoding/pem" + "flag" + "fmt" + "github.com/ipfs/go-cid" + _ "github.com/lib/pq" + "github.com/multiformats/go-multihash" + "log" + "math/big" + "net/http" + "net/url" + "os" + "path" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + "unicode" +) + +type ContentCacheEntry struct { + Entry ContentEntry + File *os.File + RefCount int64 + AccessTime time.Time +} + +func (e *ContentCacheEntry) valid() bool { + return atomic.LoadInt64(&e.RefCount) > 0 +} + +func (e *ContentCacheEntry) borrow() *ContentCacheEntry { + if e.valid() { + if atomic.AddInt64(&e.RefCount, 1) > 1 { + return e + } else { //Has already been deallocated or is in the process + atomic.AddInt64(&e.RefCount, -1) + } + } + return nil +} + +func (e *ContentCacheEntry) release() { + objectCacheMutex.Lock() + if atomic.AddInt64(&e.RefCount, -1) == 0 { + defer e.File.Close() + delete(objectCache, e.Entry.Identifier) + } + objectCacheMutex.Unlock() +} + +var base32Encoding = base32.NewEncoding("qpzry9x8gf2tvdw0s3jn54khce6mua7l").WithPadding(base32.NoPadding) + +var dbHandle *sql.DB +var sniAddress string +var sha256Statement *sql.Stmt +var md5Statement *sql.Stmt +var fdlimit int + +var messageCacheLimit int + +var objectCacheMutex sync.RWMutex +var objectCache = make(map[cid.Cid]*ContentCacheEntry) + +var messageCacheMutex sync.RWMutex +var messageCache = make(map[string]*ContentMessage) + +var trustedPublicKeys []ed25519.PublicKey + +func isSNIAllowed(sni string) bool { + return len(sniAddress) == 0 || sni == sniAddress +} + +func getFirstValidContentEntry(entries *[]ContentEntry) *ContentEntry { + log.Printf("Entered getFirstValidContentEntry") + defer log.Printf("Exited getFirstValidContentEntry") + for _, entry := range *entries { + stat, err := os.Stat(entry.Path) + if err == nil /*&& uint64(stat.Size()) == entry.Size*/ { //TODO: update Size if not found + copiedEntry := entry + copiedEntry.Size = uint64(stat.Size()) + return &copiedEntry + } + } + + return nil +} + +func handleQueryRequest(w http.ResponseWriter, r *http.Request, identifier cid.Cid) { + log.Printf("Entered handleQueryRequest") + defer log.Printf("Exited handleQueryRequest") + + var cacheEntry = tryGetCacheEntryForIdentifier(identifier) + + if cacheEntry == nil { + result := getEntriesForCID(identifier) + + entry := getFirstValidContentEntry(&result) + + if entry != nil { + cacheEntry = getCacheEntryForContentEntry(entry) + } + } + + if cacheEntry == nil { + w.WriteHeader(http.StatusNotFound) + return + } + defer cacheEntry.release() + + w.Header().Set("Accept-Ranges", "bytes") + w.Header().Set("ETag", cacheEntry.Entry.Identifier.String()) + w.Header().Set("Content-Length", strconv.FormatUint(cacheEntry.Entry.Size, 10)) + w.Header().Set("Cache-Control", "public, max-age=2592000, immutable") + + filename := path.Base(cacheEntry.Entry.Path) + + //TODO: setting to hide filename + w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename*=utf-8''%s", url.PathEscape(filename))) + + http.ServeContent(w, r, filename, time.Date(1970, 0, 0, 0, 0, 0, 0, time.UTC), cacheEntry.File) +} + +func isASCII(s string) bool { + for _, c := range s { + if c > unicode.MaxASCII { + return false + } + } + return true +} + +func setOtherHeaders(w http.ResponseWriter) { + w.Header().Set("Server", "OrbitalBeat") + w.Header().Set("Vary", "Content-Encoding") //Firefox caps this to 86400, Chrome to 7200. Default is 5 seconds (!!!) +} +func setCORSHeaders(w http.ResponseWriter) { + w.Header().Set("Access-Control-Allow-Credentials", "true") + w.Header().Set("Access-Control-Max-Age", "7200") //Firefox caps this to 86400, Chrome to 7200. Default is 5 seconds (!!!) + w.Header().Set("Access-Control-Allow-Methods", "GET,HEAD,OPTIONS") + w.Header().Set("Access-Control-Allow-Headers", "DNT,Origin,Accept,Accept-Language") + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Expose-Headers", "*") +} + +func getCacheEntryForContentEntry(entry *ContentEntry) *ContentCacheEntry { + log.Printf("Entered getCacheEntryForContentEntry") + defer log.Printf("Exited getCacheEntryForContentEntry") + + cacheEntry := tryGetCacheEntryForIdentifier(entry.Identifier) + + if cacheEntry != nil { + return cacheEntry + } + objectCacheMutex.Lock() + defer objectCacheMutex.Unlock() + + if len(objectCache) >= fdlimit { + //Find oldest value, remove it + var item *ContentCacheEntry + for _, e := range objectCache { + if item == nil || e.AccessTime.Before(item.AccessTime) { + item = e + } + } + + if item != nil { + delete(objectCache, item.Entry.Identifier) + item.release() + } + } + + f, err := os.Open(entry.Path) + if err != nil { + return nil + } + + objectCache[entry.Identifier] = &ContentCacheEntry{ + Entry: *entry, + File: f, + RefCount: 1, + AccessTime: time.Now(), + } + + return objectCache[entry.Identifier].borrow() +} + +func tryGetCacheEntryForIdentifier(identifier cid.Cid) *ContentCacheEntry { + log.Printf("Entered tryGetCacheEntryForIdentifier") + defer log.Printf("Exited tryGetCacheEntryForIdentifier") + + objectCacheMutex.RLock() + defer objectCacheMutex.RUnlock() + + cacheEntry, ok := objectCache[identifier] + if ok { + cacheEntry := cacheEntry.borrow() + cacheEntry.AccessTime = time.Now().UTC() + return cacheEntry + } + + return nil +} + +func IsTrustedPublicKey(key ed25519.PublicKey) bool { + for _, k := range trustedPublicKeys { + if bytes.Compare(k, key) == 0 { + return true + } + } + return false +} + +func handle(w http.ResponseWriter, r *http.Request) { + if len(r.Host) > 0 && r.Host == r.TLS.ServerName { //Prevents rebinding / DNS stuff + w.WriteHeader(http.StatusNotFound) + return + } + + if r.Method == "GET" || r.Method == "HEAD" { + log.Printf("Serve %s", r.URL.Path) + setOtherHeaders(w) + setCORSHeaders(w) + pathElements := strings.Split(r.URL.Path, "/") + if len(pathElements) < 2 { + log.Printf("1") + w.WriteHeader(http.StatusBadRequest) + return + } + + messageBytes, err := base32Encoding.DecodeString(pathElements[1]) + if err != nil { + log.Printf("2") + w.WriteHeader(http.StatusBadRequest) + return + } + message := DecodeContentMessage(messageBytes) + if message == nil { + log.Printf("3") + w.WriteHeader(http.StatusBadRequest) + return + } + + if !IsTrustedPublicKey(message.PublicKey) { + log.Printf("4") + w.WriteHeader(http.StatusForbidden) + return + } + + if !message.verify() { + log.Printf("5") + w.WriteHeader(http.StatusForbidden) + return + } + + log.Printf("Valid %s %s", r.URL.Path, message.Identifier.String()) + + handleQueryRequest(w, r, message.Identifier) + } else if r.Method == "OPTIONS" { + setOtherHeaders(w) + setCORSHeaders(w) + w.WriteHeader(http.StatusNoContent) + } else { + w.WriteHeader(http.StatusNotImplemented) + } +} + +type ContentMessage struct { + Version uint64 + PublicKey ed25519.PublicKey + IssueTime int64 + Identifier cid.Cid + VerificationResult *bool + Signature []byte +} + +func (s *ContentMessage) sign(privateKey ed25519.PrivateKey) { + s.PublicKey = make([]byte, ed25519.PublicKeySize) + copy(s.PublicKey, privateKey[32:]) + s.Signature = ed25519.Sign(privateKey, s.encodeMessage()) + s.VerificationResult = nil +} + +func (s *ContentMessage) verify() bool { + currentTime := time.Now() + + notBefore := currentTime.Add(-time.Hour) //Only one hour before time + notAfter := currentTime.Add(time.Hour * 24) //Only 24 hours after time + issueTime := time.Unix(s.IssueTime, 0) + + if issueTime.Before(notBefore) { + return false + } + + if issueTime.After(notAfter) { + return false + } + + messageCacheMutex.RLock() + k := string(s.encode()) + cachedMessage, ok := messageCache[k] + messageCacheMutex.RUnlock() + if ok { + return *cachedMessage.VerificationResult + } + + messageCacheMutex.Lock() + defer messageCacheMutex.Unlock() + + if s.VerificationResult == nil { + makeBool := func(v bool) *bool { return &v } + s.VerificationResult = makeBool(ed25519.Verify(s.PublicKey, s.encodeMessage(), s.Signature)) + } + + if len(messageCache) >= messageCacheLimit { + //Find oldest value, remove it + var item *ContentMessage + for _, e := range messageCache { + if item == nil || e.IssueTime < item.IssueTime { + item = e + } + } + + if item != nil { + delete(messageCache, string(item.encode())) + } + } + + messageCache[k] = s + + return *s.VerificationResult +} + +func (s *ContentMessage) encodeMessage() []byte { + message := &bytes.Buffer{} + + buf := make([]byte, binary.MaxVarintLen64) + + n := binary.PutUvarint(buf, s.Version) //signature version + _, _ = message.Write(buf[:n]) + + if s.Version == 0 { + _, _ = message.Write(s.PublicKey) + n = binary.PutVarint(buf, s.IssueTime) //time + _, _ = message.Write(buf[:n]) + + _, _ = s.Identifier.WriteBytes(message) + + return message.Bytes() + } + + return nil +} + +func (s *ContentMessage) encode() []byte { + message := s.encodeMessage() + + if message == nil || len(s.Signature) != ed25519.SignatureSize { + return nil + } + + return append(message, s.Signature...) +} + +func (s ContentMessage) String() string { + return fmt.Sprintf("%d %x %d %s %x", s.Version, s.PublicKey, s.IssueTime, s.Identifier.String(), s.Signature) +} + +func DecodeContentMessage(signatureBytes []byte) *ContentMessage { + message := ContentMessage{} + + buffer := bytes.NewBuffer(signatureBytes) + + var err error + + message.Version, err = binary.ReadUvarint(buffer) + if err != nil { + log.Print(err) + return nil + } + + if message.Version == 0 { + message.PublicKey = make([]byte, ed25519.PublicKeySize) + _, err := buffer.Read(message.PublicKey) + if err != nil { + log.Print(message.String()) + log.Print(err) + return nil + } + message.IssueTime, err = binary.ReadVarint(buffer) + if err != nil { + log.Print(message.String()) + log.Print(err) + return nil + } + + _, message.Identifier, err = cid.CidFromReader(buffer) + if err != nil { + log.Print(message.String()) + log.Print(err) + return nil + } + + message.Signature = make([]byte, ed25519.SignatureSize) + _, err = buffer.Read(message.Signature) + if err != nil { + log.Print(message.String()) + log.Print(err) + return nil + } + + if buffer.Len() != 0 { //Unknown extra data + log.Print(message.String()) + log.Printf("%x", buffer.Bytes()) + return nil + } + + return &message + + } + + return nil +} + +type ContentEntry struct { + Identifier cid.Cid + Path string + Size uint64 +} + +func handleQuery(rows *sql.Rows, err error) []ContentEntry { + if err != nil { + log.Print(err) + return []ContentEntry{} + } + defer rows.Close() + + var result []ContentEntry + + for rows.Next() { + var entry ContentEntry + var sha256 multihash.Multihash + var size sql.NullInt64 + err := rows.Scan(&entry.Path, &size, &sha256) + if err != nil { + log.Print(err) + break + } + if size.Valid { + entry.Size = uint64(size.Int64) + } + mh, _ := multihash.Encode(sha256, multihash.SHA2_256) + entry.Identifier = cid.NewCidV1(cid.Raw, mh) + + result = append(result, entry) + } + + return result +} + +func getEntriesForCID(identifier cid.Cid) []ContentEntry { + log.Printf("Entered getEntriesForCID") + defer log.Printf("Exited getEntriesForCID") + + mh, _ := multihash.Decode(identifier.Hash()) + if mh.Code == multihash.SHA2_256 { + return handleQuery(sha256Statement.Query(mh.Digest)) + } else if mh.Code == multihash.MD5 { + return handleQuery(md5Statement.Query(mh.Digest)) + } + + return []ContentEntry{} +} + +func createSelfSignedCertificate() ([]byte, []byte) { + x509Template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{}, + NotBefore: time.Unix(0, 0).UTC(), + NotAfter: time.Date(time.Now().UTC().Year()+10, 0, 0, 0, 0, 0, 0, time.UTC), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + MaxPathLen: 1, + } + + privateBogusKey, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) + if err != nil { + log.Fatal(err) + } + + certBytes, err := x509.CreateCertificate(rand.Reader, &x509Template, &x509Template, privateBogusKey.Public(), privateBogusKey) + if err != nil { + log.Fatal(err) + } + + keyBytes, err := x509.MarshalPKCS8PrivateKey(privateBogusKey) + if err != nil { + log.Fatal(err) + } + + return pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }), + pem.EncodeToMemory(&pem.Block{ + Type: "PRIVATE KEY", + Bytes: keyBytes, + }) +} + +func main() { + certificatePath := flag.String("certificate", "ssl.crt", "Path to SSL certificate file.") + keypairPath := flag.String("keypair", "ssl.key", "Path to SSL key file.") + + pgConnStr := flag.String("connstr", "", "Postgres connection string for postgres database") + + listenAddress := flag.String("listen", ":7777", "Path to SSL key file.") + + trustedKeys := flag.String("trusted_keys", "", "Trusted list of public keys, comma separated.") + + sniAddressOption := flag.String("sni", "", "Define SNI address if desired. Empty will serve any requests regardless.") + + fdLimitOption := flag.Int("fdlimit", 128, "Maximum number of lingering cached open files.") + + signatureCacheLimitOption := flag.Int("siglimit", 4096, "Maximum number of lingering valid signature cache results.") + + flag.Parse() + + fdlimit = *fdLimitOption + messageCacheLimit = *signatureCacheLimitOption + + var err error + + for _, k := range strings.Split(*trustedKeys, ",") { + var publicKey ed25519.PublicKey + publicKey, err = base32Encoding.DecodeString(strings.Trim(k, " ")) + if err != nil { + log.Fatal(err) + } + + if len(publicKey) != ed25519.PublicKeySize { + continue + } + trustedPublicKeys = append(trustedPublicKeys, publicKey) + log.Printf("Added public key %x", publicKey) + } + + dbHandle, err = sql.Open("postgres", *pgConnStr) + if err != nil { + log.Fatal(err) + } + + defer dbHandle.Close() + + sha256Statement, err = dbHandle.Prepare("SELECT path, size, sha256 FROM entries WHERE sha256 = $1;") + if err != nil { + log.Fatal(err) + } + defer sha256Statement.Close() + + md5Statement, err = dbHandle.Prepare("SELECT path, size, sha256 FROM entries WHERE md5 = $1;") + if err != nil { + log.Fatal(err) + } + defer md5Statement.Close() + + //TODO: create postgres tables + + sniAddress = strings.ToLower(*sniAddressOption) + + bogusCertificatePEM, bogusKeyPairPEM := createSelfSignedCertificate() + + bogusCertificate, err := tls.X509KeyPair(bogusCertificatePEM, bogusKeyPairPEM) + if err != nil { + log.Fatal(err) + } + serverCertificate, err := tls.LoadX509KeyPair(*certificatePath, *keypairPath) + if err != nil { + log.Fatal(err) + } + + server := &http.Server{ + Addr: *listenAddress, + Handler: http.HandlerFunc(handle), + TLSConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + MaxVersion: 0, //max supported, currently TLS 1.3 + CurvePreferences: []tls.CurveID{ + tls.X25519, + tls.CurveP384, + //tls.CurveP256, + }, + CipherSuites: []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + }, + PreferServerCipherSuites: false, + SessionTicketsDisabled: false, + Renegotiation: tls.RenegotiateFreelyAsClient, + NextProtos: []string{ + "h2", + "http/1.1", + }, + GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { + if isSNIAllowed(info.ServerName) { + return &serverCertificate, nil + } + return &bogusCertificate, nil + }, + }, + } + + log.Printf("Serving on %s", *listenAddress) + + log.Fatal(server.ListenAndServeTLS(*certificatePath, *keypairPath)) +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..ec23b48 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# OrbitalBeat + +Content-addressable storage serving of blobs. + + +## Usage +Needs a Postgres database. + +`$ go run .` + +Build via `$ go build -o orbeat` \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..9db9644 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,77 @@ +version: "2.2" + +networks: + orbeat: + external: false + +volumes: + db: + external: false + keystore: + external: false + +services: + db: + image: postgres:14 + restart: always + read_only: true + security_opt: + - no-new-privileges:true + environment: + - POSTGRES_USER=orbeat + - POSTGRES_PASSWORD=orbeat + - POSTGRES_DB=orbeat + networks: + - orbeat + healthcheck: + test: [ "CMD-SHELL", "pg_isready --dbname \"postgres://orbeat:orbeat@db/orbeat\"" ] + interval: 10s + timeout: 5s + retries: 5 + volumes: + - db:/var/lib/postgresql/data:rw + tmpfs: + # For read-only filesystem, need to create a volume/tmpfs for PostgreSQL to run its much + # needed configuration. The read-only flag does not make volumes and tmpfs read-only. + - /tmp + - /run + - /run/postgresql + orbeat: + build: + context: ./ + dockerfile: Dockerfile + restart: always + read_only: true + security_opt: + - no-new-privileges:true + networks: + - orbeat + volumes: + - ${DATA_MOUNT_PATH}:${DATA_MOUNT_PATH}:ro + - ${CERTIFICATE_PATH}:${CERTIFICATE_PATH}:ro + - ${KEYPAIR_PATH}:${KEYPAIR_PATH}:ro + depends_on: + - db + command: >- + -certificate "${CERTIFICATE_PATH}" -keypair "${KEYPAIR_PATH}" + -connstr "user=orbeat password=orbeat dbname=orbeat sslmode=disable host=db" + -sni "${SNI}" + tmpfs: + - /tmp + ports: + - 7777:7777 + srg: + image: srg + build: https://git.gammaspectra.live/S.O.N.G/SynchRoGazer.git#master + restart: "no" + read_only: true + security_opt: + - no-new-privileges:true + networks: + - orbeat + volumes: + - ${DATA_MOUNT_PATH}:${DATA_MOUNT_PATH}:ro + depends_on: + - orbeat + tmpfs: + - /tmp \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..11c8af5 --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module git.gammaspectra.live/S.O.N.G/OrbitalBeat + +go 1.14 + +require ( + github.com/ipfs/go-cid v0.1.0 + github.com/lib/pq v1.10.4 + github.com/multiformats/go-multihash v0.0.15 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..018d04e --- /dev/null +++ b/go.sum @@ -0,0 +1,39 @@ +github.com/ipfs/go-cid v0.1.0 h1:YN33LQulcRHjfom/i25yoOZR4Telp1Hr/2RU3d0PnC0= +github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= +github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= +github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= +github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= +github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= +github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= +github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multihash v0.0.15 h1:hWOPdrNqDjwHDx82vsYGSDZNyktOJJ2dzZJzFkOV1jM= +github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= +github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY= +github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf h1:B2n+Zi5QeYRDAEodEu72OS36gmTWjgpXr2+cWcBW90o= +golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/schema.sql b/schema.sql new file mode 100644 index 0000000..f1f4538 --- /dev/null +++ b/schema.sql @@ -0,0 +1,9 @@ +CREATE TABLE entries ( + path TEXT PRIMARY KEY, + size BIGINT, + sha256 BYTEA NOT NULL, + md5 BYTEA NOT NULL +); + +CREATE INDEX ON entries(sha256); +CREATE INDEX ON entries(md5); \ No newline at end of file diff --git a/scripts/add.sh b/scripts/add.sh new file mode 100755 index 0000000..48a9399 --- /dev/null +++ b/scripts/add.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +source .env + +DIR="${DATA_MOUNT_PATH}" +if [[ "$1" != "" ]]; then + DIR="$1" +fi + +#find "${DIR}" -type f | docker-compose run srg -format postgres -pg_table entries -pg_mode insert -pg_connstr "user=orbeat password=orbeat dbname=orbeat sslmode=disable host=db" +#find "${DIR}" -type f | docker-compose run srg -format text -pg_binary true -pg_table entries -pg_mode insert -pg_connstr "user=orbeat password=orbeat dbname=orbeat sslmode=disable host=db" +#exit 0 +find "${DIR}" -type f | docker-compose run srg -format postgres -pg_table entries -pg_mode insert_binary -pg_connstr "user=orbeat password=orbeat dbname=orbeat sslmode=disable host=db" + diff --git a/scripts/psql.sh b/scripts/psql.sh new file mode 100755 index 0000000..505b311 --- /dev/null +++ b/scripts/psql.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +source .env + +docker-compose exec --env PGPASSWORD=orbeat db psql --host db --username orbeat "$@" orbeat +