Add AppendStore

METANOIA.go

@@ -13,8 +13,8 @@ import (
 	"git.gammaspectra.live/S.O.N.G/Hibiki/utilities/audio/format/flac"
 	"git.gammaspectra.live/S.O.N.G/Hibiki/utilities/audio/format/mp3"
 	"git.gammaspectra.live/S.O.N.G/Hibiki/utilities/audio/format/opus"
-	"git.gammaspectra.live/S.O.N.G/Hibiki/utilities/specializedstore"
 	"git.gammaspectra.live/S.O.N.G/METANOIA/database"
+	"git.gammaspectra.live/S.O.N.G/METANOIA/store"
 	"git.gammaspectra.live/S.O.N.G/MakyuuIchaival/httputils"
 	"git.gammaspectra.live/S.O.N.G/MakyuuIchaival/tlsutils"
 	"github.com/dhowden/tag"
@@ -33,7 +33,7 @@ import (
 	"time"
 )
 
-var printDb *specializedstore.BPTreeFileStore
+var printDb *store.AppendStore
 var printStrategy *panako.Strategy
 var db *database.Database
 var resourceCreationMutex sync.Mutex
@@ -321,7 +321,7 @@ func main() {
 	}
 	defer db.Close()
 
-	printDb, err = specializedstore.NewBPTreeFileStore(*printDbOption, true)
+	printDb, err = store.NewAppendStore(*printDbOption, true, true)
 	if err != nil {
 		log.Fatal(err)
 	}

store/append.go

@@ -0,0 +1,174 @@
+package store
+
+import (
+	"bytes"
+	"encoding/binary"
+	"git.gammaspectra.live/S.O.N.G/Hibiki/strategy/panako"
+	"log"
+	"os"
+	"syscall"
+)
+
+type AppendStore struct {
+	f *os.File
+	//writeMutex       sync.RWMutex
+	useCompactHashes bool
+	usePackedValues  bool
+	totalPrints      int
+}
+
+func NewAppendStore(pathName string, useCompactHashes, usePackedValues bool) (*AppendStore, error) {
+	f, err := os.OpenFile(pathName, os.O_WRONLY|os.O_CREATE|os.O_APPEND|syscall.O_DSYNC, 0666)
+	if err != nil {
+		return nil, err
+	}
+
+	return &AppendStore{
+		f:                f,
+		useCompactHashes: useCompactHashes,
+		usePackedValues:  usePackedValues,
+	}, nil
+}
+
+func (s *AppendStore) GetPanakoRecords(resourceId panako.ResourceId) []panako.StoreRecord {
+	return nil
+}
+
+func (s *AppendStore) GetPanakoMatch(record panako.StoreRecord, lookupRange int) chan *panako.MatchedRecord {
+
+	channel := make(chan *panako.MatchedRecord)
+
+	close(channel)
+
+	return channel
+}
+
+func (s *AppendStore) GetPanakoMatches(records []panako.StoreRecord, lookupRange int) chan *panako.MatchedRecord {
+
+	channel := make(chan *panako.MatchedRecord)
+
+	close(channel)
+
+	return channel
+}
+
+func (s *AppendStore) encodeRecord(record *panako.StoreRecord, buf *bytes.Buffer) {
+
+	if s.useCompactHashes {
+		binary.Write(buf, binary.LittleEndian, uint32(record.Hash))
+	} else {
+		binary.Write(buf, binary.LittleEndian, record.Hash)
+	}
+	if s.usePackedValues {
+		_, v := record.GetCompactPackedPrint()
+		binary.Write(buf, binary.LittleEndian, v)
+	} else {
+		binary.Write(buf, binary.LittleEndian, record.Time)
+		binary.Write(buf, binary.LittleEndian, record.Frequency)
+	}
+
+}
+
+type AppendStoreControlData struct {
+	resourceId panako.ResourceId
+	length     uint32
+}
+
+func (s *AppendStore) encodeControl(d AppendStoreControlData, buf *bytes.Buffer) {
+
+	//Reset record mode
+	if s.useCompactHashes {
+		binary.Write(buf, binary.LittleEndian, uint32(0))
+	} else {
+		binary.Write(buf, binary.LittleEndian, uint64(0))
+	}
+	if s.usePackedValues {
+		binary.Write(buf, binary.LittleEndian, uint32(0))
+	} else {
+		binary.Write(buf, binary.LittleEndian, uint32(0))
+		binary.Write(buf, binary.LittleEndian, uint32(0))
+	}
+
+	//Write value
+	if s.useCompactHashes {
+		binary.Write(buf, binary.LittleEndian, uint64(d.resourceId))
+		binary.Write(buf, binary.LittleEndian, uint32(d.length))
+		if s.usePackedValues {
+			binary.Write(buf, binary.LittleEndian, uint32(0))
+		}
+
+	} else {
+		binary.Write(buf, binary.LittleEndian, uint64(d.resourceId))
+		binary.Write(buf, binary.LittleEndian, uint32(d.length))
+		if !s.usePackedValues {
+			binary.Write(buf, binary.LittleEndian, uint32(0))
+		}
+	}
+
+}
+
+func (s *AppendStore) GetEntrySize() int {
+	if s.useCompactHashes && s.usePackedValues {
+		return 4 + 4
+	} else if !s.useCompactHashes && s.usePackedValues {
+		return 8 + 4
+	} else if s.useCompactHashes && !s.usePackedValues {
+		return 4 + 8
+	} else {
+		log.Panic()
+		return 0
+	}
+}
+
+func (s *AppendStore) StorePanakoPrint(record panako.StoreRecord) {
+	buf := new(bytes.Buffer)
+	s.encodeControl(AppendStoreControlData{
+		resourceId: record.ResourceId,
+		length:     1,
+	}, buf)
+	s.encodeRecord(&record, buf)
+	//s.writeMutex.Lock()
+	//defer s.writeMutex.Unlock()
+	_, err := s.f.Write(buf.Bytes())
+
+	if err != nil {
+		log.Panic(err)
+	}
+}
+
+func (s *AppendStore) StorePanakoPrints(records []panako.StoreRecord) {
+	m := make(map[panako.ResourceId][]*panako.StoreRecord)
+	for i := range records {
+		m[records[i].ResourceId] = append(m[records[i].ResourceId], &records[i])
+	}
+
+	s.totalPrints += len(records)
+	//s.writeMutex.Lock()
+	//defer s.writeMutex.Unlock()
+
+	buf := new(bytes.Buffer)
+	for k, l := range m {
+		s.encodeControl(AppendStoreControlData{
+			resourceId: k,
+			length:     uint32(len(l)),
+		}, buf)
+
+		for _, v := range l {
+			s.encodeRecord(v, buf)
+		}
+	}
+
+	_, err := s.f.Write(buf.Bytes())
+
+	if err != nil {
+		log.Panic(err)
+	}
+}
+
+func (s *AppendStore) Sync() error {
+	return s.f.Sync()
+}
+
+func (s *AppendStore) Close() error {
+	return s.f.Close()
+}