diff --git a/METANOIA.go b/METANOIA.go index faf4ba2..c93f99f 100644 --- a/METANOIA.go +++ b/METANOIA.go @@ -13,8 +13,8 @@ import ( "git.gammaspectra.live/S.O.N.G/Hibiki/utilities/audio/format/flac" "git.gammaspectra.live/S.O.N.G/Hibiki/utilities/audio/format/mp3" "git.gammaspectra.live/S.O.N.G/Hibiki/utilities/audio/format/opus" - "git.gammaspectra.live/S.O.N.G/Hibiki/utilities/specializedstore" "git.gammaspectra.live/S.O.N.G/METANOIA/database" + "git.gammaspectra.live/S.O.N.G/METANOIA/store" "git.gammaspectra.live/S.O.N.G/MakyuuIchaival/httputils" "git.gammaspectra.live/S.O.N.G/MakyuuIchaival/tlsutils" "github.com/dhowden/tag" @@ -33,7 +33,7 @@ import ( "time" ) -var printDb *specializedstore.BPTreeFileStore +var printDb *store.AppendStore var printStrategy *panako.Strategy var db *database.Database var resourceCreationMutex sync.Mutex @@ -321,7 +321,7 @@ func main() { } defer db.Close() - printDb, err = specializedstore.NewBPTreeFileStore(*printDbOption, true) + printDb, err = store.NewAppendStore(*printDbOption, true, true) if err != nil { log.Fatal(err) } diff --git a/store/append.go b/store/append.go new file mode 100644 index 0000000..7a18f9e --- /dev/null +++ b/store/append.go @@ -0,0 +1,174 @@ +package store + +import ( + "bytes" + "encoding/binary" + "git.gammaspectra.live/S.O.N.G/Hibiki/strategy/panako" + "log" + "os" + "syscall" +) + +type AppendStore struct { + f *os.File + //writeMutex sync.RWMutex + useCompactHashes bool + usePackedValues bool + totalPrints int +} + +func NewAppendStore(pathName string, useCompactHashes, usePackedValues bool) (*AppendStore, error) { + f, err := os.OpenFile(pathName, os.O_WRONLY|os.O_CREATE|os.O_APPEND|syscall.O_DSYNC, 0666) + if err != nil { + return nil, err + } + + return &AppendStore{ + f: f, + useCompactHashes: useCompactHashes, + usePackedValues: usePackedValues, + }, nil +} + +func (s *AppendStore) GetPanakoRecords(resourceId panako.ResourceId) []panako.StoreRecord { + return nil +} + +func (s *AppendStore) GetPanakoMatch(record panako.StoreRecord, lookupRange int) chan *panako.MatchedRecord { + + channel := make(chan *panako.MatchedRecord) + + close(channel) + + return channel +} + +func (s *AppendStore) GetPanakoMatches(records []panako.StoreRecord, lookupRange int) chan *panako.MatchedRecord { + + channel := make(chan *panako.MatchedRecord) + + close(channel) + + return channel +} + +func (s *AppendStore) encodeRecord(record *panako.StoreRecord, buf *bytes.Buffer) { + + if s.useCompactHashes { + binary.Write(buf, binary.LittleEndian, uint32(record.Hash)) + } else { + binary.Write(buf, binary.LittleEndian, record.Hash) + } + if s.usePackedValues { + _, v := record.GetCompactPackedPrint() + binary.Write(buf, binary.LittleEndian, v) + } else { + binary.Write(buf, binary.LittleEndian, record.Time) + binary.Write(buf, binary.LittleEndian, record.Frequency) + } + +} + +type AppendStoreControlData struct { + resourceId panako.ResourceId + length uint32 +} + +func (s *AppendStore) encodeControl(d AppendStoreControlData, buf *bytes.Buffer) { + + //Reset record mode + if s.useCompactHashes { + binary.Write(buf, binary.LittleEndian, uint32(0)) + } else { + binary.Write(buf, binary.LittleEndian, uint64(0)) + } + if s.usePackedValues { + binary.Write(buf, binary.LittleEndian, uint32(0)) + } else { + binary.Write(buf, binary.LittleEndian, uint32(0)) + binary.Write(buf, binary.LittleEndian, uint32(0)) + } + + //Write value + if s.useCompactHashes { + binary.Write(buf, binary.LittleEndian, uint64(d.resourceId)) + binary.Write(buf, binary.LittleEndian, uint32(d.length)) + if s.usePackedValues { + binary.Write(buf, binary.LittleEndian, uint32(0)) + } + + } else { + binary.Write(buf, binary.LittleEndian, uint64(d.resourceId)) + binary.Write(buf, binary.LittleEndian, uint32(d.length)) + if !s.usePackedValues { + binary.Write(buf, binary.LittleEndian, uint32(0)) + } + } + +} + +func (s *AppendStore) GetEntrySize() int { + if s.useCompactHashes && s.usePackedValues { + return 4 + 4 + } else if !s.useCompactHashes && s.usePackedValues { + return 8 + 4 + } else if s.useCompactHashes && !s.usePackedValues { + return 4 + 8 + } else { + log.Panic() + return 0 + } +} + +func (s *AppendStore) StorePanakoPrint(record panako.StoreRecord) { + buf := new(bytes.Buffer) + s.encodeControl(AppendStoreControlData{ + resourceId: record.ResourceId, + length: 1, + }, buf) + s.encodeRecord(&record, buf) + //s.writeMutex.Lock() + //defer s.writeMutex.Unlock() + _, err := s.f.Write(buf.Bytes()) + + if err != nil { + log.Panic(err) + } +} + +func (s *AppendStore) StorePanakoPrints(records []panako.StoreRecord) { + m := make(map[panako.ResourceId][]*panako.StoreRecord) + for i := range records { + m[records[i].ResourceId] = append(m[records[i].ResourceId], &records[i]) + } + + s.totalPrints += len(records) + //s.writeMutex.Lock() + //defer s.writeMutex.Unlock() + + buf := new(bytes.Buffer) + for k, l := range m { + s.encodeControl(AppendStoreControlData{ + resourceId: k, + length: uint32(len(l)), + }, buf) + + for _, v := range l { + s.encodeRecord(v, buf) + } + } + + _, err := s.f.Write(buf.Bytes()) + + if err != nil { + log.Panic(err) + } +} + +func (s *AppendStore) Sync() error { + return s.f.Sync() +} + +func (s *AppendStore) Close() error { + return s.f.Close() +}