add BPTree converter

This commit is contained in:
DataHoarder 2022-02-04 12:15:09 +01:00
parent 01e6f0e8ce
commit c925b3ae68
2 changed files with 48 additions and 0 deletions

View file

@ -295,6 +295,26 @@ func (c *AppendConverter) ConvertToBadger(s *specializedstore.BadgerStore, batch
return ts
}
func (c *AppendConverter) ConvertToBPTree(s *specializedstore.BPTreeFileStore) *TaskState {
nentries := c.fileSize / c.recordSize
ts := &TaskState{
total: nentries,
}
ts.wg.Add(1)
go func() {
defer ts.wg.Done()
for _, control := range c.resourceIndex {
records := c.ReadRecords(control.resourceId, control.recordsStartIndex, int64(control.length))
atomic.AddInt64(&ts.processed, int64(len(records))+c.controlSize)
s.StorePanakoPrints(records)
}
}()
return ts
}
func (c *AppendConverter) ReadRecords(id panako.ResourceId, recordIndex, n int64) (results []panako.StoreRecord) {
buf := make([]byte, c.recordSize*n)
_, err := c.f.ReadAt(buf, recordIndex*c.recordSize)

View file

@ -72,6 +72,34 @@ func main() {
time.Sleep(time.Second * 5)
}
taskState.Wait()
} else if *command == "bptree" {
source, err := store.NewAppendConverter(*sourcePath, true, true)
if err != nil {
log.Panic(err)
}
defer source.Close()
destination, err := specializedstore.NewBPTreeFileStore(*destinationPath, true)
if err != nil {
log.Panic(err)
}
defer destination.Close()
taskState := source.ConvertToBPTree(destination)
for {
p, t := taskState.Progress()
log.Printf("%.03f%% %d/%d", float64(p*100)/float64(t), p, t)
if p >= t {
break
}
time.Sleep(time.Second * 5)
}
taskState.Wait()
}
}