Add badger converter
This commit is contained in:
parent
58334f237e
commit
291b0ca96c
|
@ -4,6 +4,7 @@ import (
|
|||
"bytes"
|
||||
"encoding/binary"
|
||||
"git.gammaspectra.live/S.O.N.G/Hibiki/strategy/panako"
|
||||
"git.gammaspectra.live/S.O.N.G/Hibiki/utilities/specializedstore"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
|
@ -258,6 +259,39 @@ func (c *AppendConverter) SortHashes(s *AppendStore, workers int) *TaskState {
|
|||
return ts
|
||||
}
|
||||
|
||||
func (c *AppendConverter) ConvertToBadger(s *specializedstore.BadgerStore, batchSize int) *TaskState {
|
||||
nentries := c.fileSize / c.recordSize
|
||||
|
||||
nrecords := int64(len(c.resourceIndex))
|
||||
splitSize := nrecords / int64(batchSize)
|
||||
if splitSize < 1 {
|
||||
splitSize = 1
|
||||
}
|
||||
|
||||
ts := &TaskState{
|
||||
total: nentries,
|
||||
}
|
||||
|
||||
ts.wg.Add(1)
|
||||
go func() {
|
||||
defer ts.wg.Done()
|
||||
for i := int64(0); i < nrecords; i += splitSize {
|
||||
l := i + splitSize
|
||||
if l > nrecords {
|
||||
l = nrecords
|
||||
}
|
||||
|
||||
for _, control := range c.resourceIndex[i:l] {
|
||||
records := c.ReadRecords(control.resourceId, control.recordsStartIndex, int64(control.length))
|
||||
s.StorePanakoPrints(records)
|
||||
atomic.AddInt64(&ts.processed, int64(len(records))+c.controlSize)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return ts
|
||||
}
|
||||
|
||||
func (c *AppendConverter) ReadRecords(id panako.ResourceId, recordIndex, n int64) (results []panako.StoreRecord) {
|
||||
c.resourceIndex = c.resourceIndex[:0]
|
||||
|
||||
|
|
|
@ -2,6 +2,7 @@ package main
|
|||
|
||||
import (
|
||||
"flag"
|
||||
"git.gammaspectra.live/S.O.N.G/Hibiki/utilities/specializedstore"
|
||||
"git.gammaspectra.live/S.O.N.G/METANOIA/store"
|
||||
"log"
|
||||
"runtime"
|
||||
|
@ -10,7 +11,7 @@ import (
|
|||
|
||||
func main() {
|
||||
command := flag.String("cmd", "sort", "Commands: sort")
|
||||
nworkers := flag.Int("workers", runtime.NumCPU(), "NUmber of workers for bulk tasks")
|
||||
nworkers := flag.Int("workers", runtime.NumCPU(), "Number of splits for bulk tasks")
|
||||
sourcePath := flag.String("src", "", "Source database append log path")
|
||||
destinationPath := flag.String("dst", "", "Destination database append log path")
|
||||
|
||||
|
@ -41,6 +42,32 @@ func main() {
|
|||
time.Sleep(time.Second * 5)
|
||||
}
|
||||
|
||||
taskState.Wait()
|
||||
} else if *command == "badger" {
|
||||
source, err := store.NewAppendConverter(*sourcePath, true, true)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
|
||||
destination, err := specializedstore.NewBadgerStore(*destinationPath, true, false, 500000)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
|
||||
taskState := source.ConvertToBadger(destination, *nworkers)
|
||||
|
||||
for {
|
||||
p, t := taskState.Progress()
|
||||
|
||||
log.Printf("%.03f%% %d/%d", float64(p*100)/float64(t), p, t)
|
||||
|
||||
if p >= t {
|
||||
break
|
||||
}
|
||||
|
||||
time.Sleep(time.Second * 5)
|
||||
}
|
||||
|
||||
taskState.Wait()
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue