408 lines
12 KiB
Go
408 lines
12 KiB
Go
|
package metadata
|
|||
|
|
|||
|
import (
|
|||
|
"encoding/binary"
|
|||
|
"facette.io/natsort"
|
|||
|
"fmt"
|
|||
|
"git.gammaspectra.live/S.O.N.G/Hibiki/panako"
|
|||
|
"git.gammaspectra.live/S.O.N.G/Hibiki/utilities/audio"
|
|||
|
"git.gammaspectra.live/S.O.N.G/Hibiki/utilities/audio/format/flac"
|
|||
|
"git.gammaspectra.live/S.O.N.G/Hibiki/utilities/audio/format/mp3"
|
|||
|
"git.gammaspectra.live/S.O.N.G/Hibiki/utilities/audio/format/opus"
|
|||
|
"git.gammaspectra.live/S.O.N.G/Hibiki/utilities/specializedstore"
|
|||
|
"git.gammaspectra.live/S.O.N.G/METANOIA/utilities"
|
|||
|
"github.com/dhowden/tag"
|
|||
|
"golang.org/x/text/unicode/norm"
|
|||
|
"io/ioutil"
|
|||
|
"log"
|
|||
|
"os"
|
|||
|
"path"
|
|||
|
"sort"
|
|||
|
"strings"
|
|||
|
"sync"
|
|||
|
"sync/atomic"
|
|||
|
"time"
|
|||
|
"unicode"
|
|||
|
)
|
|||
|
|
|||
|
type fileEntryList []fileEntry
|
|||
|
|
|||
|
var flacFormat = flac.NewFormat()
|
|||
|
var mp3Format = mp3.NewFormat()
|
|||
|
var opusFormat = opus.NewFormat()
|
|||
|
|
|||
|
const separatorTrimSet = ",.-_()[]{}"
|
|||
|
|
|||
|
func isSeparator(b byte) bool {
|
|||
|
if b == ' ' {
|
|||
|
return true
|
|||
|
}
|
|||
|
for i := 0; i < len(separatorTrimSet); i++ {
|
|||
|
if separatorTrimSet[i] == b {
|
|||
|
return true
|
|||
|
}
|
|||
|
}
|
|||
|
|
|||
|
return false
|
|||
|
}
|
|||
|
|
|||
|
type analyzeEntry struct {
|
|||
|
fileEntry
|
|||
|
fileHandle *os.File
|
|||
|
audioMetadata struct {
|
|||
|
sampleRate int
|
|||
|
channels int
|
|||
|
samples int
|
|||
|
}
|
|||
|
fileMetadata tag.Metadata
|
|||
|
panakoFingerprints []*panako.Fingerprint
|
|||
|
hasherCrc32 *Hasher
|
|||
|
hasherCueToolsCrc32 *Hasher
|
|||
|
hasherAccurateRipV1 *Hasher
|
|||
|
hasherAccurateRipV2 *Hasher
|
|||
|
}
|
|||
|
|
|||
|
func (l fileEntryList) analyze(directory string, panakoInstance *panako.Instance) (entries []*analyzeEntry, fullCRC32 uint32, fullCTDBCRC32 uint32) {
|
|||
|
var waitGroups []*sync.WaitGroup
|
|||
|
|
|||
|
printStrategy := panakoInstance.GetStrategy(specializedstore.NewMemoryStore(), audio.RESAMPLER_QUALITY_LINEAR)
|
|||
|
|
|||
|
var joinedCTDBChannels []HasherChannel
|
|||
|
var joinedChannels []HasherChannel
|
|||
|
|
|||
|
var preLastTotalSamplesWaitGroup sync.WaitGroup
|
|||
|
preLastTotalSamples := uint32(0)
|
|||
|
|
|||
|
for trackIndex, e := range l {
|
|||
|
f, err := os.Open(path.Join(directory, e.Name))
|
|||
|
if err != nil {
|
|||
|
//TODO
|
|||
|
log.Print(err)
|
|||
|
continue
|
|||
|
}
|
|||
|
|
|||
|
var stream *audio.Stream
|
|||
|
var analyzer HasherChannel
|
|||
|
|
|||
|
meta, err := tag.ReadFrom(f)
|
|||
|
if err != nil {
|
|||
|
log.Print(err)
|
|||
|
err = nil
|
|||
|
}
|
|||
|
f.Seek(0, 0)
|
|||
|
|
|||
|
switch utilities.GetMimeTypeFromExtension(path.Ext(e.Name)) {
|
|||
|
case "audio/flac":
|
|||
|
stream, analyzer, err = flacFormat.OpenAnalyzer(f, panakoInstance.BlockSize)
|
|||
|
|
|||
|
case "audio/mpeg;codecs=mp3":
|
|||
|
stream, err = mp3Format.Open(f, panakoInstance.BlockSize)
|
|||
|
|
|||
|
case "audio/ogg":
|
|||
|
fallthrough
|
|||
|
case "audio/opus":
|
|||
|
stream, err = opusFormat.Open(f, panakoInstance.BlockSize)
|
|||
|
}
|
|||
|
|
|||
|
if err != nil { //cannot decode
|
|||
|
//TODO
|
|||
|
log.Print(err)
|
|||
|
f.Close()
|
|||
|
continue
|
|||
|
}
|
|||
|
|
|||
|
if stream == nil { //no known decoder
|
|||
|
//TODO
|
|||
|
log.Print(fmt.Errorf("no known decoder for %s", f.Name()))
|
|||
|
f.Close()
|
|||
|
continue
|
|||
|
}
|
|||
|
|
|||
|
entry := &analyzeEntry{
|
|||
|
fileEntry: e,
|
|||
|
fileHandle: f,
|
|||
|
fileMetadata: meta,
|
|||
|
}
|
|||
|
|
|||
|
var panakoWaitGroup sync.WaitGroup
|
|||
|
panakoWaitGroup.Add(1)
|
|||
|
preLastTotalSamplesWaitGroup.Add(1)
|
|||
|
go func(add bool) {
|
|||
|
defer panakoWaitGroup.Done()
|
|||
|
defer preLastTotalSamplesWaitGroup.Done()
|
|||
|
entry.panakoFingerprints = printStrategy.StreamToFingerprints(stream)
|
|||
|
entry.audioMetadata.sampleRate = int(stream.GetSampleRate())
|
|||
|
entry.audioMetadata.channels = stream.GetChannels()
|
|||
|
entry.audioMetadata.samples = stream.GetSamplesProcessed()
|
|||
|
if add {
|
|||
|
atomic.AddUint32(&preLastTotalSamples, uint32(entry.audioMetadata.samples/entry.audioMetadata.channels))
|
|||
|
}
|
|||
|
}(trackIndex < len(l)-1)
|
|||
|
//TODO: handle extra appended/prepended silence
|
|||
|
|
|||
|
if analyzer != nil {
|
|||
|
if trackIndex == 0 {
|
|||
|
channels := analyzer.Split(4)
|
|||
|
joinedChannels = append(joinedChannels, channels[0])
|
|||
|
ctChannels := channels[1].SkipStartSamples(Int16SamplesPerSector * 10).Split(2)
|
|||
|
joinedCTDBChannels = append(joinedCTDBChannels, ctChannels[0])
|
|||
|
entry.hasherCueToolsCrc32 = NewHasher(ctChannels[1], HashtypeCrc32)
|
|||
|
arChannels := channels[2].SkipStartSamples(Int16SamplesPerSector*5 - 1).Split(2)
|
|||
|
entry.hasherAccurateRipV1 = NewHasher(arChannels[0], HashtypeAccurateRipV1Start)
|
|||
|
entry.hasherAccurateRipV2 = NewHasher(arChannels[1], HashtypeAccurateRipV2Start)
|
|||
|
entry.hasherCrc32 = NewHasher(channels[3], HashtypeCrc32)
|
|||
|
} else if trackIndex == len(l)-1 {
|
|||
|
|
|||
|
channels := analyzer.Split(4)
|
|||
|
joinedChannels = append(joinedChannels, channels[0])
|
|||
|
ctChannels := channels[1].SkipEndSamplesMultiple(&preLastTotalSamplesWaitGroup, &preLastTotalSamples, Int16SamplesPerSector*10).Split(2)
|
|||
|
joinedCTDBChannels = append(joinedCTDBChannels, ctChannels[0])
|
|||
|
entry.hasherCueToolsCrc32 = NewHasher(ctChannels[1], HashtypeCrc32)
|
|||
|
arChannels := channels[2].SkipEndSamples(Int16SamplesPerSector * 5).Split(2)
|
|||
|
entry.hasherAccurateRipV1 = NewHasher(arChannels[0], HashtypeAccurateRipV1)
|
|||
|
entry.hasherAccurateRipV2 = NewHasher(arChannels[1], HashtypeAccurateRipV2)
|
|||
|
entry.hasherCrc32 = NewHasher(channels[3], HashtypeCrc32)
|
|||
|
} else {
|
|||
|
channels := analyzer.Split(5)
|
|||
|
joinedChannels = append(joinedChannels, channels[0])
|
|||
|
joinedCTDBChannels = append(joinedCTDBChannels, channels[1])
|
|||
|
entry.hasherCrc32 = NewHasher(channels[2], HashtypeCrc32)
|
|||
|
entry.hasherAccurateRipV1 = NewHasher(channels[3], HashtypeAccurateRipV1)
|
|||
|
entry.hasherAccurateRipV2 = NewHasher(channels[4], HashtypeAccurateRipV2)
|
|||
|
}
|
|||
|
|
|||
|
waitGroups = append(waitGroups, entry.hasherCrc32.GetWaitGroup(), entry.hasherAccurateRipV1.GetWaitGroup(), entry.hasherAccurateRipV2.GetWaitGroup())
|
|||
|
if entry.hasherCueToolsCrc32 != nil {
|
|||
|
waitGroups = append(waitGroups, entry.hasherCueToolsCrc32.GetWaitGroup())
|
|||
|
}
|
|||
|
}
|
|||
|
|
|||
|
waitGroups = append(waitGroups, &panakoWaitGroup)
|
|||
|
entries = append(entries, entry)
|
|||
|
|
|||
|
}
|
|||
|
|
|||
|
fullHasher := NewHasher(MergeHasherChannels(joinedChannels...), HashtypeCrc32)
|
|||
|
fullCTDBHasher := NewHasher(MergeHasherChannels(joinedCTDBChannels...), HashtypeCrc32)
|
|||
|
|
|||
|
fullHasher.Wait()
|
|||
|
fullCTDBHasher.Wait()
|
|||
|
|
|||
|
fullCRC32 = binary.BigEndian.Uint32(fullHasher.GetResult())
|
|||
|
fullCTDBCRC32 = binary.BigEndian.Uint32(fullCTDBHasher.GetResult())
|
|||
|
|
|||
|
//Wait for all tasks
|
|||
|
for _, wg := range waitGroups {
|
|||
|
wg.Wait()
|
|||
|
}
|
|||
|
|
|||
|
return
|
|||
|
|
|||
|
}
|
|||
|
|
|||
|
type fileEntry struct {
|
|||
|
Name string
|
|||
|
NormalizedSortName string
|
|||
|
NormalizedName string
|
|||
|
}
|
|||
|
|
|||
|
func processAudioFiles(files []string) (result fileEntryList) {
|
|||
|
|
|||
|
result = make(fileEntryList, 0, len(files))
|
|||
|
|
|||
|
for _, f := range files {
|
|||
|
normalized := norm.NFC.String(f)
|
|||
|
normalized = strings.ReplaceAll(normalized, "0", "0")
|
|||
|
normalized = strings.ReplaceAll(normalized, "1", "1")
|
|||
|
normalized = strings.ReplaceAll(normalized, "2", "2")
|
|||
|
normalized = strings.ReplaceAll(normalized, "3", "3")
|
|||
|
normalized = strings.ReplaceAll(normalized, "4", "4")
|
|||
|
normalized = strings.ReplaceAll(normalized, "5", "5")
|
|||
|
normalized = strings.ReplaceAll(normalized, "6", "6")
|
|||
|
normalized = strings.ReplaceAll(normalized, "7", "7")
|
|||
|
normalized = strings.ReplaceAll(normalized, "8", "8")
|
|||
|
normalized = strings.ReplaceAll(normalized, "9", "9")
|
|||
|
|
|||
|
ext := strings.LastIndex(normalized, ".")
|
|||
|
for k := 0; k < ext; k++ {
|
|||
|
index := strings.IndexFunc(normalized[k:], unicode.IsNumber)
|
|||
|
if index == -1 {
|
|||
|
//wtf, no numbers?
|
|||
|
result = append(result, fileEntry{
|
|||
|
Name: f,
|
|||
|
NormalizedSortName: strings.TrimSpace(strings.TrimLeft(strings.TrimSpace(normalized[:ext]), separatorTrimSet)),
|
|||
|
NormalizedName: strings.TrimSpace(strings.TrimLeft(strings.TrimSpace(normalized[:ext]), separatorTrimSet)),
|
|||
|
})
|
|||
|
break
|
|||
|
}
|
|||
|
|
|||
|
index += k
|
|||
|
|
|||
|
if index == 0 || isSeparator(normalized[index-1]) { //If it's start of string or prefixed by a space
|
|||
|
normalized = normalized[index:ext]
|
|||
|
|
|||
|
firstNotNumber := strings.IndexFunc(normalized, func(r rune) bool {
|
|||
|
return !unicode.IsNumber(r)
|
|||
|
})
|
|||
|
|
|||
|
r := fileEntry{
|
|||
|
Name: f,
|
|||
|
NormalizedSortName: strings.TrimSpace(strings.TrimLeft(strings.TrimSpace(normalized), separatorTrimSet)),
|
|||
|
NormalizedName: strings.TrimSpace(strings.TrimLeft(strings.TrimSpace(normalized[firstNotNumber:]), separatorTrimSet)),
|
|||
|
}
|
|||
|
result = append(result, r)
|
|||
|
break
|
|||
|
}
|
|||
|
|
|||
|
k = index
|
|||
|
}
|
|||
|
|
|||
|
}
|
|||
|
|
|||
|
//Sort files naturally
|
|||
|
sort.SliceStable(result, func(i, j int) bool {
|
|||
|
return natsort.Compare(result[i].NormalizedSortName, result[j].NormalizedSortName)
|
|||
|
})
|
|||
|
|
|||
|
return
|
|||
|
}
|
|||
|
|
|||
|
type DiscHandlerResult struct {
|
|||
|
TOC TOC
|
|||
|
CRC32 uint32
|
|||
|
CueToolsCRC32 uint32
|
|||
|
Directory string
|
|||
|
Tracks []DiscHandlerTrack
|
|||
|
}
|
|||
|
|
|||
|
type DiscHandlerTrack struct {
|
|||
|
FileName string
|
|||
|
TrackName string
|
|||
|
SortName string
|
|||
|
Fingerprints struct {
|
|||
|
Panako []*panako.Fingerprint
|
|||
|
CRC32 uint32
|
|||
|
CueToolsCRC32 uint32
|
|||
|
AccurateRipV1 uint32
|
|||
|
AccurateRipV2 uint32
|
|||
|
}
|
|||
|
FileMetadata struct {
|
|||
|
DiscNumber int
|
|||
|
Album string
|
|||
|
AlbumArtist string
|
|||
|
Artist string
|
|||
|
Composer string
|
|||
|
Year int
|
|||
|
TrackNumber int
|
|||
|
Title string
|
|||
|
EmbeddedPicture []byte
|
|||
|
}
|
|||
|
AudioMetadata struct {
|
|||
|
SampleRate int
|
|||
|
Channels int
|
|||
|
NumberOfFullSamples int
|
|||
|
Duration time.Duration
|
|||
|
}
|
|||
|
}
|
|||
|
|
|||
|
func HandleDiscEntry(panakoInstance *panako.Instance, pathEntry string) *DiscHandlerResult {
|
|||
|
log.Printf("Handling %q", pathEntry)
|
|||
|
entries, err := ioutil.ReadDir(pathEntry)
|
|||
|
if err != nil {
|
|||
|
return nil
|
|||
|
}
|
|||
|
|
|||
|
var audioFiles []string
|
|||
|
var imageFiles []string
|
|||
|
var metadataFiles []string
|
|||
|
var folders []string
|
|||
|
|
|||
|
for _, entry := range entries {
|
|||
|
if !entry.IsDir() {
|
|||
|
ext := path.Ext(entry.Name())
|
|||
|
mime := utilities.GetMimeTypeFromExtension(ext)
|
|||
|
isAudio := mime[0:6] == "audio/"
|
|||
|
isAudioMetadata := mime == "text/x-log" || mime == "text/x-accurip" || mime == "text/x-cue" || mime == "text/x-toc"
|
|||
|
isImage := mime[0:6] == "image/"
|
|||
|
|
|||
|
if isAudio {
|
|||
|
audioFiles = append(audioFiles, entry.Name())
|
|||
|
} else if isImage {
|
|||
|
imageFiles = append(imageFiles, entry.Name())
|
|||
|
} else if isAudioMetadata {
|
|||
|
metadataFiles = append(metadataFiles, entry.Name())
|
|||
|
}
|
|||
|
} else {
|
|||
|
folders = append(folders, entry.Name())
|
|||
|
}
|
|||
|
}
|
|||
|
|
|||
|
if len(audioFiles) == 0 {
|
|||
|
return nil
|
|||
|
}
|
|||
|
|
|||
|
sortedAudioEntries := processAudioFiles(audioFiles)
|
|||
|
|
|||
|
disc := &DiscHandlerResult{
|
|||
|
TOC: TOC{TocPregap},
|
|||
|
}
|
|||
|
|
|||
|
result, fullCRC32, fullCTDBCRC32 := sortedAudioEntries.analyze(pathEntry, panakoInstance)
|
|||
|
defer func() {
|
|||
|
for _, entry := range result {
|
|||
|
entry.fileHandle.Close()
|
|||
|
}
|
|||
|
}()
|
|||
|
|
|||
|
disc.CRC32 = fullCRC32
|
|||
|
disc.CueToolsCRC32 = fullCTDBCRC32
|
|||
|
|
|||
|
for _, entry := range result {
|
|||
|
|
|||
|
track := DiscHandlerTrack{
|
|||
|
FileName: entry.Name,
|
|||
|
TrackName: entry.NormalizedName,
|
|||
|
SortName: entry.NormalizedSortName,
|
|||
|
}
|
|||
|
track.AudioMetadata.SampleRate = entry.audioMetadata.sampleRate
|
|||
|
track.AudioMetadata.Channels = entry.audioMetadata.channels
|
|||
|
track.AudioMetadata.NumberOfFullSamples = entry.audioMetadata.samples / entry.audioMetadata.channels
|
|||
|
track.AudioMetadata.Duration = time.Duration(float64(time.Second) * float64(track.AudioMetadata.NumberOfFullSamples) / float64(track.AudioMetadata.SampleRate))
|
|||
|
track.Fingerprints.Panako = entry.panakoFingerprints
|
|||
|
|
|||
|
disc.TOC = append(disc.TOC, disc.TOC[len(disc.TOC)-1]+track.AudioMetadata.NumberOfFullSamples/Int16SamplesPerSector)
|
|||
|
|
|||
|
if entry.hasherCrc32 != nil {
|
|||
|
track.Fingerprints.CRC32 = binary.BigEndian.Uint32(entry.hasherCrc32.GetResult())
|
|||
|
track.Fingerprints.CueToolsCRC32 = track.Fingerprints.CRC32
|
|||
|
}
|
|||
|
if entry.hasherCueToolsCrc32 != nil {
|
|||
|
track.Fingerprints.CueToolsCRC32 = binary.BigEndian.Uint32(entry.hasherCueToolsCrc32.GetResult())
|
|||
|
}
|
|||
|
if entry.hasherAccurateRipV1 != nil {
|
|||
|
track.Fingerprints.AccurateRipV1 = binary.BigEndian.Uint32(entry.hasherAccurateRipV1.GetResult())
|
|||
|
}
|
|||
|
if entry.hasherAccurateRipV2 != nil {
|
|||
|
track.Fingerprints.AccurateRipV2 = binary.BigEndian.Uint32(entry.hasherAccurateRipV2.GetResult())
|
|||
|
}
|
|||
|
|
|||
|
track.FileMetadata.DiscNumber, _ = entry.fileMetadata.Disc()
|
|||
|
track.FileMetadata.TrackNumber, _ = entry.fileMetadata.Track()
|
|||
|
track.FileMetadata.Year = entry.fileMetadata.Year()
|
|||
|
track.FileMetadata.AlbumArtist = entry.fileMetadata.AlbumArtist()
|
|||
|
track.FileMetadata.Artist = entry.fileMetadata.Artist()
|
|||
|
track.FileMetadata.Composer = entry.fileMetadata.Composer()
|
|||
|
track.FileMetadata.Title = entry.fileMetadata.Title()
|
|||
|
|
|||
|
if entry.fileMetadata.Picture() != nil {
|
|||
|
track.FileMetadata.EmbeddedPicture = entry.fileMetadata.Picture().Data
|
|||
|
}
|
|||
|
|
|||
|
disc.Tracks = append(disc.Tracks, track)
|
|||
|
}
|
|||
|
|
|||
|
disc.TOC = append(TOC{disc.TOC[len(disc.TOC)-1]}, disc.TOC[0:len(disc.TOC)-1]...)
|
|||
|
|
|||
|
return disc
|
|||
|
}
|