METANOIA/metadata/dischandler.go

package metadata

import (
	"encoding/binary"
	"facette.io/natsort"
	"fmt"
	"git.gammaspectra.live/S.O.N.G/Hibiki/panako"
	"git.gammaspectra.live/S.O.N.G/Hibiki/utilities/audio"
	"git.gammaspectra.live/S.O.N.G/Hibiki/utilities/audio/format/flac"
	"git.gammaspectra.live/S.O.N.G/Hibiki/utilities/audio/format/mp3"
	"git.gammaspectra.live/S.O.N.G/Hibiki/utilities/audio/format/opus"
	"git.gammaspectra.live/S.O.N.G/Hibiki/utilities/specializedstore"
	"git.gammaspectra.live/S.O.N.G/METANOIA/utilities"
	"github.com/dhowden/tag"
	"golang.org/x/text/unicode/norm"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"
	"unicode"
)

type fileEntryList []fileEntry

var flacFormat = flac.NewFormat()
var mp3Format = mp3.NewFormat()
var opusFormat = opus.NewFormat()

const separatorTrimSet = ",.-_()[]{}"

func isSeparator(b byte) bool {
	if b == ' ' {
		return true
	}
	for i := 0; i < len(separatorTrimSet); i++ {
		if separatorTrimSet[i] == b {
			return true
		}
	}

	return false
}

type analyzeEntry struct {
	fileEntry
	fileHandle    *os.File
	audioMetadata struct {
		sampleRate int
		channels   int
		samples    int
	}
	fileMetadata        tag.Metadata
	panakoFingerprints  []*panako.Fingerprint
	hasherCrc32         *Hasher
	hasherCueToolsCrc32 *Hasher
	hasherAccurateRipV1 *Hasher
	hasherAccurateRipV2 *Hasher
}

func (l fileEntryList) analyze(directory string, panakoInstance *panako.Instance) (entries []*analyzeEntry, fullCRC32 uint32, fullCTDBCRC32 uint32) {
	var waitGroups []*sync.WaitGroup

	printStrategy := panakoInstance.GetStrategy(specializedstore.NewMemoryStore(), audio.RESAMPLER_QUALITY_LINEAR)

	var joinedCTDBChannels []HasherChannel
	var joinedChannels []HasherChannel

	var preLastTotalSamplesWaitGroup sync.WaitGroup
	preLastTotalSamples := uint32(0)

	for trackIndex, e := range l {
		f, err := os.Open(path.Join(directory, e.Name))
		if err != nil {
			//TODO
			log.Print(err)
			continue
		}

		var stream *audio.Stream
		var analyzer HasherChannel

		meta, err := tag.ReadFrom(f)
		if err != nil {
			log.Print(err)
			err = nil
		}
		f.Seek(0, 0)

		switch utilities.GetMimeTypeFromExtension(path.Ext(e.Name)) {
		case "audio/flac":
			stream, analyzer, err = flacFormat.OpenAnalyzer(f, panakoInstance.BlockSize)

		case "audio/mpeg;codecs=mp3":
			stream, err = mp3Format.Open(f, panakoInstance.BlockSize)

		case "audio/ogg":
			fallthrough
		case "audio/opus":
			stream, err = opusFormat.Open(f, panakoInstance.BlockSize)
		}

		if err != nil { //cannot decode
			//TODO
			log.Print(err)
			f.Close()
			continue
		}

		if stream == nil { //no known decoder
			//TODO
			log.Print(fmt.Errorf("no known decoder for %s", f.Name()))
			f.Close()
			continue
		}

		entry := &analyzeEntry{
			fileEntry:    e,
			fileHandle:   f,
			fileMetadata: meta,
		}

		var panakoWaitGroup sync.WaitGroup
		panakoWaitGroup.Add(1)
		preLastTotalSamplesWaitGroup.Add(1)
		go func(add bool) {
			defer panakoWaitGroup.Done()
			defer preLastTotalSamplesWaitGroup.Done()
			entry.panakoFingerprints = printStrategy.StreamToFingerprints(stream)
			entry.audioMetadata.sampleRate = int(stream.GetSampleRate())
			entry.audioMetadata.channels = stream.GetChannels()
			entry.audioMetadata.samples = stream.GetSamplesProcessed()
			if add {
				atomic.AddUint32(&preLastTotalSamples, uint32(entry.audioMetadata.samples/entry.audioMetadata.channels))
			}
		}(trackIndex < len(l)-1)
		//TODO: handle extra appended/prepended silence

		if analyzer != nil {
			if trackIndex == 0 {
				channels := analyzer.Split(4)
				joinedChannels = append(joinedChannels, channels[0])
				ctChannels := channels[1].SkipStartSamples(Int16SamplesPerSector * 10).Split(2)
				joinedCTDBChannels = append(joinedCTDBChannels, ctChannels[0])
				entry.hasherCueToolsCrc32 = NewHasher(ctChannels[1], HashtypeCrc32)
				arChannels := channels[2].SkipStartSamples(Int16SamplesPerSector*5 - 1).Split(2)
				entry.hasherAccurateRipV1 = NewHasher(arChannels[0], HashtypeAccurateRipV1Start)
				entry.hasherAccurateRipV2 = NewHasher(arChannels[1], HashtypeAccurateRipV2Start)
				entry.hasherCrc32 = NewHasher(channels[3], HashtypeCrc32)
			} else if trackIndex == len(l)-1 {

				channels := analyzer.Split(4)
				joinedChannels = append(joinedChannels, channels[0])
				ctChannels := channels[1].SkipEndSamplesMultiple(&preLastTotalSamplesWaitGroup, &preLastTotalSamples, Int16SamplesPerSector*10).Split(2)
				joinedCTDBChannels = append(joinedCTDBChannels, ctChannels[0])
				entry.hasherCueToolsCrc32 = NewHasher(ctChannels[1], HashtypeCrc32)
				arChannels := channels[2].SkipEndSamples(Int16SamplesPerSector * 5).Split(2)
				entry.hasherAccurateRipV1 = NewHasher(arChannels[0], HashtypeAccurateRipV1)
				entry.hasherAccurateRipV2 = NewHasher(arChannels[1], HashtypeAccurateRipV2)
				entry.hasherCrc32 = NewHasher(channels[3], HashtypeCrc32)
			} else {
				channels := analyzer.Split(5)
				joinedChannels = append(joinedChannels, channels[0])
				joinedCTDBChannels = append(joinedCTDBChannels, channels[1])
				entry.hasherCrc32 = NewHasher(channels[2], HashtypeCrc32)
				entry.hasherAccurateRipV1 = NewHasher(channels[3], HashtypeAccurateRipV1)
				entry.hasherAccurateRipV2 = NewHasher(channels[4], HashtypeAccurateRipV2)
			}

			waitGroups = append(waitGroups, entry.hasherCrc32.GetWaitGroup(), entry.hasherAccurateRipV1.GetWaitGroup(), entry.hasherAccurateRipV2.GetWaitGroup())
			if entry.hasherCueToolsCrc32 != nil {
				waitGroups = append(waitGroups, entry.hasherCueToolsCrc32.GetWaitGroup())
			}
		}

		waitGroups = append(waitGroups, &panakoWaitGroup)
		entries = append(entries, entry)

	}

	fullHasher := NewHasher(MergeHasherChannels(joinedChannels...), HashtypeCrc32)
	fullCTDBHasher := NewHasher(MergeHasherChannels(joinedCTDBChannels...), HashtypeCrc32)

	fullHasher.Wait()
	fullCTDBHasher.Wait()

	fullCRC32 = binary.BigEndian.Uint32(fullHasher.GetResult())
	fullCTDBCRC32 = binary.BigEndian.Uint32(fullCTDBHasher.GetResult())

	//Wait for all tasks
	for _, wg := range waitGroups {
		wg.Wait()
	}

	return

}

type fileEntry struct {
	Name               string
	NormalizedSortName string
	NormalizedName     string
}

func processAudioFiles(files []string) (result fileEntryList) {

	result = make(fileEntryList, 0, len(files))

	for _, f := range files {
		normalized := norm.NFC.String(f)
		normalized = strings.ReplaceAll(normalized, "０", "0")
		normalized = strings.ReplaceAll(normalized, "１", "1")
		normalized = strings.ReplaceAll(normalized, "２", "2")
		normalized = strings.ReplaceAll(normalized, "３", "3")
		normalized = strings.ReplaceAll(normalized, "４", "4")
		normalized = strings.ReplaceAll(normalized, "５", "5")
		normalized = strings.ReplaceAll(normalized, "６", "6")
		normalized = strings.ReplaceAll(normalized, "７", "7")
		normalized = strings.ReplaceAll(normalized, "８", "8")
		normalized = strings.ReplaceAll(normalized, "９", "9")

		ext := strings.LastIndex(normalized, ".")
		for k := 0; k < ext; k++ {
			index := strings.IndexFunc(normalized[k:], unicode.IsNumber)
			if index == -1 {
				//wtf, no numbers?
				result = append(result, fileEntry{
					Name:               f,
					NormalizedSortName: strings.TrimSpace(strings.TrimLeft(strings.TrimSpace(normalized[:ext]), separatorTrimSet)),
					NormalizedName:     strings.TrimSpace(strings.TrimLeft(strings.TrimSpace(normalized[:ext]), separatorTrimSet)),
				})
				break
			}

			index += k

			if index == 0 || isSeparator(normalized[index-1]) { //If it's start of string or prefixed by a space
				normalized = normalized[index:ext]

				firstNotNumber := strings.IndexFunc(normalized, func(r rune) bool {
					return !unicode.IsNumber(r)
				})

				r := fileEntry{
					Name:               f,
					NormalizedSortName: strings.TrimSpace(strings.TrimLeft(strings.TrimSpace(normalized), separatorTrimSet)),
					NormalizedName:     strings.TrimSpace(strings.TrimLeft(strings.TrimSpace(normalized[firstNotNumber:]), separatorTrimSet)),
				}
				result = append(result, r)
				break
			}

			k = index
		}

	}

	//Sort files naturally
	sort.SliceStable(result, func(i, j int) bool {
		return natsort.Compare(result[i].NormalizedSortName, result[j].NormalizedSortName)
	})

	return
}

type DiscHandlerResult struct {
	TOC           TOC
	CRC32         uint32
	CueToolsCRC32 uint32
	Directory     string
	Tracks        []DiscHandlerTrack
}

type DiscHandlerTrack struct {
	FileName     string
	TrackName    string
	SortName     string
	Fingerprints struct {
		Panako        []*panako.Fingerprint
		CRC32         uint32
		CueToolsCRC32 uint32
		AccurateRipV1 uint32
		AccurateRipV2 uint32
	}
	FileMetadata struct {
		DiscNumber      int
		Album           string
		AlbumArtist     string
		Artist          string
		Composer        string
		Year            int
		TrackNumber     int
		Title           string
		EmbeddedPicture []byte
	}
	AudioMetadata struct {
		SampleRate          int
		Channels            int
		NumberOfFullSamples int
		Duration            time.Duration
	}
}

func HandleDiscEntry(panakoInstance *panako.Instance, pathEntry string) *DiscHandlerResult {
	log.Printf("Handling %q", pathEntry)
	entries, err := ioutil.ReadDir(pathEntry)
	if err != nil {
		return nil
	}

	var audioFiles []string
	var imageFiles []string
	var metadataFiles []string
	var folders []string

	for _, entry := range entries {
		if !entry.IsDir() {
			ext := path.Ext(entry.Name())
			mime := utilities.GetMimeTypeFromExtension(ext)
			isAudio := mime[0:6] == "audio/"
			isAudioMetadata := mime == "text/x-log" || mime == "text/x-accurip" || mime == "text/x-cue" || mime == "text/x-toc"
			isImage := mime[0:6] == "image/"

			if isAudio {
				audioFiles = append(audioFiles, entry.Name())
			} else if isImage {
				imageFiles = append(imageFiles, entry.Name())
			} else if isAudioMetadata {
				metadataFiles = append(metadataFiles, entry.Name())
			}
		} else {
			folders = append(folders, entry.Name())
		}
	}

	if len(audioFiles) == 0 {
		return nil
	}

	sortedAudioEntries := processAudioFiles(audioFiles)

	disc := &DiscHandlerResult{
		Directory: pathEntry,
		TOC:       TOC{TocPregap},
	}

	result, fullCRC32, fullCTDBCRC32 := sortedAudioEntries.analyze(pathEntry, panakoInstance)
	defer func() {
		for _, entry := range result {
			entry.fileHandle.Close()
		}
	}()

	disc.CRC32 = fullCRC32
	disc.CueToolsCRC32 = fullCTDBCRC32

	for _, entry := range result {

		track := DiscHandlerTrack{
			FileName:  entry.Name,
			TrackName: entry.NormalizedName,
			SortName:  entry.NormalizedSortName,
		}
		track.AudioMetadata.SampleRate = entry.audioMetadata.sampleRate
		track.AudioMetadata.Channels = entry.audioMetadata.channels
		track.AudioMetadata.NumberOfFullSamples = entry.audioMetadata.samples / entry.audioMetadata.channels
		track.AudioMetadata.Duration = time.Duration(float64(time.Second) * float64(track.AudioMetadata.NumberOfFullSamples) / float64(track.AudioMetadata.SampleRate))
		track.Fingerprints.Panako = entry.panakoFingerprints

		disc.TOC = append(disc.TOC, disc.TOC[len(disc.TOC)-1]+track.AudioMetadata.NumberOfFullSamples/Int16SamplesPerSector)

		if entry.hasherCrc32 != nil {
			track.Fingerprints.CRC32 = binary.BigEndian.Uint32(entry.hasherCrc32.GetResult())
			track.Fingerprints.CueToolsCRC32 = track.Fingerprints.CRC32
		}
		if entry.hasherCueToolsCrc32 != nil {
			track.Fingerprints.CueToolsCRC32 = binary.BigEndian.Uint32(entry.hasherCueToolsCrc32.GetResult())
		}
		if entry.hasherAccurateRipV1 != nil {
			track.Fingerprints.AccurateRipV1 = binary.BigEndian.Uint32(entry.hasherAccurateRipV1.GetResult())
		}
		if entry.hasherAccurateRipV2 != nil {
			track.Fingerprints.AccurateRipV2 = binary.BigEndian.Uint32(entry.hasherAccurateRipV2.GetResult())
		}

		track.FileMetadata.DiscNumber, _ = entry.fileMetadata.Disc()
		track.FileMetadata.TrackNumber, _ = entry.fileMetadata.Track()
		track.FileMetadata.Year = entry.fileMetadata.Year()
		track.FileMetadata.AlbumArtist = entry.fileMetadata.AlbumArtist()
		track.FileMetadata.Artist = entry.fileMetadata.Artist()
		track.FileMetadata.Composer = entry.fileMetadata.Composer()
		track.FileMetadata.Album = entry.fileMetadata.Album()
		track.FileMetadata.Title = entry.fileMetadata.Title()

		if entry.fileMetadata.Picture() != nil {
			track.FileMetadata.EmbeddedPicture = entry.fileMetadata.Picture().Data
		}

		disc.Tracks = append(disc.Tracks, track)
	}

	disc.TOC = append(TOC{disc.TOC[len(disc.TOC)-1]}, disc.TOC[0:len(disc.TOC)-1]...)

	return disc
}