1164 lines
36 KiB
Go
1164 lines
36 KiB
Go
package main
|
||
|
||
import (
|
||
"encoding/json"
|
||
"flag"
|
||
"fmt"
|
||
"git.gammaspectra.live/S.O.N.G/touhouwiki-mirror/utilities"
|
||
wikitext_parser "git.gammaspectra.live/S.O.N.G/wikitext-parser"
|
||
"golang.org/x/text/cases"
|
||
"golang.org/x/text/language"
|
||
"golang.org/x/text/runes"
|
||
"golang.org/x/text/transform"
|
||
"golang.org/x/text/unicode/norm"
|
||
"io/fs"
|
||
"io/ioutil"
|
||
"log"
|
||
"math"
|
||
"net/http"
|
||
"os"
|
||
"path"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
"unicode"
|
||
)
|
||
|
||
var cdIndex = make(map[int]*albumEntry)
|
||
var cdIndexLock sync.Mutex
|
||
|
||
var albumTitleLookup = make(map[string][]*albumEntry)
|
||
|
||
//TODO: make this work with discs
|
||
var discTracksLookup = make(map[int][]*albumEntry)
|
||
|
||
type categoryPageIndex struct {
|
||
PageId int `json:"pageid"`
|
||
Namespace int `json:"ns"`
|
||
Title string `json:"title"`
|
||
Timestamp string `json:"timestamp"`
|
||
}
|
||
|
||
func parseCategoryPageIndex(filePath string) []categoryPageIndex {
|
||
s := &struct {
|
||
Query struct {
|
||
Members []categoryPageIndex `json:"categorymembers"`
|
||
} `json:"query"`
|
||
}{}
|
||
|
||
fileData, err := ioutil.ReadFile(filePath)
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
|
||
err = json.Unmarshal(fileData, s)
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
return s.Query.Members
|
||
}
|
||
|
||
type JSONTime struct {
|
||
time.Time
|
||
}
|
||
|
||
func (t JSONTime) MarshalJSON() ([]byte, error) {
|
||
if t.IsZero() {
|
||
return nil, nil
|
||
}
|
||
return []byte(fmt.Sprintf("\"%s\"", t.Format("2006-01-02"))), nil
|
||
}
|
||
|
||
type albumEntry struct {
|
||
Id int `json:"pageid"`
|
||
Type string `json:"type"`
|
||
MainTitle string `json:"pagetitle"`
|
||
Titles map[string]string `json:"titles"`
|
||
Image string `json:"image,omitempty"`
|
||
CatalogNumbers []string `json:"catalognumbers,omitempty"`
|
||
Genre []string `json:"genre,omitempty"`
|
||
TrackCount int `json:"trackcount,omitempty"`
|
||
Duration int `json:"duration,omitempty"`
|
||
ReleaseDate JSONTime `json:"releasedate,omitempty"`
|
||
ReleaseEvent string `json:"releaseevent,omitempty"`
|
||
Links []string `json:"links,omitempty"`
|
||
Artists []artistEntry `json:"artists,omitempty"`
|
||
Discs []discEntry `json:"discs,omitempty"`
|
||
}
|
||
type discEntry struct {
|
||
Name string `json:"name,omitempty"`
|
||
Duration int `json:"duration,omitempty"`
|
||
TrackCount int `json:"trackcount,omitempty"`
|
||
Tracks []trackEntry `json:"tracks"`
|
||
}
|
||
type trackEntry struct {
|
||
Duration int `json:"duration,omitempty"`
|
||
MainTitle string `json:"title,omitempty"`
|
||
Titles map[string]string `json:"titles,omitempty"`
|
||
Artists []artistEntry `json:"artists,omitempty"`
|
||
Lyrics string `json:"lyrics,omitempty"`
|
||
Links []string `json:"links,omitempty"`
|
||
Original []string `json:"original,omitempty"`
|
||
}
|
||
type artistEntry struct {
|
||
Position string `json:"position"`
|
||
Names []string `json:"names"`
|
||
}
|
||
|
||
func getArtistEntries(kind string, entries []interface{}, opts *wikitext_parser.WikiStringValueOptions) (artists []artistEntry) {
|
||
|
||
artist := artistEntry{
|
||
Position: kind,
|
||
}
|
||
|
||
recreateArtist := func(kinds ...string) {
|
||
var names []string
|
||
for _, n := range artist.Names {
|
||
//TODO
|
||
//n = strings.Trim(n, " ()[]（）")
|
||
if opts.Trim {
|
||
n = strings.TrimSpace(n)
|
||
}
|
||
if len(n) > 0 {
|
||
var curName string
|
||
if len(names) > 0 {
|
||
curName = names[len(names)-1]
|
||
}
|
||
if len(curName) > 0 && curName[len(curName)-1] == '(' {
|
||
names[len(names)-1] += n
|
||
} else if len(curName) > 0 && len(n) > 1 && n[len(n)-1] == ')' && strings.Index(n, "(") <= 0 {
|
||
names[len(names)-1] += " " + n
|
||
} else if len(curName) > 0 && n[0] == '(' && (strings.Index(n, ")") == -1 || n[len(n)-1] == ')') {
|
||
names[len(names)-1] += " " + n
|
||
} else if len(curName) > 0 && n == ")" {
|
||
names[len(names)-1] += n
|
||
} else if len(curName) > 0 && curName[len(curName)-1] == '/' {
|
||
names[len(names)-1] += n
|
||
} else if len(curName) > 0 && n == "/" {
|
||
names[len(names)-1] += n
|
||
} else {
|
||
names = append(names, n)
|
||
}
|
||
}
|
||
}
|
||
|
||
artist.Names = names
|
||
if len(artist.Names) > 0 {
|
||
artists = append(artists, artist)
|
||
artist = artistEntry{
|
||
Position: strings.Join(kinds, ", "),
|
||
}
|
||
}
|
||
}
|
||
|
||
for _, value := range entries {
|
||
if text, ok := value.(string); ok {
|
||
if i := strings.Index(text, " & "); i != -1 {
|
||
for _, vv := range strings.Split(text, " & ") {
|
||
vv = normalizeStringCharacters(vv)
|
||
if len(vv) > 0 {
|
||
artist.Names = append(artist.Names, vv)
|
||
}
|
||
recreateArtist(kind)
|
||
}
|
||
} else if i := strings.Index(text, "& "); i == 0 {
|
||
for _, vv := range strings.Split(text, "& ") {
|
||
vv = normalizeStringCharacters(vv)
|
||
if len(vv) > 0 {
|
||
artist.Names = append(artist.Names, vv)
|
||
}
|
||
recreateArtist(kind)
|
||
}
|
||
} else if i := strings.Index(text, " &"); i != -1 && i == len(text)-2 {
|
||
for _, vv := range strings.Split(text, " &") {
|
||
vv = normalizeStringCharacters(vv)
|
||
if len(vv) > 0 {
|
||
artist.Names = append(artist.Names, vv)
|
||
}
|
||
recreateArtist(kind)
|
||
}
|
||
} else if i := strings.Index(text, " and "); i != -1 {
|
||
for _, vv := range strings.Split(text, " and ") {
|
||
vv = normalizeStringCharacters(vv)
|
||
if len(vv) > 0 {
|
||
artist.Names = append(artist.Names, vv)
|
||
}
|
||
recreateArtist(kind)
|
||
}
|
||
} else if i := strings.Index(text, "and "); i == 0 {
|
||
for _, vv := range strings.Split(text, "and ") {
|
||
vv = normalizeStringCharacters(vv)
|
||
if len(vv) > 0 {
|
||
artist.Names = append(artist.Names, vv)
|
||
}
|
||
recreateArtist(kind)
|
||
}
|
||
} else if i := strings.Index(text, " and"); i != -1 && i == len(text)-4 {
|
||
for _, vv := range strings.Split(text, " and") {
|
||
vv = normalizeStringCharacters(vv)
|
||
if len(vv) > 0 {
|
||
artist.Names = append(artist.Names, vv)
|
||
}
|
||
recreateArtist(kind)
|
||
}
|
||
} else {
|
||
for _, vv := range strings.Split(text, " and") {
|
||
vv = normalizeStringCharacters(vv)
|
||
if len(vv) > 0 {
|
||
artist.Names = append(artist.Names, vv)
|
||
}
|
||
}
|
||
}
|
||
} else if _, ok := value.(wikitext_parser.NewLineToken); ok {
|
||
recreateArtist(kind)
|
||
} else if tpl, ok := value.(*wikitext_parser.Template); ok {
|
||
if tpl.IsLink {
|
||
var result []string
|
||
for _, vv := range tpl.Parameters {
|
||
result = append(result, getStringValue(vv, opts)...)
|
||
}
|
||
if len(result) == 0 {
|
||
result = append(result, tpl.Name)
|
||
}
|
||
artist.Names = append(artist.Names, strings.Join(result, " "))
|
||
} else {
|
||
artist.Names = append(artist.Names, strings.Join(getStringValue([]interface{}{tpl}, opts), " "))
|
||
}
|
||
} else if link, ok := value.(*wikitext_parser.Link); ok {
|
||
if len(link.Name) > 0 {
|
||
artist.Names = append(artist.Names, strings.Join(getStringValue(link.Name, opts), " "))
|
||
} else if !link.IsExternal {
|
||
artist.Names = append(artist.Names, link.URL)
|
||
}
|
||
} else if unorderedList, ok := value.(*wikitext_parser.UnorderedList); ok {
|
||
for _, val := range unorderedList.Entries {
|
||
recreateArtist(kind)
|
||
|
||
if text, ok := val.(string); ok {
|
||
artist.Names = append(artist.Names, text)
|
||
} else if tpl, ok := val.(*wikitext_parser.Template); ok {
|
||
if tpl.IsLink {
|
||
artist.Names = append(artist.Names, tpl.Name)
|
||
for _, val := range tpl.Parameters {
|
||
artist.Names = append(artist.Names, getStringValue(val, opts)...)
|
||
}
|
||
} else {
|
||
artist.Names = append(artist.Names, getStringValue([]interface{}{tpl}, opts)...)
|
||
}
|
||
}
|
||
}
|
||
recreateArtist(kind)
|
||
} else if descriptionList, ok := value.(*wikitext_parser.DescriptionList); ok {
|
||
for _, val := range descriptionList.Entries {
|
||
recreateArtist(kind, strings.Join(getStringValue(descriptionList.Name, opts), ", "))
|
||
|
||
if text, ok := val.(string); ok {
|
||
artist.Names = append(artist.Names, text)
|
||
} else if tpl, ok := val.(*wikitext_parser.Template); ok {
|
||
if tpl.IsLink {
|
||
artist.Names = append(artist.Names, tpl.Name)
|
||
for _, val := range tpl.Parameters {
|
||
artist.Names = append(artist.Names, getStringValue(val, opts)...)
|
||
}
|
||
} else {
|
||
artist.Names = append(artist.Names, getStringValue([]interface{}{tpl}, opts)...)
|
||
}
|
||
}
|
||
|
||
recreateArtist(kind)
|
||
}
|
||
}
|
||
}
|
||
|
||
recreateArtist(kind)
|
||
|
||
return
|
||
}
|
||
|
||
func getWikiStringOptions(title string, trim bool) *wikitext_parser.WikiStringValueOptions {
|
||
|
||
opts := &wikitext_parser.WikiStringValueOptions{}
|
||
opts.Default()
|
||
opts.PageName = title
|
||
opts.TemplateHandler = func(template *wikitext_parser.Template, opt *wikitext_parser.WikiStringValueOptions) (result []string) {
|
||
switch strings.ToUpper(template.Name) {
|
||
case "H:TITLE":
|
||
if val, ok := template.Parameters["0"]; ok && len(val) > 0 {
|
||
result = append(result, wikitext_parser.GetWikiStringValue(val, opt)[0])
|
||
}
|
||
case "LANG":
|
||
if val, ok := template.Parameters["1"]; ok && len(val) > 0 {
|
||
result = append(result, wikitext_parser.GetWikiStringValue(val, opt)[0])
|
||
}
|
||
case "GENRE":
|
||
if val, ok := template.Parameters["0"]; ok && len(val) > 0 {
|
||
result = append(result, wikitext_parser.GetWikiStringValue(val, opt)[0])
|
||
}
|
||
case "MUSICARTICLE/DIVIDER":
|
||
case "PAGENAME", "SUBPAGENAME":
|
||
result = append(result, opt.PageName)
|
||
default:
|
||
result = append(result, template.Name)
|
||
}
|
||
return
|
||
}
|
||
opts.Trim = trim
|
||
return opts
|
||
}
|
||
|
||
func getStringValue(v []interface{}, opt *wikitext_parser.WikiStringValueOptions) []string {
|
||
return wikitext_parser.GetWikiStringValue(v, opt)
|
||
}
|
||
|
||
func processIndexDirectory(filePath, indexPath, kind string, wg *sync.WaitGroup) {
|
||
entries, err := ioutil.ReadDir(indexPath)
|
||
if err != nil {
|
||
return
|
||
}
|
||
|
||
for _, e := range entries {
|
||
if path.Ext(e.Name()) == ".json" {
|
||
for _, v := range parseCategoryPageIndex(path.Join(indexPath, e.Name())) {
|
||
wg.Add(1)
|
||
go func(entry *albumEntry) {
|
||
defer wg.Done()
|
||
|
||
contents, err := ioutil.ReadFile(path.Join(filePath, "pages", fmt.Sprintf("%d.wiki", entry.Id)))
|
||
if err != nil {
|
||
return
|
||
}
|
||
result := wikitext_parser.ParseWikiText(string(contents))
|
||
|
||
if len(result) == 0 {
|
||
return
|
||
}
|
||
|
||
tpl, ok := result[0].(*wikitext_parser.Template)
|
||
if !ok {
|
||
return
|
||
}
|
||
|
||
if tpl.Name != "MusicArticle" {
|
||
return
|
||
}
|
||
|
||
opts := getWikiStringOptions(entry.MainTitle, true)
|
||
|
||
//var discLengths []int
|
||
//var discTrackNumbers []int
|
||
|
||
var val []interface{}
|
||
var stringVal []string
|
||
|
||
if val, ok = tpl.Parameters["titleen"]; ok {
|
||
if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
|
||
entry.Titles["english"] = stringVal[0]
|
||
}
|
||
}
|
||
|
||
if val, ok = tpl.Parameters["titlejp"]; ok {
|
||
if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
|
||
entry.Titles["japanese"] = stringVal[0]
|
||
}
|
||
}
|
||
|
||
if val, ok = tpl.Parameters["titlejprom"]; ok {
|
||
if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
|
||
entry.Titles["romaji"] = stringVal[0]
|
||
}
|
||
}
|
||
|
||
if val, ok = tpl.Parameters["catalogno"]; ok {
|
||
if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
|
||
for _, catnos := range stringVal {
|
||
for _, catno := range strings.Split(catnos, " ") {
|
||
if strings.ToLower(catno) != "n/a" {
|
||
catno = strings.ReplaceAll(catno, "(", "")
|
||
catno = strings.ReplaceAll(catno, ")", "")
|
||
entry.CatalogNumbers = append(entry.CatalogNumbers, catno)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if val, ok = tpl.Parameters["genre"]; ok {
|
||
if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
|
||
for _, genre := range strings.Split(strings.Join(stringVal, " "), ",") {
|
||
entry.Genre = append(entry.Genre, strings.ToLower(strings.TrimSpace(genre)))
|
||
}
|
||
}
|
||
}
|
||
|
||
if val, ok = tpl.Parameters["image"]; ok && len(val) > 0 {
|
||
entry.Image = strings.TrimSpace(strings.Join(getStringValue(val, opts), ""))
|
||
}
|
||
if val, ok = tpl.Parameters["group"]; ok {
|
||
entry.Artists = append(entry.Artists, getArtistEntries("group", val, opts)...)
|
||
}
|
||
if val, ok = tpl.Parameters["masterer"]; ok {
|
||
entry.Artists = append(entry.Artists, getArtistEntries("mastering", val, opts)...)
|
||
}
|
||
if val, ok = tpl.Parameters["illustrator"]; ok {
|
||
entry.Artists = append(entry.Artists, getArtistEntries("illustration", val, opts)...)
|
||
}
|
||
if val, ok = tpl.Parameters["arranger"]; ok {
|
||
entry.Artists = append(entry.Artists, getArtistEntries("arranger", val, opts)...)
|
||
}
|
||
if val, ok = tpl.Parameters["lyricist"]; ok {
|
||
entry.Artists = append(entry.Artists, getArtistEntries("lyrics", val, opts)...)
|
||
}
|
||
if val, ok = tpl.Parameters["vocalist"]; ok {
|
||
entry.Artists = append(entry.Artists, getArtistEntries("vocals", val, opts)...)
|
||
}
|
||
if val, ok = tpl.Parameters["producer"]; ok {
|
||
entry.Artists = append(entry.Artists, getArtistEntries("producer", val, opts)...)
|
||
}
|
||
if val, ok = tpl.Parameters["designer"]; ok {
|
||
entry.Artists = append(entry.Artists, getArtistEntries("design", val, opts)...)
|
||
}
|
||
if val, ok = tpl.Parameters["other"]; ok {
|
||
entry.Artists = append(entry.Artists, getArtistEntries("other", val, opts)...)
|
||
}
|
||
if val, ok = tpl.Parameters["convention"]; ok && len(val) > 0 {
|
||
value := getStringValue(val, opts)
|
||
if len(value) > 0 {
|
||
entry.ReleaseEvent = value[0]
|
||
}
|
||
}
|
||
|
||
if val, ok = tpl.Parameters["released"]; ok {
|
||
if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
|
||
if releaseDate, err := time.ParseInLocation("2006-01-02", stringVal[0], time.UTC); err == nil {
|
||
entry.ReleaseDate = JSONTime{Time: releaseDate}
|
||
}
|
||
}
|
||
}
|
||
|
||
if val, ok = tpl.Parameters["website"]; ok {
|
||
for _, value := range val {
|
||
if linkVal, ok := value.(*wikitext_parser.Link); ok && linkVal.IsExternal {
|
||
entry.Links = append(entry.Links, linkVal.URL)
|
||
}
|
||
}
|
||
}
|
||
|
||
if val, ok = tpl.Parameters["digital"]; ok {
|
||
for _, value := range val {
|
||
if linkVal, ok := value.(*wikitext_parser.Link); ok && linkVal.IsExternal {
|
||
entry.Links = append(entry.Links, linkVal.URL)
|
||
}
|
||
}
|
||
}
|
||
|
||
if val, ok = tpl.Parameters["tracklist"]; ok {
|
||
var disc discEntry
|
||
createDisc := func() {
|
||
if len(disc.Tracks) > 0 {
|
||
disc.TrackCount = len(disc.Tracks)
|
||
entry.Duration += disc.Duration
|
||
entry.TrackCount += disc.TrackCount
|
||
entry.Discs = append(entry.Discs, disc)
|
||
}
|
||
}
|
||
for _, listEntry := range val {
|
||
if strVal, ok := listEntry.(string); ok && len(strVal) > 3 && strVal[0:3] == "===" {
|
||
//title header
|
||
if disc.Name != "" {
|
||
createDisc()
|
||
}
|
||
disc.Name = strings.Trim(strVal, "= ")
|
||
} else if listVal, ok := listEntry.(*wikitext_parser.UnorderedList); ok && len(listVal.Entries) > 0 && len(listVal.Entries) > 0 {
|
||
if sliceVal, ok := listVal.Entries[0].([]interface{}); ok && len(sliceVal) > 0 {
|
||
|
||
if trackTpl, ok := sliceVal[0].(*wikitext_parser.Template); ok && strings.ToUpper(trackTpl.Name) == "TRACK" && len(trackTpl.Parameters) >= 3 {
|
||
track := trackEntry{
|
||
Titles: make(map[string]string),
|
||
}
|
||
|
||
if mainTitleValue := getStringValue(trackTpl.Parameters["1"], opts); len(mainTitleValue) > 0 {
|
||
track.MainTitle = strings.Join(mainTitleValue, "")
|
||
track.Titles["original"] = track.MainTitle
|
||
}
|
||
if durations := getStringValue(trackTpl.Parameters["2"], opts); len(durations) > 0 {
|
||
split := strings.Split(durations[0], ":")
|
||
numbers := make([]int, len(split))
|
||
for i, numVal := range split {
|
||
numbers[i], _ = strconv.Atoi(numVal)
|
||
}
|
||
|
||
var duration int
|
||
if len(numbers) == 3 {
|
||
duration = 3600*numbers[0] + 60*numbers[1] + numbers[2]
|
||
} else if len(numbers) == 2 {
|
||
duration = 60*numbers[0] + numbers[1]
|
||
} else if len(numbers) == 1 {
|
||
duration = numbers[0]
|
||
}
|
||
track.Duration = duration
|
||
}
|
||
|
||
if lyrics, ok := trackTpl.Parameters["lyrics"]; ok {
|
||
if stringVal = getStringValue(lyrics, opts); ok && len(stringVal) > 0 {
|
||
track.Lyrics = wikitext_parser.NormalizeWikiTitle(strings.TrimSpace(strings.TrimPrefix(stringVal[0], "Lyrics:")))
|
||
}
|
||
}
|
||
|
||
if len(listVal.Entries) > 1 {
|
||
if extraListData, ok := listVal.Entries[1].(*wikitext_parser.UnorderedList); ok && len(extraListData.Entries) > 0 {
|
||
for i, entryValue := range extraListData.Entries {
|
||
if descVal, ok := entryValue.([]interface{}); ok && len(descVal) > 0 {
|
||
|
||
keyValue := strings.Split(strings.Join(getStringValue(descVal, opts), " "), ":")
|
||
if len(keyValue) > 0 {
|
||
if i == 0 && len(keyValue[0]) > 2 && keyValue[0][0:2] == "''" {
|
||
track.Titles["english"] = strings.Trim(strings.Join(keyValue, " "), "'' ")
|
||
continue
|
||
}
|
||
keyEntry := strings.ToLower(keyValue[0])
|
||
var values []interface{}
|
||
for j, fullValue := range descVal {
|
||
if strVal, ok = fullValue.(string); j == 0 && ok {
|
||
values = append(values, strings.Join(strings.Split(strVal, ":")[1:], ":"))
|
||
} else {
|
||
values = append(values, fullValue)
|
||
}
|
||
}
|
||
switch keyEntry {
|
||
case "original arrangement":
|
||
track.Artists = append(track.Artists, getArtistEntries("original arranger", values, opts)...)
|
||
case "original title":
|
||
track.Original = append(track.Original, strings.TrimSpace(strings.Join(getStringValue(values, opts), " ")))
|
||
case "original album":
|
||
var clean []string
|
||
for _, albumValue := range getStringValue(values, opts) {
|
||
albumValue = strings.Trim(albumValue, " \t'\"")
|
||
if len(albumValue) > 0 {
|
||
clean = append(clean, albumValue)
|
||
}
|
||
}
|
||
track.Original = append(track.Original, "Album: "+strings.TrimSpace(strings.Join(clean, " ")))
|
||
case "lyrics", "vocals", "chorus", "arranger", "composer", "producer", "remix":
|
||
track.Artists = append(track.Artists, getArtistEntries(keyEntry, values, opts)...)
|
||
case "arrangement":
|
||
track.Artists = append(track.Artists, getArtistEntries("arranger", values, opts)...)
|
||
case "composition":
|
||
track.Artists = append(track.Artists, getArtistEntries("composer", values, opts)...)
|
||
case "promotional video":
|
||
for _, value := range values {
|
||
if linkVal, ok := value.(*wikitext_parser.Link); ok && linkVal.IsExternal {
|
||
track.Links = append(track.Links, linkVal.URL)
|
||
}
|
||
}
|
||
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
}
|
||
}
|
||
|
||
disc.Duration += track.Duration
|
||
disc.Tracks = append(disc.Tracks, track)
|
||
|
||
}
|
||
}
|
||
|
||
}
|
||
}
|
||
|
||
createDisc()
|
||
|
||
}
|
||
|
||
cdIndexLock.Lock()
|
||
cdIndex[entry.Id] = entry
|
||
for _, d := range entry.Discs {
|
||
if _, ok := discTracksLookup[d.TrackCount]; !ok {
|
||
discTracksLookup[d.TrackCount] = []*albumEntry{entry}
|
||
} else {
|
||
for _, val := range discTracksLookup[d.TrackCount] {
|
||
if val.Id == entry.Id {
|
||
goto exit2
|
||
}
|
||
}
|
||
discTracksLookup[d.TrackCount] = append(discTracksLookup[d.TrackCount], entry)
|
||
|
||
exit2:
|
||
}
|
||
}
|
||
|
||
if len(entry.CatalogNumbers) > 0 {
|
||
for _, catno := range entry.CatalogNumbers {
|
||
normalized := normalizeSearchTitle(catno)
|
||
if _, ok := albumTitleLookup[normalized]; !ok {
|
||
albumTitleLookup[normalized] = []*albumEntry{entry}
|
||
} else {
|
||
albumTitleLookup[normalized] = append(albumTitleLookup[normalized], entry)
|
||
}
|
||
}
|
||
}
|
||
for _, title := range entry.Titles {
|
||
normalized := normalizeSearchTitle(title)
|
||
if _, ok := albumTitleLookup[normalized]; !ok {
|
||
albumTitleLookup[normalized] = []*albumEntry{entry}
|
||
} else {
|
||
for _, val := range albumTitleLookup[normalized] {
|
||
if val.Id == entry.Id {
|
||
goto exit
|
||
}
|
||
}
|
||
albumTitleLookup[normalized] = append(albumTitleLookup[normalized], entry)
|
||
|
||
exit:
|
||
}
|
||
}
|
||
defer cdIndexLock.Unlock()
|
||
}(&albumEntry{
|
||
Id: v.PageId,
|
||
MainTitle: v.Title,
|
||
Type: kind,
|
||
Titles: make(map[string]string),
|
||
})
|
||
}
|
||
}
|
||
}
|
||
|
||
}
|
||
|
||
func processIndex(filePath string) {
|
||
|
||
var wg sync.WaitGroup
|
||
processIndexDirectory(filePath, path.Join(filePath, "pageindex", "Official_CDs"), "official", &wg)
|
||
processIndexDirectory(filePath, path.Join(filePath, "pageindex", "Arrangement_CDs"), "arrangement", &wg)
|
||
|
||
wg.Wait()
|
||
}
|
||
|
||
type Lyrics struct {
|
||
MainTitle string `json:"pagetitle"`
|
||
Titles []string `json:"titles"`
|
||
Links []string `json:"links,omitempty"`
|
||
Duration int `json:"duration,omitempty"`
|
||
Artists []artistEntry `json:"artists"`
|
||
|
||
Entries struct {
|
||
Kanji []string `json:"kanji,omitempty"`
|
||
Romaji []string `json:"romaji,omitempty"`
|
||
English []string `json:"english,omitempty"`
|
||
} `json:"entries"`
|
||
}
|
||
|
||
func parseLyrics(filePath, pageName string) (lyrics *Lyrics) {
|
||
if strings.Index(pageName, "/") != -1 {
|
||
return
|
||
}
|
||
|
||
var err error
|
||
var contents []byte
|
||
if contents, err = ioutil.ReadFile(path.Join(filePath, "pages_by_name", "Lyrics:_"+pageName+".wiki")); err != nil {
|
||
if contents, err = ioutil.ReadFile(path.Join(filePath, "pages_by_name", "Lyrics:"+pageName+".wiki")); err != nil {
|
||
if contents, err = ioutil.ReadFile(path.Join(filePath, "pages", pageName+".wiki")); err != nil {
|
||
return
|
||
}
|
||
}
|
||
}
|
||
|
||
result := wikitext_parser.ParseWikiText(string(contents))
|
||
|
||
if len(result) == 0 {
|
||
return
|
||
}
|
||
|
||
tpl, ok := result[0].(*wikitext_parser.Template)
|
||
if !ok {
|
||
return
|
||
}
|
||
|
||
if tpl.Name != "Lyrics" {
|
||
return
|
||
}
|
||
|
||
lyrics = &Lyrics{
|
||
MainTitle: strings.ReplaceAll(pageName, "_", " "),
|
||
}
|
||
|
||
opts := getWikiStringOptions(lyrics.MainTitle, true)
|
||
|
||
//var discLengths []int
|
||
//var discTrackNumbers []int
|
||
|
||
var val []interface{}
|
||
var stringVal []string
|
||
|
||
if val, ok = tpl.Parameters["titleen"]; ok {
|
||
if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
|
||
lyrics.Titles = append(lyrics.Titles, stringVal[0])
|
||
}
|
||
}
|
||
|
||
if val, ok = tpl.Parameters["titlejp"]; ok {
|
||
if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
|
||
lyrics.Titles = append(lyrics.Titles, stringVal[0])
|
||
}
|
||
}
|
||
|
||
if val, ok = tpl.Parameters["titlerom"]; ok {
|
||
if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
|
||
lyrics.Titles = append(lyrics.Titles, stringVal[0])
|
||
}
|
||
}
|
||
|
||
if val, ok = tpl.Parameters["group"]; ok {
|
||
lyrics.Artists = append(lyrics.Artists, getArtistEntries("group", val, opts)...)
|
||
}
|
||
if val, ok = tpl.Parameters["arranger"]; ok {
|
||
lyrics.Artists = append(lyrics.Artists, getArtistEntries("arranger", val, opts)...)
|
||
}
|
||
if val, ok = tpl.Parameters["lyricist"]; ok {
|
||
lyrics.Artists = append(lyrics.Artists, getArtistEntries("lyrics", val, opts)...)
|
||
}
|
||
if val, ok = tpl.Parameters["vocalist"]; ok {
|
||
lyrics.Artists = append(lyrics.Artists, getArtistEntries("vocals", val, opts)...)
|
||
}
|
||
|
||
if val, ok = tpl.Parameters["length"]; ok {
|
||
if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
|
||
split := strings.Split(stringVal[0], ":")
|
||
numbers := make([]int, len(split))
|
||
for i, numVal := range split {
|
||
numbers[i], _ = strconv.Atoi(numVal)
|
||
}
|
||
|
||
var duration int
|
||
if len(numbers) == 3 {
|
||
duration = 3600*numbers[0] + 60*numbers[1] + numbers[2]
|
||
} else if len(numbers) == 2 {
|
||
duration = 60*numbers[0] + numbers[1]
|
||
} else if len(numbers) == 1 {
|
||
duration = numbers[0]
|
||
}
|
||
|
||
lyrics.Duration = duration
|
||
}
|
||
}
|
||
|
||
noTrimOpts := getWikiStringOptions(lyrics.MainTitle, false)
|
||
|
||
var kan []interface{}
|
||
var okKan bool
|
||
var rom []interface{}
|
||
var okRom bool
|
||
var eng []interface{}
|
||
var okEng bool
|
||
for i := 1; ; i++ {
|
||
if kan, okKan = tpl.Parameters[fmt.Sprintf("kan%d", i)]; okKan {
|
||
lyrics.Entries.Kanji = append(lyrics.Entries.Kanji, normalizeStringCharacters(strings.Join(getStringValue(kan, noTrimOpts), "")))
|
||
}
|
||
if rom, okRom = tpl.Parameters[fmt.Sprintf("rom%d", i)]; okRom {
|
||
lyrics.Entries.Romaji = append(lyrics.Entries.Romaji, normalizeStringCharacters(strings.Join(getStringValue(rom, noTrimOpts), "")))
|
||
}
|
||
if eng, okEng = tpl.Parameters[fmt.Sprintf("eng%d", i)]; okEng {
|
||
lyrics.Entries.English = append(lyrics.Entries.English, normalizeStringCharacters(strings.Join(getStringValue(eng, noTrimOpts), "")))
|
||
}
|
||
|
||
if !okKan && !okRom && !okEng {
|
||
break
|
||
}
|
||
}
|
||
|
||
return
|
||
}
|
||
|
||
var normalizeSearchTitleTransformer = transform.Chain(
|
||
norm.NFKD,
|
||
//width.Narrow,
|
||
runes.Remove(runes.In(unicode.Cc)),
|
||
runes.Remove(runes.In(unicode.Cf)),
|
||
runes.Remove(runes.In(unicode.Mn)),
|
||
runes.Remove(runes.In(unicode.Me)),
|
||
runes.Remove(runes.In(unicode.Mc)),
|
||
runes.Remove(runes.In(unicode.Po)),
|
||
runes.Remove(runes.In(unicode.Pe)),
|
||
runes.Remove(runes.In(unicode.Ps)),
|
||
runes.Remove(runes.In(unicode.Pf)),
|
||
runes.Remove(runes.In(unicode.Pi)),
|
||
runes.Remove(runes.In(unicode.Pd)),
|
||
runes.Remove(runes.In(unicode.Pc)),
|
||
runes.Remove(runes.In(unicode.Sc)),
|
||
runes.Remove(runes.In(unicode.Sk)),
|
||
runes.Remove(runes.In(unicode.Sm)),
|
||
runes.Remove(runes.In(unicode.So)),
|
||
runes.Remove(runes.In(unicode.Space)),
|
||
cases.Lower(language.Und),
|
||
norm.NFC,
|
||
)
|
||
|
||
func normalizeSearchTitle(title string) (normalized string) {
|
||
normalized, _, _ = transform.String(normalizeSearchTitleTransformer, title)
|
||
return
|
||
}
|
||
|
||
func normalizeStringCharacters(text string) (normalized string) {
|
||
normalized = strings.TrimSpace(norm.NFKC.String(text))
|
||
|
||
return
|
||
}
|
||
|
||
type findByDurationResult struct {
|
||
albumEntry *albumEntry
|
||
cddb1 utilities.CDDB1
|
||
toc utilities.TOC
|
||
discIndex int
|
||
}
|
||
|
||
func findByTOC(toc utilities.TOC, trackThreshold, discThreshold int) (results []findByDurationResult) {
|
||
for _, result := range findByCDDB1(toc.GetCDDB1(), discThreshold) {
|
||
func() {
|
||
for i, track := range result.albumEntry.Discs[result.discIndex].Tracks {
|
||
diff := math.Abs(float64(track.Duration) - toc.GetTrackDuration(i).Seconds())
|
||
if diff > float64(trackThreshold) { //too much variation
|
||
return
|
||
}
|
||
}
|
||
results = append(results, result)
|
||
}()
|
||
}
|
||
|
||
return
|
||
}
|
||
|
||
func findByCDDB1(cddb1 utilities.CDDB1, discThreshold int) []findByDurationResult {
|
||
return findByTracksAndDuration(cddb1.GetTrackNumber(), int(cddb1.GetDuration().Seconds()), discThreshold)
|
||
}
|
||
|
||
func findByTracksAndDuration(tracks, duration, threshold int) (results []findByDurationResult) {
|
||
for _, r := range discTracksLookup[tracks] {
|
||
for i, d := range r.Discs {
|
||
|
||
diff := d.Duration - duration
|
||
if diff < 0 {
|
||
diff = -diff
|
||
}
|
||
|
||
if diff <= threshold {
|
||
results = append(results, findByDurationResult{
|
||
albumEntry: r,
|
||
discIndex: i,
|
||
})
|
||
}
|
||
}
|
||
}
|
||
|
||
return
|
||
}
|
||
|
||
func main() {
|
||
wd, _ := os.Getwd()
|
||
servePath := flag.String("path", wd, "Path that will be served by default")
|
||
flag.Parse()
|
||
|
||
processIndex(*servePath)
|
||
|
||
server := &http.Server{
|
||
Addr: ":8777",
|
||
Handler: http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||
if request.URL.Path == "/cddb" {
|
||
//cddb emulator
|
||
writer.Header().Set("Content-Type", "text/plain; charset=UTF-8")
|
||
writer.WriteHeader(http.StatusOK)
|
||
|
||
cmd := request.URL.Query().Get("cmd")
|
||
if len(cmd) > 4 && cmd[0:4] == "cddb" {
|
||
splits := strings.Split(cmd, " ")
|
||
if len(splits) < 3 {
|
||
writer.Write([]byte("500 Unrecognized command\n.\n"))
|
||
return
|
||
}
|
||
switch splits[1] {
|
||
case "hello":
|
||
writer.Write([]byte("200 hello and welcome\n.\n"))
|
||
return
|
||
|
||
case "query":
|
||
formatEntry := func(cddb1 utilities.CDDB1, result findByDurationResult) (out string) {
|
||
var group string
|
||
for _, a := range result.albumEntry.Artists {
|
||
if a.Position == "group" {
|
||
group = a.Names[0]
|
||
break
|
||
}
|
||
}
|
||
if len(result.albumEntry.CatalogNumbers) > result.discIndex {
|
||
out = fmt.Sprintf("%sSoundtrack%d_%d %s [%s] %s / %s", strings.ToUpper(result.albumEntry.Type[0:1])+result.albumEntry.Type[1:], result.albumEntry.Id, result.discIndex, cddb1.String(), result.albumEntry.CatalogNumbers[result.discIndex], group, result.albumEntry.MainTitle)
|
||
} else {
|
||
out = fmt.Sprintf("%sSoundtrack%d_%d %s %s / %s", strings.ToUpper(result.albumEntry.Type[0:1])+result.albumEntry.Type[1:], result.albumEntry.Id, result.discIndex, cddb1.String(), group, result.albumEntry.MainTitle)
|
||
}
|
||
|
||
if len(result.albumEntry.Discs) > 1 {
|
||
out += fmt.Sprintf(" (Disc %d)", result.discIndex+1)
|
||
}
|
||
return
|
||
}
|
||
|
||
cddb1 := utilities.NewCDDB1FromString(splits[2])
|
||
|
||
toc := utilities.NewTOCFromCDDBString(strings.Join(splits[3:], " "))
|
||
if cddb1 == 0 {
|
||
writer.Write([]byte("500 Command syntax error\n.\n"))
|
||
return
|
||
}
|
||
|
||
var entries []findByDurationResult
|
||
if toc != nil {
|
||
cddb1 = toc.GetCDDB1()
|
||
entries = findByTOC(toc, 3, 10)
|
||
} else {
|
||
entries = findByCDDB1(cddb1, 10)
|
||
}
|
||
|
||
if len(entries) == 0 {
|
||
writer.Write([]byte("202 No match found\n.\n"))
|
||
return
|
||
} else if len(entries) == 1 {
|
||
writer.Write([]byte(fmt.Sprintf("200 %s\n", formatEntry(cddb1, entries[0]))))
|
||
return
|
||
} else {
|
||
writer.Write([]byte("211 Found inexact matches list follows (until terminating marker `.')\n"))
|
||
for _, e := range entries {
|
||
writer.Write([]byte(fmt.Sprintf("%s\n", formatEntry(cddb1, e))))
|
||
}
|
||
writer.Write([]byte(".\n"))
|
||
return
|
||
}
|
||
case "read":
|
||
if len(splits) < 4 {
|
||
writer.Write([]byte("500 Command syntax error\n.\n"))
|
||
return
|
||
}
|
||
cddb1 := utilities.NewCDDB1FromString(splits[3])
|
||
if cddb1 == 0 {
|
||
writer.Write([]byte("500 Command syntax error\n.\n"))
|
||
return
|
||
}
|
||
firstDigit := strings.IndexFunc(splits[2], unicode.IsNumber)
|
||
if firstDigit == -1 {
|
||
writer.Write([]byte("500 Command syntax error\n.\n"))
|
||
return
|
||
}
|
||
|
||
params := strings.Split(splits[2][firstDigit:], "_")
|
||
pageid, err := strconv.Atoi(params[0])
|
||
var discIndex int
|
||
if err != nil {
|
||
writer.Write([]byte("500 Command syntax error\n.\n"))
|
||
return
|
||
}
|
||
|
||
if len(params) > 1 {
|
||
discIndex, err = strconv.Atoi(params[1])
|
||
if err != nil {
|
||
writer.Write([]byte("500 Command syntax error\n.\n"))
|
||
return
|
||
}
|
||
}
|
||
|
||
entry, ok := cdIndex[pageid]
|
||
if !ok || len(entry.Discs) <= discIndex {
|
||
writer.Write([]byte("401 Entry not found\n.\n"))
|
||
return
|
||
}
|
||
|
||
var group string
|
||
for _, a := range entry.Artists {
|
||
if a.Position == "group" {
|
||
group = a.Names[0]
|
||
break
|
||
}
|
||
}
|
||
writer.Write([]byte(fmt.Sprintf("210 %sSoundtrack %s\n", strings.ToUpper(entry.Type[0:1])+entry.Type[1:], cddb1.String())))
|
||
writer.Write([]byte("# xmcd\n# Track frame offsets:\n"))
|
||
for i := 0; i < entry.Discs[discIndex].TrackCount; i++ {
|
||
writer.Write([]byte("#\t0\n"))
|
||
}
|
||
writer.Write([]byte(fmt.Sprintf("# Disc length: %d seconds\n#\n", entry.Duration)))
|
||
writer.Write([]byte(fmt.Sprintf("DISCID=%s\n", cddb1.String())))
|
||
writer.Write([]byte(fmt.Sprintf("DNUM=%d\n", len(entry.Discs))))
|
||
writer.Write([]byte(fmt.Sprintf("DINDEX=%d\n", discIndex+1)))
|
||
if len(entry.CatalogNumbers) > discIndex {
|
||
writer.Write([]byte(fmt.Sprintf("DTITLE=%s / [%s] %s\n", group, entry.CatalogNumbers[discIndex], entry.MainTitle)))
|
||
} else {
|
||
writer.Write([]byte(fmt.Sprintf("DTITLE=%s / %s\n", group, entry.MainTitle)))
|
||
}
|
||
if !entry.ReleaseDate.IsZero() {
|
||
writer.Write([]byte(fmt.Sprintf("DYEAR=%d\n", entry.ReleaseDate.Year())))
|
||
}
|
||
|
||
var genres []string
|
||
if entry.Type == "arrangement" {
|
||
genres = append(genres, entry.Type)
|
||
}
|
||
genres = append(genres, entry.Genre...)
|
||
if len(genres) == 0 {
|
||
genres = append(genres, "soundtrack")
|
||
}
|
||
writer.Write([]byte(fmt.Sprintf("DGENRE=%s\n", strings.Join(genres, ", "))))
|
||
|
||
for i, t := range entry.Discs[discIndex].Tracks {
|
||
writer.Write([]byte(fmt.Sprintf("TTITLE%d=%s\n", i, t.MainTitle)))
|
||
}
|
||
|
||
writer.Write([]byte(fmt.Sprintf("EXTD=https://en.touhouwiki.net/index.php?curid=%d\n", entry.Id)))
|
||
|
||
for i, t := range entry.Discs[discIndex].Tracks {
|
||
result := make(map[string][]string)
|
||
for _, a := range t.Artists {
|
||
if _, ok := result[a.Position]; !ok {
|
||
result[a.Position] = []string{}
|
||
}
|
||
result[a.Position] = append(result[a.Position], strings.Join(a.Names, "/"))
|
||
}
|
||
var resultLines []string
|
||
for k, v := range result {
|
||
resultLines = append(resultLines, k+": "+strings.Join(v, ", "))
|
||
}
|
||
sort.SliceStable(resultLines, func(i, j int) bool {
|
||
return resultLines[i] < resultLines[j]
|
||
})
|
||
writer.Write([]byte(fmt.Sprintf("EXTT%d=%s\n", i, strings.Join(resultLines, "\\n"))))
|
||
}
|
||
|
||
writer.Write([]byte("PLAYORDER=\n.\n"))
|
||
return
|
||
}
|
||
}
|
||
writer.Write([]byte("500 Unrecognized command\n.\n"))
|
||
} else if request.URL.Path == "/search" {
|
||
writer.Header().Set("Content-Type", "application/json")
|
||
|
||
switch request.URL.Query().Get("type") {
|
||
case "album": //search by title or catalog number, "exact" after normalization
|
||
entries, ok := albumTitleLookup[normalizeSearchTitle(request.URL.Query().Get("query"))]
|
||
|
||
if !ok {
|
||
writer.Write([]byte("[]"))
|
||
} else {
|
||
jsonBytes, _ := json.Marshal(entries)
|
||
|
||
writer.Write(jsonBytes)
|
||
}
|
||
case "loosealbum": //search by title or catalog number, loosely
|
||
|
||
var entries []*albumEntry
|
||
normalized := normalizeSearchTitle(request.URL.Query().Get("query"))
|
||
for k, v := range albumTitleLookup {
|
||
if strings.Index(k, normalized) != -1 {
|
||
entries = append(entries, v...)
|
||
}
|
||
}
|
||
|
||
sort.SliceStable(entries, func(i, j int) bool {
|
||
return entries[i].Id < entries[j].Id
|
||
})
|
||
|
||
jsonBytes, _ := json.Marshal(entries)
|
||
|
||
writer.Write(jsonBytes)
|
||
default:
|
||
writer.WriteHeader(http.StatusNotFound)
|
||
writer.Write([]byte("[]"))
|
||
}
|
||
|
||
} else if strings.Index(request.URL.Path, "/lyrics/") == 0 {
|
||
writer.Header().Set("Content-Type", "application/json")
|
||
|
||
lyrics := parseLyrics(*servePath, wikitext_parser.NormalizeWikiTitle(strings.TrimPrefix(request.URL.Path, "/lyrics/")))
|
||
if lyrics == nil {
|
||
|
||
writer.WriteHeader(http.StatusNotFound)
|
||
writer.Write([]byte("{}"))
|
||
return
|
||
}
|
||
|
||
jsonBytes, _ := json.Marshal(lyrics)
|
||
|
||
writer.Write(jsonBytes)
|
||
} else if strings.Index(request.URL.Path, "/album/") == 0 {
|
||
writer.Header().Set("Content-Type", "application/json")
|
||
|
||
if pageid, err := strconv.Atoi(strings.TrimPrefix(request.URL.Path, "/album/")); err == nil {
|
||
if album, ok := cdIndex[pageid]; ok {
|
||
jsonBytes, _ := json.Marshal(album)
|
||
|
||
writer.Write(jsonBytes)
|
||
return
|
||
}
|
||
}
|
||
|
||
writer.WriteHeader(http.StatusNotFound)
|
||
writer.Write([]byte("{}"))
|
||
return
|
||
} else {
|
||
filePath := path.Join(*servePath, strings.TrimLeft(request.URL.Path, "/"))
|
||
if request.URL.Path == "/" {
|
||
filePath = path.Join(*servePath, "README.md")
|
||
}
|
||
|
||
stat, err := os.Stat(filePath)
|
||
if err != nil {
|
||
http.NotFound(writer, request)
|
||
return
|
||
}
|
||
|
||
if stat.IsDir() {
|
||
writer.Header().Set("Content-Type", "application/json")
|
||
type dirEntry struct {
|
||
Name string `json:"name"`
|
||
Size int64 `json:"size"`
|
||
Mtime int64 `json:"mtime"`
|
||
Link string `json:"link,omitempty"`
|
||
Type string `json:"type"`
|
||
}
|
||
dirEntries := make([]dirEntry, 0, 1024)
|
||
|
||
entries, _ := ioutil.ReadDir(filePath)
|
||
for _, e := range entries {
|
||
entry := dirEntry{
|
||
Name: e.Name(),
|
||
Size: e.Size(),
|
||
Mtime: e.ModTime().UTC().Unix(),
|
||
}
|
||
if e.Mode()&fs.ModeSymlink > 0 {
|
||
entry.Link, _ = os.Readlink(path.Join(filePath, entry.Name))
|
||
entry.Type = "l"
|
||
} else if e.IsDir() {
|
||
entry.Type = "d"
|
||
} else {
|
||
entry.Type = "f"
|
||
}
|
||
dirEntries = append(dirEntries, entry)
|
||
}
|
||
|
||
writer.WriteHeader(http.StatusOK)
|
||
byteEntries, _ := json.Marshal(dirEntries)
|
||
writer.Write(byteEntries)
|
||
} else {
|
||
if path.Ext(filePath) == ".wiki" {
|
||
writer.Header().Set("Content-Type", "text/plain")
|
||
} else if path.Ext(filePath) == ".md" {
|
||
writer.Header().Set("Content-Type", "text/markdown")
|
||
}
|
||
http.ServeFile(writer, request, filePath)
|
||
}
|
||
}
|
||
}),
|
||
}
|
||
err := server.ListenAndServe()
|
||
if err != nil {
|
||
log.Panic(err)
|
||
}
|
||
}
|