package main import ( "encoding/json" "flag" "fmt" "git.gammaspectra.live/S.O.N.G/touhouwiki-mirror/utilities" wikitext_parser "git.gammaspectra.live/S.O.N.G/wikitext-parser" "golang.org/x/text/cases" "golang.org/x/text/language" "golang.org/x/text/runes" "golang.org/x/text/transform" "golang.org/x/text/unicode/norm" "io/fs" "io/ioutil" "log" "math" "net/http" "os" "path" "sort" "strconv" "strings" "sync" "time" "unicode" ) var cdIndex = make(map[int]*albumEntry) var cdIndexLock sync.Mutex var albumTitleLookup = make(map[string][]*albumEntry) //TODO: make this work with discs var discTracksLookup = make(map[int][]*albumEntry) type categoryPageIndex struct { PageId int `json:"pageid"` Namespace int `json:"ns"` Title string `json:"title"` Timestamp string `json:"timestamp"` } func parseCategoryPageIndex(filePath string) []categoryPageIndex { s := &struct { Query struct { Members []categoryPageIndex `json:"categorymembers"` } `json:"query"` }{} fileData, err := ioutil.ReadFile(filePath) if err != nil { return nil } err = json.Unmarshal(fileData, s) if err != nil { return nil } return s.Query.Members } type JSONTime struct { time.Time } func (t JSONTime) MarshalJSON() ([]byte, error) { if t.IsZero() { return nil, nil } return []byte(fmt.Sprintf("\"%s\"", t.Format("2006-01-02"))), nil } type albumEntry struct { Id int `json:"pageid"` Type string `json:"type"` MainTitle string `json:"pagetitle"` Titles map[string]string `json:"titles"` Image string `json:"image,omitempty"` CatalogNumbers []string `json:"catalognumbers,omitempty"` Genre []string `json:"genre,omitempty"` TrackCount int `json:"trackcount,omitempty"` Duration int `json:"duration,omitempty"` ReleaseDate JSONTime `json:"releasedate,omitempty"` ReleaseEvent string `json:"releaseevent,omitempty"` Links []string `json:"links,omitempty"` Artists []artistEntry `json:"artists,omitempty"` Discs []discEntry `json:"discs,omitempty"` } type discEntry struct { Name string `json:"name,omitempty"` Duration int `json:"duration,omitempty"` TrackCount int `json:"trackcount,omitempty"` Tracks []trackEntry `json:"tracks"` } type trackEntry struct { Duration int `json:"duration,omitempty"` MainTitle string `json:"title,omitempty"` Titles map[string]string `json:"titles,omitempty"` Artists []artistEntry `json:"artists,omitempty"` Lyrics string `json:"lyrics,omitempty"` Links []string `json:"links,omitempty"` Original []string `json:"original,omitempty"` } type artistEntry struct { Position string `json:"position"` Names []string `json:"names"` } func getArtistEntries(kind string, entries []interface{}, opts *wikitext_parser.WikiStringValueOptions) (artists []artistEntry) { artist := artistEntry{ Position: kind, } recreateArtist := func(kinds ...string) { var names []string for _, n := range artist.Names { //TODO //n = strings.Trim(n, " ()[]（）") if opts.Trim { n = strings.TrimSpace(n) } if len(n) > 0 { var curName string if len(names) > 0 { curName = names[len(names)-1] } if len(curName) > 0 && curName[len(curName)-1] == '(' { names[len(names)-1] += n } else if len(curName) > 0 && len(n) > 1 && n[len(n)-1] == ')' && strings.Index(n, "(") <= 0 { names[len(names)-1] += " " + n } else if len(curName) > 0 && n[0] == '(' && (strings.Index(n, ")") == -1 || n[len(n)-1] == ')') { names[len(names)-1] += " " + n } else if len(curName) > 0 && n == ")" { names[len(names)-1] += n } else if len(curName) > 0 && curName[len(curName)-1] == '/' { names[len(names)-1] += n } else if len(curName) > 0 && n == "/" { names[len(names)-1] += n } else { names = append(names, n) } } } artist.Names = names if len(artist.Names) > 0 { artists = append(artists, artist) artist = artistEntry{ Position: strings.Join(kinds, ", "), } } } for _, value := range entries { if text, ok := value.(string); ok { if i := strings.Index(text, " & "); i != -1 { for _, vv := range strings.Split(text, " & ") { vv = normalizeStringCharacters(vv) if len(vv) > 0 { artist.Names = append(artist.Names, vv) } recreateArtist(kind) } } else if i := strings.Index(text, "& "); i == 0 { for _, vv := range strings.Split(text, "& ") { vv = normalizeStringCharacters(vv) if len(vv) > 0 { artist.Names = append(artist.Names, vv) } recreateArtist(kind) } } else if i := strings.Index(text, " &"); i != -1 && i == len(text)-2 { for _, vv := range strings.Split(text, " &") { vv = normalizeStringCharacters(vv) if len(vv) > 0 { artist.Names = append(artist.Names, vv) } recreateArtist(kind) } } else if i := strings.Index(text, " and "); i != -1 { for _, vv := range strings.Split(text, " and ") { vv = normalizeStringCharacters(vv) if len(vv) > 0 { artist.Names = append(artist.Names, vv) } recreateArtist(kind) } } else if i := strings.Index(text, "and "); i == 0 { for _, vv := range strings.Split(text, "and ") { vv = normalizeStringCharacters(vv) if len(vv) > 0 { artist.Names = append(artist.Names, vv) } recreateArtist(kind) } } else if i := strings.Index(text, " and"); i != -1 && i == len(text)-4 { for _, vv := range strings.Split(text, " and") { vv = normalizeStringCharacters(vv) if len(vv) > 0 { artist.Names = append(artist.Names, vv) } recreateArtist(kind) } } else { for _, vv := range strings.Split(text, " and") { vv = normalizeStringCharacters(vv) if len(vv) > 0 { artist.Names = append(artist.Names, vv) } } } } else if _, ok := value.(wikitext_parser.NewLineToken); ok { recreateArtist(kind) } else if tpl, ok := value.(*wikitext_parser.Template); ok { if tpl.IsLink { var result []string for _, vv := range tpl.Parameters { result = append(result, getStringValue(vv, opts)...) } if len(result) == 0 { result = append(result, tpl.Name) } artist.Names = append(artist.Names, strings.Join(result, " ")) } else { artist.Names = append(artist.Names, strings.Join(getStringValue([]interface{}{tpl}, opts), " ")) } } else if link, ok := value.(*wikitext_parser.Link); ok { if len(link.Name) > 0 { artist.Names = append(artist.Names, strings.Join(getStringValue(link.Name, opts), " ")) } else if !link.IsExternal { artist.Names = append(artist.Names, link.URL) } } else if unorderedList, ok := value.(*wikitext_parser.UnorderedList); ok { for _, val := range unorderedList.Entries { recreateArtist(kind) if text, ok := val.(string); ok { artist.Names = append(artist.Names, text) } else if tpl, ok := val.(*wikitext_parser.Template); ok { if tpl.IsLink { artist.Names = append(artist.Names, tpl.Name) for _, val := range tpl.Parameters { artist.Names = append(artist.Names, getStringValue(val, opts)...) } } else { artist.Names = append(artist.Names, getStringValue([]interface{}{tpl}, opts)...) } } } recreateArtist(kind) } else if descriptionList, ok := value.(*wikitext_parser.DescriptionList); ok { for _, val := range descriptionList.Entries { recreateArtist(kind, strings.Join(getStringValue(descriptionList.Name, opts), ", ")) if text, ok := val.(string); ok { artist.Names = append(artist.Names, text) } else if tpl, ok := val.(*wikitext_parser.Template); ok { if tpl.IsLink { artist.Names = append(artist.Names, tpl.Name) for _, val := range tpl.Parameters { artist.Names = append(artist.Names, getStringValue(val, opts)...) } } else { artist.Names = append(artist.Names, getStringValue([]interface{}{tpl}, opts)...) } } recreateArtist(kind) } } } recreateArtist(kind) return } func getWikiStringOptions(title string, trim bool) *wikitext_parser.WikiStringValueOptions { opts := &wikitext_parser.WikiStringValueOptions{} opts.Default() opts.PageName = title opts.TemplateHandler = func(template *wikitext_parser.Template, opt *wikitext_parser.WikiStringValueOptions) (result []string) { switch strings.ToUpper(template.Name) { case "H:TITLE": if val, ok := template.Parameters["0"]; ok && len(val) > 0 { result = append(result, wikitext_parser.GetWikiStringValue(val, opt)[0]) } case "LANG": if val, ok := template.Parameters["1"]; ok && len(val) > 0 { result = append(result, wikitext_parser.GetWikiStringValue(val, opt)[0]) } case "GENRE": if val, ok := template.Parameters["0"]; ok && len(val) > 0 { result = append(result, wikitext_parser.GetWikiStringValue(val, opt)[0]) } case "MUSICARTICLE/DIVIDER": case "PAGENAME", "SUBPAGENAME": result = append(result, opt.PageName) default: result = append(result, template.Name) } return } opts.Trim = trim return opts } func getStringValue(v []interface{}, opt *wikitext_parser.WikiStringValueOptions) []string { return wikitext_parser.GetWikiStringValue(v, opt) } func processIndexDirectory(filePath, indexPath, kind string, wg *sync.WaitGroup) { entries, err := ioutil.ReadDir(indexPath) if err != nil { return } for _, e := range entries { if path.Ext(e.Name()) == ".json" { for _, v := range parseCategoryPageIndex(path.Join(indexPath, e.Name())) { wg.Add(1) go func(entry *albumEntry) { defer wg.Done() contents, err := ioutil.ReadFile(path.Join(filePath, "pages", fmt.Sprintf("%d.wiki", entry.Id))) if err != nil { return } result := wikitext_parser.ParseWikiText(string(contents)) if len(result) == 0 { return } tpl, ok := result[0].(*wikitext_parser.Template) if !ok { return } if tpl.Name != "MusicArticle" { return } opts := getWikiStringOptions(entry.MainTitle, true) //var discLengths []int //var discTrackNumbers []int var val []interface{} var stringVal []string if val, ok = tpl.Parameters["titleen"]; ok { if stringVal = getStringValue(val, opts); len(stringVal) > 0 { entry.Titles["english"] = stringVal[0] } } if val, ok = tpl.Parameters["titlejp"]; ok { if stringVal = getStringValue(val, opts); len(stringVal) > 0 { entry.Titles["japanese"] = stringVal[0] } } if val, ok = tpl.Parameters["titlejprom"]; ok { if stringVal = getStringValue(val, opts); len(stringVal) > 0 { entry.Titles["romaji"] = stringVal[0] } } if val, ok = tpl.Parameters["catalogno"]; ok { if stringVal = getStringValue(val, opts); len(stringVal) > 0 { for _, catnos := range stringVal { for _, catno := range strings.Split(catnos, " ") { if strings.ToLower(catno) != "n/a" { catno = strings.ReplaceAll(catno, "(", "") catno = strings.ReplaceAll(catno, ")", "") entry.CatalogNumbers = append(entry.CatalogNumbers, catno) } } } } } if val, ok = tpl.Parameters["genre"]; ok { if stringVal = getStringValue(val, opts); len(stringVal) > 0 { for _, genre := range strings.Split(strings.Join(stringVal, " "), ",") { entry.Genre = append(entry.Genre, strings.ToLower(strings.TrimSpace(genre))) } } } if val, ok = tpl.Parameters["image"]; ok && len(val) > 0 { entry.Image = strings.TrimSpace(strings.Join(getStringValue(val, opts), "")) } if val, ok = tpl.Parameters["group"]; ok { entry.Artists = append(entry.Artists, getArtistEntries("group", val, opts)...) } if val, ok = tpl.Parameters["masterer"]; ok { entry.Artists = append(entry.Artists, getArtistEntries("mastering", val, opts)...) } if val, ok = tpl.Parameters["illustrator"]; ok { entry.Artists = append(entry.Artists, getArtistEntries("illustration", val, opts)...) } if val, ok = tpl.Parameters["arranger"]; ok { entry.Artists = append(entry.Artists, getArtistEntries("arranger", val, opts)...) } if val, ok = tpl.Parameters["lyricist"]; ok { entry.Artists = append(entry.Artists, getArtistEntries("lyrics", val, opts)...) } if val, ok = tpl.Parameters["vocalist"]; ok { entry.Artists = append(entry.Artists, getArtistEntries("vocals", val, opts)...) } if val, ok = tpl.Parameters["producer"]; ok { entry.Artists = append(entry.Artists, getArtistEntries("producer", val, opts)...) } if val, ok = tpl.Parameters["designer"]; ok { entry.Artists = append(entry.Artists, getArtistEntries("design", val, opts)...) } if val, ok = tpl.Parameters["other"]; ok { entry.Artists = append(entry.Artists, getArtistEntries("other", val, opts)...) } if val, ok = tpl.Parameters["convention"]; ok && len(val) > 0 { value := getStringValue(val, opts) if len(value) > 0 { entry.ReleaseEvent = value[0] } } if val, ok = tpl.Parameters["released"]; ok { if stringVal = getStringValue(val, opts); len(stringVal) > 0 { if releaseDate, err := time.ParseInLocation("2006-01-02", stringVal[0], time.UTC); err == nil { entry.ReleaseDate = JSONTime{Time: releaseDate} } } } if val, ok = tpl.Parameters["website"]; ok { for _, value := range val { if linkVal, ok := value.(*wikitext_parser.Link); ok && linkVal.IsExternal { entry.Links = append(entry.Links, linkVal.URL) } } } if val, ok = tpl.Parameters["digital"]; ok { for _, value := range val { if linkVal, ok := value.(*wikitext_parser.Link); ok && linkVal.IsExternal { entry.Links = append(entry.Links, linkVal.URL) } } } if val, ok = tpl.Parameters["tracklist"]; ok { var disc discEntry createDisc := func() { if len(disc.Tracks) > 0 { disc.TrackCount = len(disc.Tracks) entry.Duration += disc.Duration entry.TrackCount += disc.TrackCount entry.Discs = append(entry.Discs, disc) } } for _, listEntry := range val { if strVal, ok := listEntry.(string); ok && len(strVal) > 3 && strVal[0:3] == "===" { //title header if disc.Name != "" { createDisc() } disc.Name = strings.Trim(strVal, "= ") } else if listVal, ok := listEntry.(*wikitext_parser.UnorderedList); ok && len(listVal.Entries) > 0 && len(listVal.Entries) > 0 { if sliceVal, ok := listVal.Entries[0].([]interface{}); ok && len(sliceVal) > 0 { if trackTpl, ok := sliceVal[0].(*wikitext_parser.Template); ok && strings.ToUpper(trackTpl.Name) == "TRACK" && len(trackTpl.Parameters) >= 3 { track := trackEntry{ Titles: make(map[string]string), } if mainTitleValue := getStringValue(trackTpl.Parameters["1"], opts); len(mainTitleValue) > 0 { track.MainTitle = strings.Join(mainTitleValue, "") track.Titles["original"] = track.MainTitle } if durations := getStringValue(trackTpl.Parameters["2"], opts); len(durations) > 0 { split := strings.Split(durations[0], ":") numbers := make([]int, len(split)) for i, numVal := range split { numbers[i], _ = strconv.Atoi(numVal) } var duration int if len(numbers) == 3 { duration = 3600*numbers[0] + 60*numbers[1] + numbers[2] } else if len(numbers) == 2 { duration = 60*numbers[0] + numbers[1] } else if len(numbers) == 1 { duration = numbers[0] } track.Duration = duration } if lyrics, ok := trackTpl.Parameters["lyrics"]; ok { if stringVal = getStringValue(lyrics, opts); ok && len(stringVal) > 0 { track.Lyrics = wikitext_parser.NormalizeWikiTitle(strings.TrimSpace(strings.TrimPrefix(stringVal[0], "Lyrics:"))) } } if len(listVal.Entries) > 1 { if extraListData, ok := listVal.Entries[1].(*wikitext_parser.UnorderedList); ok && len(extraListData.Entries) > 0 { for i, entryValue := range extraListData.Entries { if descVal, ok := entryValue.([]interface{}); ok && len(descVal) > 0 { keyValue := strings.Split(strings.Join(getStringValue(descVal, opts), " "), ":") if len(keyValue) > 0 { if i == 0 && len(keyValue[0]) > 2 && keyValue[0][0:2] == "''" { track.Titles["english"] = strings.Trim(strings.Join(keyValue, " "), "'' ") continue } keyEntry := strings.ToLower(keyValue[0]) var values []interface{} for j, fullValue := range descVal { if strVal, ok = fullValue.(string); j == 0 && ok { values = append(values, strings.Join(strings.Split(strVal, ":")[1:], ":")) } else { values = append(values, fullValue) } } switch keyEntry { case "original arrangement": track.Artists = append(track.Artists, getArtistEntries("original arranger", values, opts)...) case "original title": track.Original = append(track.Original, strings.TrimSpace(strings.Join(getStringValue(values, opts), " "))) case "original album": var clean []string for _, albumValue := range getStringValue(values, opts) { albumValue = strings.Trim(albumValue, " \t'\"") if len(albumValue) > 0 { clean = append(clean, albumValue) } } track.Original = append(track.Original, "Album: "+strings.TrimSpace(strings.Join(clean, " "))) case "lyrics", "vocals", "chorus", "arranger", "composer", "producer", "remix": track.Artists = append(track.Artists, getArtistEntries(keyEntry, values, opts)...) case "arrangement": track.Artists = append(track.Artists, getArtistEntries("arranger", values, opts)...) case "composition": track.Artists = append(track.Artists, getArtistEntries("composer", values, opts)...) case "promotional video": for _, value := range values { if linkVal, ok := value.(*wikitext_parser.Link); ok && linkVal.IsExternal { track.Links = append(track.Links, linkVal.URL) } } } } } } } } disc.Duration += track.Duration disc.Tracks = append(disc.Tracks, track) } } } } createDisc() } cdIndexLock.Lock() cdIndex[entry.Id] = entry for _, d := range entry.Discs { if _, ok := discTracksLookup[d.TrackCount]; !ok { discTracksLookup[d.TrackCount] = []*albumEntry{entry} } else { for _, val := range discTracksLookup[d.TrackCount] { if val.Id == entry.Id { goto exit2 } } discTracksLookup[d.TrackCount] = append(discTracksLookup[d.TrackCount], entry) exit2: } } if len(entry.CatalogNumbers) > 0 { for _, catno := range entry.CatalogNumbers { normalized := normalizeSearchTitle(catno) if _, ok := albumTitleLookup[normalized]; !ok { albumTitleLookup[normalized] = []*albumEntry{entry} } else { albumTitleLookup[normalized] = append(albumTitleLookup[normalized], entry) } } } for _, title := range entry.Titles { normalized := normalizeSearchTitle(title) if _, ok := albumTitleLookup[normalized]; !ok { albumTitleLookup[normalized] = []*albumEntry{entry} } else { for _, val := range albumTitleLookup[normalized] { if val.Id == entry.Id { goto exit } } albumTitleLookup[normalized] = append(albumTitleLookup[normalized], entry) exit: } } defer cdIndexLock.Unlock() }(&albumEntry{ Id: v.PageId, MainTitle: v.Title, Type: kind, Titles: make(map[string]string), }) } } } } func processIndex(filePath string) { var wg sync.WaitGroup processIndexDirectory(filePath, path.Join(filePath, "pageindex", "Official_CDs"), "official", &wg) processIndexDirectory(filePath, path.Join(filePath, "pageindex", "Arrangement_CDs"), "arrangement", &wg) wg.Wait() } type Lyrics struct { MainTitle string `json:"pagetitle"` Titles []string `json:"titles"` Links []string `json:"links,omitempty"` Duration int `json:"duration,omitempty"` Artists []artistEntry `json:"artists"` Entries struct { Kanji []string `json:"kanji,omitempty"` Romaji []string `json:"romaji,omitempty"` English []string `json:"english,omitempty"` } `json:"entries"` } func parseLyrics(filePath, pageName string) (lyrics *Lyrics) { if strings.Index(pageName, "/") != -1 { return } var err error var contents []byte if contents, err = ioutil.ReadFile(path.Join(filePath, "pages_by_name", "Lyrics:_"+pageName+".wiki")); err != nil { if contents, err = ioutil.ReadFile(path.Join(filePath, "pages_by_name", "Lyrics:"+pageName+".wiki")); err != nil { if contents, err = ioutil.ReadFile(path.Join(filePath, "pages", pageName+".wiki")); err != nil { return } } } result := wikitext_parser.ParseWikiText(string(contents)) if len(result) == 0 { return } tpl, ok := result[0].(*wikitext_parser.Template) if !ok { return } if tpl.Name != "Lyrics" { return } lyrics = &Lyrics{ MainTitle: strings.ReplaceAll(pageName, "_", " "), } opts := getWikiStringOptions(lyrics.MainTitle, true) //var discLengths []int //var discTrackNumbers []int var val []interface{} var stringVal []string if val, ok = tpl.Parameters["titleen"]; ok { if stringVal = getStringValue(val, opts); len(stringVal) > 0 { lyrics.Titles = append(lyrics.Titles, stringVal[0]) } } if val, ok = tpl.Parameters["titlejp"]; ok { if stringVal = getStringValue(val, opts); len(stringVal) > 0 { lyrics.Titles = append(lyrics.Titles, stringVal[0]) } } if val, ok = tpl.Parameters["titlerom"]; ok { if stringVal = getStringValue(val, opts); len(stringVal) > 0 { lyrics.Titles = append(lyrics.Titles, stringVal[0]) } } if val, ok = tpl.Parameters["group"]; ok { lyrics.Artists = append(lyrics.Artists, getArtistEntries("group", val, opts)...) } if val, ok = tpl.Parameters["arranger"]; ok { lyrics.Artists = append(lyrics.Artists, getArtistEntries("arranger", val, opts)...) } if val, ok = tpl.Parameters["lyricist"]; ok { lyrics.Artists = append(lyrics.Artists, getArtistEntries("lyrics", val, opts)...) } if val, ok = tpl.Parameters["vocalist"]; ok { lyrics.Artists = append(lyrics.Artists, getArtistEntries("vocals", val, opts)...) } if val, ok = tpl.Parameters["length"]; ok { if stringVal = getStringValue(val, opts); len(stringVal) > 0 { split := strings.Split(stringVal[0], ":") numbers := make([]int, len(split)) for i, numVal := range split { numbers[i], _ = strconv.Atoi(numVal) } var duration int if len(numbers) == 3 { duration = 3600*numbers[0] + 60*numbers[1] + numbers[2] } else if len(numbers) == 2 { duration = 60*numbers[0] + numbers[1] } else if len(numbers) == 1 { duration = numbers[0] } lyrics.Duration = duration } } noTrimOpts := getWikiStringOptions(lyrics.MainTitle, false) var kan []interface{} var okKan bool var rom []interface{} var okRom bool var eng []interface{} var okEng bool for i := 1; ; i++ { if kan, okKan = tpl.Parameters[fmt.Sprintf("kan%d", i)]; okKan { lyrics.Entries.Kanji = append(lyrics.Entries.Kanji, normalizeStringCharacters(strings.Join(getStringValue(kan, noTrimOpts), ""))) } if rom, okRom = tpl.Parameters[fmt.Sprintf("rom%d", i)]; okRom { lyrics.Entries.Romaji = append(lyrics.Entries.Romaji, normalizeStringCharacters(strings.Join(getStringValue(rom, noTrimOpts), ""))) } if eng, okEng = tpl.Parameters[fmt.Sprintf("eng%d", i)]; okEng { lyrics.Entries.English = append(lyrics.Entries.English, normalizeStringCharacters(strings.Join(getStringValue(eng, noTrimOpts), ""))) } if !okKan && !okRom && !okEng { break } } return } var normalizeSearchTitleTransformer = transform.Chain( norm.NFKD, //width.Narrow, runes.Remove(runes.In(unicode.Cc)), runes.Remove(runes.In(unicode.Cf)), runes.Remove(runes.In(unicode.Mn)), runes.Remove(runes.In(unicode.Me)), runes.Remove(runes.In(unicode.Mc)), runes.Remove(runes.In(unicode.Po)), runes.Remove(runes.In(unicode.Pe)), runes.Remove(runes.In(unicode.Ps)), runes.Remove(runes.In(unicode.Pf)), runes.Remove(runes.In(unicode.Pi)), runes.Remove(runes.In(unicode.Pd)), runes.Remove(runes.In(unicode.Pc)), runes.Remove(runes.In(unicode.Sc)), runes.Remove(runes.In(unicode.Sk)), runes.Remove(runes.In(unicode.Sm)), runes.Remove(runes.In(unicode.So)), runes.Remove(runes.In(unicode.Space)), cases.Lower(language.Und), norm.NFC, ) func normalizeSearchTitle(title string) (normalized string) { normalized, _, _ = transform.String(normalizeSearchTitleTransformer, title) return } func normalizeStringCharacters(text string) (normalized string) { normalized = strings.TrimSpace(norm.NFKC.String(text)) return } type findByDurationResult struct { albumEntry *albumEntry cddb1 utilities.CDDB1 toc utilities.TOC discIndex int } func findByTOC(toc utilities.TOC, trackThreshold, discThreshold int) (results []findByDurationResult) { for _, result := range findByCDDB1(toc.GetCDDB1(), discThreshold) { func() { for i, track := range result.albumEntry.Discs[result.discIndex].Tracks { diff := math.Abs(float64(track.Duration) - toc.GetTrackDuration(i).Seconds()) if diff > float64(trackThreshold) { //too much variation return } } results = append(results, result) }() } return } func findByCDDB1(cddb1 utilities.CDDB1, discThreshold int) []findByDurationResult { return findByTracksAndDuration(cddb1.GetTrackNumber(), int(cddb1.GetDuration().Seconds()), discThreshold) } func findByTracksAndDuration(tracks, duration, threshold int) (results []findByDurationResult) { for _, r := range discTracksLookup[tracks] { for i, d := range r.Discs { diff := d.Duration - duration if diff < 0 { diff = -diff } if diff <= threshold { results = append(results, findByDurationResult{ albumEntry: r, discIndex: i, }) } } } return } func main() { wd, _ := os.Getwd() servePath := flag.String("path", wd, "Path that will be served by default") flag.Parse() processIndex(*servePath) server := &http.Server{ Addr: ":8777", Handler: http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { if request.URL.Path == "/cddb" { //cddb emulator writer.Header().Set("Content-Type", "text/plain; charset=UTF-8") writer.WriteHeader(http.StatusOK) cmd := request.URL.Query().Get("cmd") if len(cmd) > 4 && cmd[0:4] == "cddb" { splits := strings.Split(cmd, " ") if len(splits) < 3 { writer.Write([]byte("500 Unrecognized command\n.\n")) return } switch splits[1] { case "hello": writer.Write([]byte("200 hello and welcome\n.\n")) return case "query": formatEntry := func(cddb1 utilities.CDDB1, result findByDurationResult) (out string) { var group string for _, a := range result.albumEntry.Artists { if a.Position == "group" { group = a.Names[0] break } } if len(result.albumEntry.CatalogNumbers) > result.discIndex { out = fmt.Sprintf("%sSoundtrack%d_%d %s [%s] %s / %s", strings.ToUpper(result.albumEntry.Type[0:1])+result.albumEntry.Type[1:], result.albumEntry.Id, result.discIndex, cddb1.String(), result.albumEntry.CatalogNumbers[result.discIndex], group, result.albumEntry.MainTitle) } else { out = fmt.Sprintf("%sSoundtrack%d_%d %s %s / %s", strings.ToUpper(result.albumEntry.Type[0:1])+result.albumEntry.Type[1:], result.albumEntry.Id, result.discIndex, cddb1.String(), group, result.albumEntry.MainTitle) } if len(result.albumEntry.Discs) > 1 { out += fmt.Sprintf(" (Disc %d)", result.discIndex+1) } return } cddb1 := utilities.NewCDDB1FromString(splits[2]) toc := utilities.NewTOCFromCDDBString(strings.Join(splits[3:], " ")) if cddb1 == 0 { writer.Write([]byte("500 Command syntax error\n.\n")) return } var entries []findByDurationResult if toc != nil { cddb1 = toc.GetCDDB1() entries = findByTOC(toc, 3, 10) } else { entries = findByCDDB1(cddb1, 10) } if len(entries) == 0 { writer.Write([]byte("202 No match found\n.\n")) return } else if len(entries) == 1 { writer.Write([]byte(fmt.Sprintf("200 %s\n", formatEntry(cddb1, entries[0])))) return } else { writer.Write([]byte("211 Found inexact matches list follows (until terminating marker `.')\n")) for _, e := range entries { writer.Write([]byte(fmt.Sprintf("%s\n", formatEntry(cddb1, e)))) } writer.Write([]byte(".\n")) return } case "read": if len(splits) < 4 { writer.Write([]byte("500 Command syntax error\n.\n")) return } cddb1 := utilities.NewCDDB1FromString(splits[3]) if cddb1 == 0 { writer.Write([]byte("500 Command syntax error\n.\n")) return } firstDigit := strings.IndexFunc(splits[2], unicode.IsNumber) if firstDigit == -1 { writer.Write([]byte("500 Command syntax error\n.\n")) return } params := strings.Split(splits[2][firstDigit:], "_") pageid, err := strconv.Atoi(params[0]) var discIndex int if err != nil { writer.Write([]byte("500 Command syntax error\n.\n")) return } if len(params) > 1 { discIndex, err = strconv.Atoi(params[1]) if err != nil { writer.Write([]byte("500 Command syntax error\n.\n")) return } } entry, ok := cdIndex[pageid] if !ok || len(entry.Discs) <= discIndex { writer.Write([]byte("401 Entry not found\n.\n")) return } var group string for _, a := range entry.Artists { if a.Position == "group" { group = a.Names[0] break } } writer.Write([]byte(fmt.Sprintf("210 %sSoundtrack %s\n", strings.ToUpper(entry.Type[0:1])+entry.Type[1:], cddb1.String()))) writer.Write([]byte("# xmcd\n# Track frame offsets:\n")) for i := 0; i < entry.Discs[discIndex].TrackCount; i++ { writer.Write([]byte("#\t0\n")) } writer.Write([]byte(fmt.Sprintf("# Disc length: %d seconds\n#\n", entry.Duration))) writer.Write([]byte(fmt.Sprintf("DISCID=%s\n", cddb1.String()))) writer.Write([]byte(fmt.Sprintf("DNUM=%d\n", len(entry.Discs)))) writer.Write([]byte(fmt.Sprintf("DINDEX=%d\n", discIndex+1))) if len(entry.CatalogNumbers) > discIndex { writer.Write([]byte(fmt.Sprintf("DTITLE=%s / [%s] %s\n", group, entry.CatalogNumbers[discIndex], entry.MainTitle))) } else { writer.Write([]byte(fmt.Sprintf("DTITLE=%s / %s\n", group, entry.MainTitle))) } if !entry.ReleaseDate.IsZero() { writer.Write([]byte(fmt.Sprintf("DYEAR=%d\n", entry.ReleaseDate.Year()))) } var genres []string if entry.Type == "arrangement" { genres = append(genres, entry.Type) } genres = append(genres, entry.Genre...) if len(genres) == 0 { genres = append(genres, "soundtrack") } writer.Write([]byte(fmt.Sprintf("DGENRE=%s\n", strings.Join(genres, ", ")))) for i, t := range entry.Discs[discIndex].Tracks { writer.Write([]byte(fmt.Sprintf("TTITLE%d=%s\n", i, t.MainTitle))) } writer.Write([]byte(fmt.Sprintf("EXTD=https://en.touhouwiki.net/index.php?curid=%d\n", entry.Id))) for i, t := range entry.Discs[discIndex].Tracks { result := make(map[string][]string) for _, a := range t.Artists { if _, ok := result[a.Position]; !ok { result[a.Position] = []string{} } result[a.Position] = append(result[a.Position], strings.Join(a.Names, "/")) } var resultLines []string for k, v := range result { resultLines = append(resultLines, k+": "+strings.Join(v, ", ")) } sort.SliceStable(resultLines, func(i, j int) bool { return resultLines[i] < resultLines[j] }) writer.Write([]byte(fmt.Sprintf("EXTT%d=%s\n", i, strings.Join(resultLines, "\\n")))) } writer.Write([]byte("PLAYORDER=\n.\n")) return } } writer.Write([]byte("500 Unrecognized command\n.\n")) } else if request.URL.Path == "/search" { writer.Header().Set("Content-Type", "application/json") switch request.URL.Query().Get("type") { case "album": //search by title or catalog number, "exact" after normalization entries, ok := albumTitleLookup[normalizeSearchTitle(request.URL.Query().Get("query"))] if !ok { writer.Write([]byte("[]")) } else { jsonBytes, _ := json.Marshal(entries) writer.Write(jsonBytes) } case "loosealbum": //search by title or catalog number, loosely var entries []*albumEntry normalized := normalizeSearchTitle(request.URL.Query().Get("query")) for k, v := range albumTitleLookup { if strings.Index(k, normalized) != -1 { entries = append(entries, v...) } } sort.SliceStable(entries, func(i, j int) bool { return entries[i].Id < entries[j].Id }) jsonBytes, _ := json.Marshal(entries) writer.Write(jsonBytes) default: writer.WriteHeader(http.StatusNotFound) writer.Write([]byte("[]")) } } else if strings.Index(request.URL.Path, "/lyrics/") == 0 { writer.Header().Set("Content-Type", "application/json") lyrics := parseLyrics(*servePath, wikitext_parser.NormalizeWikiTitle(strings.TrimPrefix(request.URL.Path, "/lyrics/"))) if lyrics == nil { writer.WriteHeader(http.StatusNotFound) writer.Write([]byte("{}")) return } jsonBytes, _ := json.Marshal(lyrics) writer.Write(jsonBytes) } else if strings.Index(request.URL.Path, "/album/") == 0 { writer.Header().Set("Content-Type", "application/json") if pageid, err := strconv.Atoi(strings.TrimPrefix(request.URL.Path, "/album/")); err == nil { if album, ok := cdIndex[pageid]; ok { jsonBytes, _ := json.Marshal(album) writer.Write(jsonBytes) return } } writer.WriteHeader(http.StatusNotFound) writer.Write([]byte("{}")) return } else { filePath := path.Join(*servePath, strings.TrimLeft(request.URL.Path, "/")) if request.URL.Path == "/" { filePath = path.Join(*servePath, "README.md") } stat, err := os.Stat(filePath) if err != nil { http.NotFound(writer, request) return } if stat.IsDir() { writer.Header().Set("Content-Type", "application/json") type dirEntry struct { Name string `json:"name"` Size int64 `json:"size"` Mtime int64 `json:"mtime"` Link string `json:"link,omitempty"` Type string `json:"type"` } dirEntries := make([]dirEntry, 0, 1024) entries, _ := ioutil.ReadDir(filePath) for _, e := range entries { entry := dirEntry{ Name: e.Name(), Size: e.Size(), Mtime: e.ModTime().UTC().Unix(), } if e.Mode()&fs.ModeSymlink > 0 { entry.Link, _ = os.Readlink(path.Join(filePath, entry.Name)) entry.Type = "l" } else if e.IsDir() { entry.Type = "d" } else { entry.Type = "f" } dirEntries = append(dirEntries, entry) } writer.WriteHeader(http.StatusOK) byteEntries, _ := json.Marshal(dirEntries) writer.Write(byteEntries) } else { if path.Ext(filePath) == ".wiki" { writer.Header().Set("Content-Type", "text/plain") } else if path.Ext(filePath) == ".md" { writer.Header().Set("Content-Type", "text/markdown") } http.ServeFile(writer, request, filePath) } } }), } err := server.ListenAndServe() if err != nil { log.Panic(err) } }