From 35b794b8a4085b6420f50aa8da460bc3d5c41a0e Mon Sep 17 00:00:00 2001 From: WeebDataHoarder <57538841+WeebDataHoarder@users.noreply.github.com> Date: Sun, 20 Feb 2022 16:20:11 +0100 Subject: [PATCH] Added lyrics support --- README.md | 2 + server.go | 189 ++++++++++++++++++++++++++++++++++++++----- wikiparser/parser.go | 14 ++-- 3 files changed, 179 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 4a820f367..36fa31dc4 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,8 @@ Small Golang server that loads all albums in an index at startup and is used to * Bare CDDB emulator for `query` and `read` commands. Used to query by track count + length of album. * query cmd ex. `/cddb?cmd=cddb+query+730dec08` (no need to provide TOC, but can be provided for more exact match) * read cmd ex. `/cddb?cmd=cddb+read+Soundtrack54742+730dec08` +* Fetch parsed track Lyrics entries + * ex. `/lyrics/Clockup_Flowers` # License diff --git a/server.go b/server.go index 23bdcd31e..8f3ab2bf7 100644 --- a/server.go +++ b/server.go @@ -275,20 +275,17 @@ func getArtistEntries(kind string, entries []interface{}) (artists []artistEntry return } -func getStringValue(pageName string, v []interface{}) (result []string) { +func getStringValue(pageName string, v []interface{}, trim ...bool) (result []string) { for _, value := range v { if text, ok := value.(string); ok { - text = normalizeStringCharacters(text) - if len(text) > 0 { - result = append(result, text) - } + result = append(result, text) } else if template, ok := value.(*wikiparser.Template); ok { if template.IsLink { output := 0 for _, vv := range template.Parameters { - for _, vvv := range getStringValue(pageName, vv) { + for _, vvv := range getStringValue(pageName, vv, trim...) { vvv = strings.TrimSpace(vvv) if len(vvv) > 0 { output++ @@ -303,15 +300,15 @@ func getStringValue(pageName string, v []interface{}) (result []string) { switch strings.ToUpper(template.Name) { case "H:TITLE": if val, ok := template.Parameters["0"]; ok && len(val) > 0 { - result = append(result, getStringValue(pageName, val)[0]) + result = append(result, getStringValue(pageName, val, trim...)[0]) } case "LANG": if val, ok := template.Parameters["1"]; ok && len(val) > 0 { - result = append(result, getStringValue(pageName, val)[0]) + result = append(result, getStringValue(pageName, val, trim...)[0]) } case "GENRE": if val, ok := template.Parameters["0"]; ok && len(val) > 0 { - result = append(result, getStringValue(pageName, val)[0]) + result = append(result, getStringValue(pageName, val, trim...)[0]) } case "PAGENAME": fallthrough @@ -322,24 +319,34 @@ func getStringValue(pageName string, v []interface{}) (result []string) { } } } else if html, ok := value.(*wikiparser.HTML); ok && html.Tag != nil { - str := strings.TrimSpace(html.Tag.String()) - if len(str) > 0 { - result = append(result, str) - } + result = append(result, html.Tag.String()) + } else if _, ok := value.(wikiparser.NewLineToken); ok { + result = append(result, "\n") } else if link, ok := value.(*wikiparser.Link); ok { if len(link.Name) > 0 { - result = append(result, getStringValue(pageName, link.Name)...) + result = append(result, getStringValue(pageName, link.Name, trim...)...) } else { result = append(result, link.URL) } - result = append(result, getStringValue(pageName, link.Name)...) + result = append(result, getStringValue(pageName, link.Name, trim...)...) } else if unorderedList, ok := value.(*wikiparser.UnorderedList); ok { - result = append(result, getStringValue(pageName, unorderedList.Entries)...) + result = append(result, getStringValue(pageName, unorderedList.Entries, trim...)...) } else if descriptionList, ok := value.(*wikiparser.DescriptionList); ok { - result = append(result, strings.Join(getStringValue("", descriptionList.Name), ", ")+": "+strings.Join(getStringValue(pageName, descriptionList.Entries), ", ")) + result = append(result, strings.Join(getStringValue("", descriptionList.Name, trim...), ", ")+": "+strings.Join(getStringValue(pageName, descriptionList.Entries, trim...), ", ")) } } + if len(trim) == 0 || trim[0] == true { + var newResults []string + for _, e := range result { + e = normalizeStringCharacters(e) + if len(e) > 0 { + newResults = append(newResults, e) + } + } + result = newResults + } + return } @@ -564,8 +571,9 @@ func processIndexDirectory(filePath, indexPath, kind string, wg *sync.WaitGroup) } if lyrics, ok := trackTpl.Parameters["lyrics"]; ok { - //TODO: parse lyrics - track.Lyrics = wikiparser.NormalizeWikiTitle(getStringValue(entry.MainTitle, lyrics)[0]) + if stringVal = getStringValue(entry.MainTitle, lyrics); ok && len(stringVal) > 0 { + track.Lyrics = wikiparser.NormalizeWikiTitle(strings.TrimSpace(strings.TrimPrefix(stringVal[0], "Lyrics:"))) + } } if len(listVal.Entries) > 1 { @@ -699,6 +707,135 @@ func processIndex(filePath string) { wg.Wait() } +type Lyrics struct { + MainTitle string `json:"pagetitle"` + Titles []string `json:"titles"` + Links []string `json:"links,omitempty"` + Duration int `json:"duration,omitempty"` + Artists []artistEntry `json:"artists"` + + Entries struct { + Kanji []string `json:"kanji,omitempty"` + Romaji []string `json:"romaji,omitempty"` + English []string `json:"english,omitempty"` + } `json:"entries"` +} + +func parseLyrics(filePath, pageName string) (lyrics *Lyrics) { + if strings.Index(pageName, "/") != -1 { + return + } + + var err error + var contents []byte + if contents, err = ioutil.ReadFile(path.Join(filePath, "pages_by_name", "Lyrics:_"+pageName+".wiki")); err != nil { + if contents, err = ioutil.ReadFile(path.Join(filePath, "pages_by_name", "Lyrics:"+pageName+".wiki")); err != nil { + return + } + } + + result := wikiparser.ParseWikiText(string(contents)) + + if len(result) == 0 { + return + } + + tpl, ok := result[0].(*wikiparser.Template) + if !ok { + return + } + + if tpl.Name != "Lyrics" { + return + } + + lyrics = &Lyrics{ + MainTitle: strings.ReplaceAll(pageName, "_", " "), + } + + //var discLengths []int + //var discTrackNumbers []int + + var val []interface{} + var stringVal []string + + if val, ok = tpl.Parameters["titleen"]; ok { + if stringVal = getStringValue(lyrics.MainTitle, val); len(stringVal) > 0 { + lyrics.Titles = append(lyrics.Titles, stringVal[0]) + } + } + + if val, ok = tpl.Parameters["titlejp"]; ok { + if stringVal = getStringValue(lyrics.MainTitle, val); len(stringVal) > 0 { + lyrics.Titles = append(lyrics.Titles, stringVal[0]) + } + } + + if val, ok = tpl.Parameters["titlerom"]; ok { + if stringVal = getStringValue(lyrics.MainTitle, val); len(stringVal) > 0 { + lyrics.Titles = append(lyrics.Titles, stringVal[0]) + } + } + + if val, ok = tpl.Parameters["group"]; ok { + lyrics.Artists = append(lyrics.Artists, getArtistEntries("group", val)...) + } + if val, ok = tpl.Parameters["arranger"]; ok { + lyrics.Artists = append(lyrics.Artists, getArtistEntries("arranger", val)...) + } + if val, ok = tpl.Parameters["lyricist"]; ok { + lyrics.Artists = append(lyrics.Artists, getArtistEntries("lyrics", val)...) + } + if val, ok = tpl.Parameters["vocalist"]; ok { + lyrics.Artists = append(lyrics.Artists, getArtistEntries("vocals", val)...) + } + + if val, ok = tpl.Parameters["length"]; ok { + if stringVal = getStringValue(lyrics.MainTitle, val); len(stringVal) > 0 { + split := strings.Split(stringVal[0], ":") + numbers := make([]int, len(split)) + for i, numVal := range split { + numbers[i], _ = strconv.Atoi(numVal) + } + + var duration int + if len(numbers) == 3 { + duration = 3600*numbers[0] + 60*numbers[1] + numbers[2] + } else if len(numbers) == 2 { + duration = 60*numbers[0] + numbers[1] + } else if len(numbers) == 1 { + duration = numbers[0] + } + + lyrics.Duration = duration + } + } + + var kan []interface{} + var okKan bool + var rom []interface{} + var okRom bool + var eng []interface{} + var okEng bool + for i := 1; ; i++ { + if kan, okKan = tpl.Parameters[fmt.Sprintf("kan%d", i)]; okKan { + lyrics.Entries.Kanji = append(lyrics.Entries.Kanji, normalizeStringCharacters(strings.Join(getStringValue(lyrics.MainTitle, kan, false), ""))) + } + if rom, okRom = tpl.Parameters[fmt.Sprintf("rom%d", i)]; okRom { + lyrics.Entries.Romaji = append(lyrics.Entries.Romaji, normalizeStringCharacters(strings.Join(getStringValue(lyrics.MainTitle, rom, false), ""))) + } + if eng, okEng = tpl.Parameters[fmt.Sprintf("eng%d", i)]; okEng { + lyrics.Entries.English = append(lyrics.Entries.English, normalizeStringCharacters(strings.Join(getStringValue(lyrics.MainTitle, eng, false), ""))) + } + + if !okKan && !okRom && !okEng { + break + } + } + + return +} + var normalizeSearchTitleTransformer = transform.Chain( norm.NFKD, //width.Narrow, @@ -998,6 +1135,20 @@ func main() { writer.Write([]byte("[]")) } + } else if strings.Index(request.URL.Path, "/lyrics/") == 0 { + writer.Header().Set("Content-Type", "application/json") + + lyrics := parseLyrics(*servePath, wikiparser.NormalizeWikiTitle(strings.Join(strings.Split(request.URL.Path, "/")[2:], "/"))) + if lyrics == nil { + + writer.WriteHeader(http.StatusNotFound) + writer.Write([]byte("{}")) + return + } + + jsonBytes, _ := json.Marshal(lyrics) + + writer.Write(jsonBytes) } else { filePath := path.Join(*servePath, strings.TrimLeft(request.URL.Path, "/")) if request.URL.Path == "/" { diff --git a/wikiparser/parser.go b/wikiparser/parser.go index c05bbb8b4..b9ad6d7bf 100644 --- a/wikiparser/parser.go +++ b/wikiparser/parser.go @@ -64,7 +64,7 @@ func ParseWikiText(text string) (result []interface{}) { if templateIndex == -1 && linkIndex == -1 { t := strings.TrimSpace(text[index:]) if len(t) > 0 { - result = append(result, t) + result = append(result, text[index:]) } break } else { @@ -79,7 +79,7 @@ func ParseWikiText(text string) (result []interface{}) { t := strings.TrimSpace(text[index : index+bestIndex]) if len(t) > 0 { - result = append(result, t) + result = append(result, text[index:index+bestIndex]) } var tpl *Template index, tpl = ParseTemplate(text, index+bestIndex+2, 0, text[index+bestIndex]) @@ -230,9 +230,9 @@ func ParseTemplate(text string, index int, depth int, startCharacter byte) (i in template = NewTemplate(t, startCharacter == '[') } else { if key == "" { - template.AddParameterUnkeyed(t) + template.AddParameterUnkeyed(text[lastToken:i]) } else { - template.AddParameter(key, t) + template.AddParameter(key, text[lastToken:i]) } } } @@ -385,7 +385,7 @@ func ParseUnorderedList(text string, index int, depth int, indent int, startChar if lastToken < len(text) && i-lastToken > 0 { t := strings.TrimSpace(text[lastToken:i]) if len(t) > 0 { - currentValue = append(currentValue, t) + currentValue = append(currentValue, text[lastToken:i]) } return len(t) @@ -501,9 +501,9 @@ func ParseDescriptionList(text string, index int, depth int) (i int, list *Descr t := strings.TrimSpace(text[lastToken:i]) if len(t) > 0 { if !hasKey { - list.Name = append(list.Name, t) + list.Name = append(list.Name, text[lastToken:i]) } else { - list.Entries = append(list.Entries, t) + list.Entries = append(list.Entries, text[lastToken:i]) } }