S.O.N.G/touhouwiki-mirror
S.O.N.G/touhouwiki-mirror
3
0
Fork 0
Code Issues Pull requests Releases Activity

Added lyrics support

Browse source
This commit is contained in:
DataHoarder 2022-02-20 16:20:11 +01:00
parent aa59f2e357
commit 35b794b8a4
3 changed files with 179 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -26,6 +26,8 @@ Small Golang server that loads all albums in an index at startup and is used to
* Bare CDDB emulator for `query` and `read` commands. Used to query by track count + length of album.
* query cmd ex. `/cddb?cmd=cddb+query+730dec08` (no need to provide TOC, but can be provided for more exact match)
* read cmd ex. `/cddb?cmd=cddb+read+Soundtrack54742+730dec08`
* Fetch parsed track Lyrics entries
* ex. `/lyrics/Clockup_Flowers`
# License

189
server.go
View file

@ -275,20 +275,17 @@ func getArtistEntries(kind string, entries []interface{}) (artists []artistEntry
return
}
func getStringValue(pageName string, v []interface{}) (result []string) {
func getStringValue(pageName string, v []interface{}, trim ...bool) (result []string) {
for _, value := range v {
if text, ok := value.(string); ok {
text = normalizeStringCharacters(text)
if len(text) > 0 {
result = append(result, text)
}
result = append(result, text)
} else if template, ok := value.(*wikiparser.Template); ok {
if template.IsLink {
output := 0
for _, vv := range template.Parameters {
for _, vvv := range getStringValue(pageName, vv) {
for _, vvv := range getStringValue(pageName, vv, trim...) {
vvv = strings.TrimSpace(vvv)
if len(vvv) > 0 {
output++
@ -303,15 +300,15 @@ func getStringValue(pageName string, v []interface{}) (result []string) {
switch strings.ToUpper(template.Name) {
case "H:TITLE":
if val, ok := template.Parameters["0"]; ok && len(val) > 0 {
result = append(result, getStringValue(pageName, val)[0])
result = append(result, getStringValue(pageName, val, trim...)[0])
}
case "LANG":
if val, ok := template.Parameters["1"]; ok && len(val) > 0 {
result = append(result, getStringValue(pageName, val)[0])
result = append(result, getStringValue(pageName, val, trim...)[0])
}
case "GENRE":
if val, ok := template.Parameters["0"]; ok && len(val) > 0 {
result = append(result, getStringValue(pageName, val)[0])
result = append(result, getStringValue(pageName, val, trim...)[0])
}
case "PAGENAME":
fallthrough
@ -322,24 +319,34 @@ func getStringValue(pageName string, v []interface{}) (result []string) {
}
}
} else if html, ok := value.(*wikiparser.HTML); ok && html.Tag != nil {
str := strings.TrimSpace(html.Tag.String())
if len(str) > 0 {
result = append(result, str)
}
result = append(result, html.Tag.String())
} else if _, ok := value.(wikiparser.NewLineToken); ok {
result = append(result, "\n")
} else if link, ok := value.(*wikiparser.Link); ok {
if len(link.Name) > 0 {
result = append(result, getStringValue(pageName, link.Name)...)
result = append(result, getStringValue(pageName, link.Name, trim...)...)
} else {
result = append(result, link.URL)
}
result = append(result, getStringValue(pageName, link.Name)...)
result = append(result, getStringValue(pageName, link.Name, trim...)...)
} else if unorderedList, ok := value.(*wikiparser.UnorderedList); ok {
result = append(result, getStringValue(pageName, unorderedList.Entries)...)
result = append(result, getStringValue(pageName, unorderedList.Entries, trim...)...)
} else if descriptionList, ok := value.(*wikiparser.DescriptionList); ok {
result = append(result, strings.Join(getStringValue("", descriptionList.Name), ", ")+": "+strings.Join(getStringValue(pageName, descriptionList.Entries), ", "))
result = append(result, strings.Join(getStringValue("", descriptionList.Name, trim...), ", ")+": "+strings.Join(getStringValue(pageName, descriptionList.Entries, trim...), ", "))
}
}
if len(trim) == 0 || trim[0] == true {
var newResults []string
for _, e := range result {
e = normalizeStringCharacters(e)
if len(e) > 0 {
newResults = append(newResults, e)
}
}
result = newResults
}
return
}
@ -564,8 +571,9 @@ func processIndexDirectory(filePath, indexPath, kind string, wg *sync.WaitGroup)
}
if lyrics, ok := trackTpl.Parameters["lyrics"]; ok {
//TODO: parse lyrics
track.Lyrics = wikiparser.NormalizeWikiTitle(getStringValue(entry.MainTitle, lyrics)[0])
if stringVal = getStringValue(entry.MainTitle, lyrics); ok && len(stringVal) > 0 {
track.Lyrics = wikiparser.NormalizeWikiTitle(strings.TrimSpace(strings.TrimPrefix(stringVal[0], "Lyrics:")))
}
}
if len(listVal.Entries) > 1 {
@ -699,6 +707,135 @@ func processIndex(filePath string) {
wg.Wait()
}
type Lyrics struct {
MainTitle string `json:"pagetitle"`
Titles []string `json:"titles"`
Links []string `json:"links,omitempty"`
Duration int `json:"duration,omitempty"`
Artists []artistEntry `json:"artists"`
Entries struct {
Kanji []string `json:"kanji,omitempty"`
Romaji []string `json:"romaji,omitempty"`
English []string `json:"english,omitempty"`
} `json:"entries"`
}
func parseLyrics(filePath, pageName string) (lyrics *Lyrics) {
if strings.Index(pageName, "/") != -1 {
return
}
var err error
var contents []byte
if contents, err = ioutil.ReadFile(path.Join(filePath, "pages_by_name", "Lyrics:_"+pageName+".wiki")); err != nil {
if contents, err = ioutil.ReadFile(path.Join(filePath, "pages_by_name", "Lyrics:"+pageName+".wiki")); err != nil {
return
}
}
result := wikiparser.ParseWikiText(string(contents))
if len(result) == 0 {
return
}
tpl, ok := result[0].(*wikiparser.Template)
if !ok {
return
}
if tpl.Name != "Lyrics" {
return
}
lyrics = &Lyrics{
MainTitle: strings.ReplaceAll(pageName, "_", " "),
}
//var discLengths []int
//var discTrackNumbers []int
var val []interface{}
var stringVal []string
if val, ok = tpl.Parameters["titleen"]; ok {
if stringVal = getStringValue(lyrics.MainTitle, val); len(stringVal) > 0 {
lyrics.Titles = append(lyrics.Titles, stringVal[0])
}
}
if val, ok = tpl.Parameters["titlejp"]; ok {
if stringVal = getStringValue(lyrics.MainTitle, val); len(stringVal) > 0 {
lyrics.Titles = append(lyrics.Titles, stringVal[0])
}
}
if val, ok = tpl.Parameters["titlerom"]; ok {
if stringVal = getStringValue(lyrics.MainTitle, val); len(stringVal) > 0 {
lyrics.Titles = append(lyrics.Titles, stringVal[0])
}
}
if val, ok = tpl.Parameters["group"]; ok {
lyrics.Artists = append(lyrics.Artists, getArtistEntries("group", val)...)
}
if val, ok = tpl.Parameters["arranger"]; ok {
lyrics.Artists = append(lyrics.Artists, getArtistEntries("arranger", val)...)
}
if val, ok = tpl.Parameters["lyricist"]; ok {
lyrics.Artists = append(lyrics.Artists, getArtistEntries("lyrics", val)...)
}
if val, ok = tpl.Parameters["vocalist"]; ok {
lyrics.Artists = append(lyrics.Artists, getArtistEntries("vocals", val)...)
}
if val, ok = tpl.Parameters["length"]; ok {
if stringVal = getStringValue(lyrics.MainTitle, val); len(stringVal) > 0 {
split := strings.Split(stringVal[0], ":")
numbers := make([]int, len(split))
for i, numVal := range split {
numbers[i], _ = strconv.Atoi(numVal)
}
var duration int
if len(numbers) == 3 {
duration = 3600*numbers[0] + 60*numbers[1] + numbers[2]
} else if len(numbers) == 2 {
duration = 60*numbers[0] + numbers[1]
} else if len(numbers) == 1 {
duration = numbers[0]
}
lyrics.Duration = duration
}
}
var kan []interface{}
var okKan bool
var rom []interface{}
var okRom bool
var eng []interface{}
var okEng bool
for i := 1; ; i++ {
if kan, okKan = tpl.Parameters[fmt.Sprintf("kan%d", i)]; okKan {
lyrics.Entries.Kanji = append(lyrics.Entries.Kanji, normalizeStringCharacters(strings.Join(getStringValue(lyrics.MainTitle, kan, false), "")))
}
if rom, okRom = tpl.Parameters[fmt.Sprintf("rom%d", i)]; okRom {
lyrics.Entries.Romaji = append(lyrics.Entries.Romaji, normalizeStringCharacters(strings.Join(getStringValue(lyrics.MainTitle, rom, false), "")))
}
if eng, okEng = tpl.Parameters[fmt.Sprintf("eng%d", i)]; okEng {
lyrics.Entries.English = append(lyrics.Entries.English, normalizeStringCharacters(strings.Join(getStringValue(lyrics.MainTitle, eng, false), "")))
}
if !okKan && !okRom && !okEng {
break
}
}
return
}
var normalizeSearchTitleTransformer = transform.Chain(
norm.NFKD,
//width.Narrow,
@ -998,6 +1135,20 @@ func main() {
writer.Write([]byte("[]"))
}
} else if strings.Index(request.URL.Path, "/lyrics/") == 0 {
writer.Header().Set("Content-Type", "application/json")
lyrics := parseLyrics(*servePath, wikiparser.NormalizeWikiTitle(strings.Join(strings.Split(request.URL.Path, "/")[2:], "/")))
if lyrics == nil {
writer.WriteHeader(http.StatusNotFound)
writer.Write([]byte("{}"))
return
}
jsonBytes, _ := json.Marshal(lyrics)
writer.Write(jsonBytes)
} else {
filePath := path.Join(*servePath, strings.TrimLeft(request.URL.Path, "/"))
if request.URL.Path == "/" {

14
wikiparser/parser.go
View file

@ -64,7 +64,7 @@ func ParseWikiText(text string) (result []interface{}) {
if templateIndex == -1 && linkIndex == -1 {
t := strings.TrimSpace(text[index:])
if len(t) > 0 {
result = append(result, t)
result = append(result, text[index:])
}
break
} else {
@ -79,7 +79,7 @@ func ParseWikiText(text string) (result []interface{}) {
t := strings.TrimSpace(text[index : index+bestIndex])
if len(t) > 0 {
result = append(result, t)
result = append(result, text[index:index+bestIndex])
}
var tpl *Template
index, tpl = ParseTemplate(text, index+bestIndex+2, 0, text[index+bestIndex])
@ -230,9 +230,9 @@ func ParseTemplate(text string, index int, depth int, startCharacter byte) (i in
template = NewTemplate(t, startCharacter == '[')
} else {
if key == "" {
template.AddParameterUnkeyed(t)
template.AddParameterUnkeyed(text[lastToken:i])
} else {
template.AddParameter(key, t)
template.AddParameter(key, text[lastToken:i])
}
}
}
@ -385,7 +385,7 @@ func ParseUnorderedList(text string, index int, depth int, indent int, startChar
if lastToken < len(text) && i-lastToken > 0 {
t := strings.TrimSpace(text[lastToken:i])
if len(t) > 0 {
currentValue = append(currentValue, t)
currentValue = append(currentValue, text[lastToken:i])
}
return len(t)
@ -501,9 +501,9 @@ func ParseDescriptionList(text string, index int, depth int) (i int, list *Descr
t := strings.TrimSpace(text[lastToken:i])
if len(t) > 0 {
if !hasKey {
list.Name = append(list.Name, t)
list.Name = append(list.Name, text[lastToken:i])
} else {
list.Entries = append(list.Entries, t)
list.Entries = append(list.Entries, text[lastToken:i])
}
}